[FFmpeg-trac] #8321(undetermined:new): signed integer overflow at libavcodec/elbg.c:243, 323

[FFmpeg]

#8321: signed integer overflow at libavcodec/elbg.c:243,323
-------------------------------------+-------------------------------------
             Reporter:  Suhwan       |                     Type:  defect
               Status:  new          |                 Priority:  normal
            Component:               |                  Version:  git-
  undetermined                       |  master
             Keywords:  ubsan        |               Blocked By:
             Blocking:               |  Reproduced by developer:  0
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
 Summary of the bug:
 There're 2 signed integer overflow at libavcodec/elbg.c:243,323

 I compiled ffmpeg with "--toolchain=clang-usan" to check the undefined-
 behaviours and attached log file.
 How to reproduce:
 {{{
 % ffmpeg_g  -y -i $PoC -filter_complex agraphmonitor -loglevel 0 -ar 22050
 tmp.roq

 ffmpeg version N-95517-g0b8956b25c Copyright (c) 2000-2019 the FFmpeg
 developers
 built with clang version 6.0.0-1ubuntu2 (tags/RELEASE_600/final)
 configuration: --cc=clang --cxx=clang++ --ld=clang --enable-debug
 --toolchain=clang-usan
 }}}

 Here's UBSAN log

 {{{
 libavcodec/elbg.c:243:24: runtime error: signed integer overflow: 255 *
 12840019 cannot be represented in type 'int'

 Thread 1 "ffmpeg_g" hit Breakpoint 1, 0x00000000004288b0 in
 __ubsan::ScopedReport::~ScopedReport() ()
 (gdb) bt
 #0  0x00000000004288b0 in __ubsan::ScopedReport::~ScopedReport() ()
 #1  0x000000000042b0eb in void
 handleIntegerOverflowImpl<__ubsan::Value>(__ubsan::OverflowData*, unsigned
 long, char const*, __ubsan::Value, __ubsan::ReportOptions) ()
 #2  0x000000000042c9df in __ubsan_handle_mul_overflow ()
 #3  0x0000000001fd6f3d in evaluate_utility_inc (elbg=<optimized out>) at
 libavcodec/elbg.c:243
 #4  0x0000000001fd555e in do_shiftings (elbg=0x180627806e) at
 libavcodec/elbg.c:320
 #5  avpriv_do_elbg (points=<optimized out>, dim=24, numpoints=57600,
 codebook=<optimized out>, numCB=<optimized out>,
     max_steps=1, closest_cb=0x7fffc36fe040, rand_state=0x93fee08) at
 libavcodec/elbg.c:434
 #6  0x0000000002feba56 in generate_codebook (enc=<optimized out>,
 tempdata=<optimized out>, points=<optimized out>,
     inputCount=<optimized out>, results=<optimized out>, size=<optimized
 out>, cbsize=<optimized out>)
     at libavcodec/roqvideoenc.c:806
 #7  0x0000000002fd9f63 in generate_new_codebooks (enc=<optimized out>,
 tempData=<optimized out>)
     at libavcodec/roqvideoenc.c:854
 #8  roq_encode_video (enc=0x93fd7c0) at libavcodec/roqvideoenc.c:907
 #9  0x0000000002fd72bb in roq_encode_frame (avctx=<optimized out>,
 pkt=<optimized out>, frame=<optimized out>,
     got_packet=0x7fffffffc1d4) at libavcodec/roqvideoenc.c:1100
 #10 0x0000000001fe242f in avcodec_encode_video2 (avctx=0x93fd340,
 avpkt=<optimized out>, frame=<optimized out>,
     got_packet_ptr=0x7fffffffc1d4) at libavcodec/encode.c:302
 #11 0x0000000001fe4160 in do_encode (avctx=0x93fd340, frame=0x942bac0,
 got_packet=0x7fffffffc1d4) at libavcodec/encode.c:371
 #12 0x0000000001fe3cda in avcodec_send_frame (avctx=0x93fd340,
 frame=0x942bac0) at libavcodec/encode.c:420
 #13 0x00000000004c51f8 in do_video_out (of=0x93d07c0, ost=<optimized out>,
 next_picture=<optimized out>,
     sync_ipts=3.0000076293945317) at fftools/ffmpeg.c:1287
 #14 0x00000000004c0f2b in reap_filters (flush=0) at fftools/ffmpeg.c:1504
 #15 0x000000000048d682 in transcode_step () at fftools/ffmpeg.c:4638
 #16 transcode () at fftools/ffmpeg.c:4682
 #17 0x0000000000487dc4 in main (argc=26, argv=<optimized out>) at
 fftools/ffmpeg.c:4884
 (gdb) c
 Continuing.
 libavcodec/elbg.c:323:24: runtime error: signed integer overflow: 255 *
 12840019 cannot be represented in type 'int'

 Thread 1 "ffmpeg_g" hit Breakpoint 1, 0x00000000004288b0 in
 __ubsan::ScopedReport::~ScopedReport() ()
 (gdb) bt
 #0  0x00000000004288b0 in __ubsan::ScopedReport::~ScopedReport() ()
 #1  0x000000000042b0eb in void
 handleIntegerOverflowImpl<__ubsan::Value>(__ubsan::OverflowData*, unsigned
 long, char const*, __ubsan::Value, __ubsan::ReportOptions) ()
 #2  0x000000000042c9df in __ubsan_handle_mul_overflow ()
 #3  0x0000000001fd58a7 in do_shiftings (elbg=<optimized out>) at
 libavcodec/elbg.c:323
 #4  avpriv_do_elbg (points=<optimized out>, dim=24, numpoints=57600,
 codebook=<optimized out>, numCB=<optimized out>,
     max_steps=1, closest_cb=0x7fffc36fe040, rand_state=0x93fee08) at
 libavcodec/elbg.c:434
 #5  0x0000000002feba56 in generate_codebook (enc=<optimized out>,
 tempdata=<optimized out>, points=<optimized out>,
     inputCount=<optimized out>, results=<optimized out>, size=<optimized
 out>, cbsize=<optimized out>)
     at libavcodec/roqvideoenc.c:806
 #6  0x0000000002fd9f63 in generate_new_codebooks (enc=<optimized out>,
 tempData=<optimized out>)
     at libavcodec/roqvideoenc.c:854
 #7  roq_encode_video (enc=0x93fd7c0) at libavcodec/roqvideoenc.c:907
 #8  0x0000000002fd72bb in roq_encode_frame (avctx=<optimized out>,
 pkt=<optimized out>, frame=<optimized out>,
     got_packet=0x7fffffffc1d4) at libavcodec/roqvideoenc.c:1100
 #9  0x0000000001fe242f in avcodec_encode_video2 (avctx=0x93fd340,
 avpkt=<optimized out>, frame=<optimized out>,
     got_packet_ptr=0x7fffffffc1d4) at libavcodec/encode.c:302
 #10 0x0000000001fe4160 in do_encode (avctx=0x93fd340, frame=0x942bac0,
 got_packet=0x7fffffffc1d4) at libavcodec/encode.c:371
 #11 0x0000000001fe3cda in avcodec_send_frame (avctx=0x93fd340,
 frame=0x942bac0) at libavcodec/encode.c:420
 #12 0x00000000004c51f8 in do_video_out (of=0x93d07c0, ost=<optimized out>,
 next_picture=<optimized out>,
     sync_ipts=3.0000076293945317) at fftools/ffmpeg.c:1287
 #13 0x00000000004c0f2b in reap_filters (flush=0) at fftools/ffmpeg.c:1504
 #14 0x000000000048d682 in transcode_step () at fftools/ffmpeg.c:4638
 #15 transcode () at fftools/ffmpeg.c:4682
 #16 0x0000000000487dc4 in main (argc=26, argv=<optimized out>) at
 fftools/ffmpeg.c:4884
 (gdb) q

 }}}
 Please confirm.
 Thanks

--
Ticket URL: <https://trac.ffmpeg.org/ticket/8321>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker