[FFmpeg-trac] #990(avcodec:open): ffmpeg segfaults in ff_msmpeg4_encode_block with "-r 12 -codec msmpeg4 -mbd rd" options

[FFmpeg]

#990: ffmpeg segfaults in ff_msmpeg4_encode_block with "-r 12 -codec msmpeg4 -mbd
rd" options
-------------------------------------+-------------------------------------
             Reporter:               |                    Owner:
  tom.vercauteren@…                  |                   Status:  open
                 Type:  defect       |                Component:  avcodec
             Priority:  normal       |               Resolution:
              Version:  git-master   |               Blocked By:
             Keywords:  crash        |  Reproduced by developer:  1
  SIGSEGV msmpeg4                    |
             Blocking:               |
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
Changes (by cehoyos):

 * keywords:   => crash SIGSEGV msmpeg4
 * status:  new => open
 * version:  0.10 => git-master
 * reproduced:  0 => 1


Comment:

 {{{
 $ (gdb) r -r 12 -i test-%04d.png -codec msmpeg4 -mbd rd test.avi
 Starting program: ffmpeg_g -r 12 -i test-%04d.png -codec msmpeg4 -mbd rd
 test.avi
 ffmpeg version N-37711-g1b7ff90 Copyright (c) 2000-2012 the FFmpeg
 developers
   built on Feb 10 2012 10:47:16 with gcc 4.5.3
   configuration: --cc=/usr/local/gcc-4.5.3/bin/gcc
   libavutil      51. 39.100 / 51. 39.100
   libavcodec     54.  1.100 / 54.  1.100
   libavformat    54.  0.100 / 54.  0.100
   libavdevice    53.  4.100 / 53.  4.100
   libavfilter     2. 62.100 /  2. 62.100
   libswscale      2.  1.100 /  2.  1.100
   libswresample   0.  6.100 /  0.  6.100
 Input #0, image2, from 'test-%04d.png':
   Duration: 00:00:00.25, start: 0.000000, bitrate: N/A
     Stream #0:0: Video: png, rgba, 912x912, 12 fps, 12 tbr, 12 tbn, 12 tbc
 Incompatible pixel format 'rgba' for codec 'msmpeg4', auto-selecting
 format 'yuv420p'
 [buffer @ 0x13d7200] w:912 h:912 pixfmt:rgba tb:1/1000000 sar:0/1
 sws_param:
 [buffersink @ 0x13d7600] auto-inserting filter 'auto-inserted scale 0'
 between the filter 'src' and the filter 'out'
 [scale @ 0x13d7ee0] w:912 h:912 fmt:rgba -> w:912 h:912 fmt:yuv420p
 flags:0x4
 Output #0, avi, to 'test.avi':
   Metadata:
     ISFT            : Lavf54.0.100
     Stream #0:0: Video: msmpeg4 (hq) (MP43 / 0x3334504D), yuv420p,
 912x912, q=2-31, 200 kb/s, 12 tbn, 12 tbc
 Stream mapping:
   Stream #0:0 -> #0:0 (png -> msmpeg4)
 Press [q] to stop, [?] for help

 Program received signal SIGSEGV, Segmentation fault.
 get_rl_index (level=1, run=0, last=0, rl=0xe44c58) at libavcodec/rl.h:78
 78          index = rl->index_run[last][run];
 (gdb) disass $pc-32,$pc+32
 Dump of assembler code for function get_rl_index:
 0x00000000007c6d6a <get_rl_index+0>:    lea    0x4(%r12),%rcx
 0x00000000007c6d6f <get_rl_index+5>:    mov    (%r9),%ebp
 0x00000000007c6d72 <get_rl_index+8>:    mov    (%r9,%rcx,8),%rcx
 0x00000000007c6d76 <get_rl_index+12>:   mov    %ebp,%r13d
 0x00000000007c6d79 <get_rl_index+15>:   movzbl (%rcx,%r14,1),%ecx
 0x00000000007c6d7e <get_rl_index+20>:   cmp    %ebp,%ecx
 0x00000000007c6d80 <get_rl_index+22>:   jge    0x7c6d9a
 <ff_msmpeg4_encode_block+666>
 0x00000000007c6d82 <get_rl_index+24>:   lea    0x6(%r12),%rbx
 0x00000000007c6d87 <get_rl_index+29>:   lea    -0x1(%rsi,%rcx,1),%ecx
 0x00000000007c6d8b <get_rl_index+33>:   mov    (%r9,%rbx,8),%rbx
 0x00000000007c6d8f <get_rl_index+37>:   movsbl (%rbx,%r14,1),%ebx
 0x00000000007c6d94 <get_rl_index+42>:   cmp    %ebx,%esi
 0x00000000007c6d96 <get_rl_index+44>:   cmovle %ecx,%r13d
 End of assembler dump.
 (gdb) info registers
 rax            0x1      1
 rbx            0x1459bb0        21339056
 rcx            0x0      0
 rdx            0xffffffff       4294967295
 rsi            0x1      1
 rdi            0x13d3671        20788849
 rbp            0x3      0x3
 rsp            0x7ffffffee268   0x7ffffffee268
 r8             0x13d34e0        20788448
 r9             0xe44c58 14961752
 r10            0x0      0
 r11            0x82     130
 r12            0x0      0
 r13            0x3      3
 r14            0x0      0
 r15            0x41     65
 rip            0x7c6d79 0x7c6d79 <get_rl_index+15>
 eflags         0x10202  [ IF RF ]
 cs             0x33     51
 ss             0x2b     43
 ds             0x0      0
 es             0x0      0
 fs             0x0      0
 gs             0x0      0
 fctrl          0x37f    895
 fstat          0x0      0
 ftag           0xaaaa   43690
 fiseg          0x0      0
 fioff          0x0      0
 foseg          0x0      0
 fooff          0x0      0
 fop            0x0      0
 mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]
 }}}

-- 
Ticket URL: <https://ffmpeg.org/trac/ffmpeg/ticket/990#comment:1>
FFmpeg <http://ffmpeg.org>
FFmpeg issue tracker