[FFmpeg-trac] #3017(avcodec:new): ff_emu_edge_vfix21_sse causes SIGILL on Pentium-III

FFmpeg trac at avcodec.org
Wed Oct 2 00:54:43 CEST 2013

#3017: ff_emu_edge_vfix21_sse causes SIGILL on Pentium-III
             Reporter:  qyot27   |                     Type:  defect
               Status:  new      |                 Priority:  normal
            Component:  avcodec  |                  Version:  git-master
             Keywords:           |               Blocked By:
             Blocking:           |  Reproduced by developer:  0
Analyzed by developer:  0        |
 After building from git yesterday, I noticed that ffmpeg would crash on
 some H.264 files (mostly 8-bit ones) but not others (10-bit, 12-bit,

 The backtrace from a debug build showed that ff_emu_edge_vfix21_sse was
 tripping it.  I'd also tested on an Athlon64 (which didn't exhibit the
 issue, so between that and the SIGILL I knew it was in the assembly).
 Using --disable-asm allows it to work correctly on my Pentium-III era

 Judging from the git log it was more than likely introduced in
 since builds from a month ago don't exhibit the issue.

 The backtrace:
 gdb> r -i "Qyot27 - Daybreak [8bit][H.264+AAC] distro.mkv" -vcodec ffvhuff
 -t 10 test.avi
 Starting program: C:\dap\vid\Incoming Files\ffmpeg\ffmpeg.exe -i "Qyot27 -
 Daybreak [8bit]
 [H.264+AAC] distro.mkv" -vcodec ffvhuff -t 10 test.avi
 [New Thread 3520.0xdd4]
 ffmpeg version N-56805-gdcc92ed Copyright (c) 2000-2013 the FFmpeg
   built on Oct  1 2013 17:29:25 with gcc 4.8.1 (GCC)
   libavutil      52. 46.100 / 52. 46.100
   libavcodec     55. 33.101 / 55. 33.101
   libavformat    55. 18.104 / 55. 18.104
   libavdevice    55.  3.100 / 55.  3.100
   libavfilter     3. 88.100 /  3. 88.100
   libavresample   1.  1.  0 /  1.  1.  0
   libswscale      2.  5.100 /  2.  5.100
   libswresample   0. 17.103 /  0. 17.103
   libpostproc    52.  3.100 / 52.  3.100
 Input #0, matroska,webm, from 'Qyot27 - Daybreak [8bit][H.264+AAC]
     creation_time   : 2013-07-26 22:04:24
   Duration: 00:05:00.47, start: 0.000000, bitrate: 1438 kb/s
     Stream #0:0(eng): Video: h264 (High), yuv420p, 848x480, SAR 1:1 DAR
 53:30, 23.98 fps, 23.98 tbr, 1k tbn, 47.95 tbc (default) (forced)
     Stream #0:1(eng): Audio: aac, 44100 Hz, stereo, fltp (default)
     Stream #0:2: Subtitle: subrip (default) (forced)
       title           : Dummy Subtitles
     Stream #0:3(eng): Subtitle: ssa
       title           : Karaoke Subtitles [ASS]
 Codec 0x18000 is not in the full list.
     Stream #0:4: Attachment: unknown_codec
       filename        : Ubuntu-R.ttf
       mimetype        : application/x-truetype-font
     Stream #0:5: Attachment: text
       filename        : knife.txt
       mimetype        : text/plain
     Stream #0:6: Attachment: png
       filename        : release_poster.png
       mimetype        : image/png
 Output #0, avi, to 'test.avi':
     ISFT            : Lavf55.18.104
     Stream #0:0(eng): Video: ffvhuff (FFVH / 0x48564646), yuv420p, 848x480
 [SAR 1:1 DAR 53:30], q=2-31, 200 kb/s, 23.98 tbn, 23.98 tbc (default)
     Stream #0:1(eng): Audio: mp3 (libmp3lame) (U[0][0][0] / 0x0055), 44100
 Hz, stereo, fltp (default) (forced)
 Stream mapping:
   Stream #0:0 -> #0:0 (h264 -> ffvhuff)
   Stream #0:1 -> #0:1 (aac -> libmp3lame)
 Press [q] to stop, [?] for help

 Program received signal SIGILL, Illegal instruction.
 0x00abf799 in ff_emu_edge_vfix21_sse ()
 (gdb) bt
 #0  0x00abf799 in ff_emu_edge_vfix21_sse ()
 #1  0x00a67f6d in emulated_edge_mc (h_extend_var=<optimized out>,
     hfix_tbl=0x14cd5c0 <hfixtbl_sse>, v_extend_var=<optimized out>,
     vfix_tbl=0x14cd640 <vfixtbl_sse>, h=480, w=138727022, src_y=<optimized
     src_x=<optimized out>, block_h=21, block_w=<optimized out>,
     src_stride=<optimized out>,
     src=0x8711d8b "\027\027\027", '\030' <repeats 17 times>,
 27\027", '\026' <repeats 38 times>, '\027' <repeats 12 times>,
 '\027' <r
 epeats 14 times>, "\026\026\026\026", '\027' <repeats 26 times>,
 \026\026\026\026", '\027' <repeats 19 times>..., dst_stride=<optimized
     dst=<optimized out>) at libavcodec/x86/videodsp_init.c:175
 #2  emulated_edge_mc_sse (buf=0x8334b60 "\027\030", '\027' <repeats 14
     buf_stride=896, src=0x87119f6 "", src_stride=896, block_w=21,
 block_h=21, src_x=6,
     src_y=-1, w=848, h=480) at libavcodec/x86/videodsp_init.c:222
 #3  0x00613cb3 in mc_dir_part (chroma_idc=1, pixel_shift=0,
     chroma_op=0xaad4b0 <ff_put_h264_chroma_mc4_mmx>, qpix_op=0x844bd54,
     src_x_offset=4, dest_cr=0x8870214 "", dest_cb=0x8853ff4 "",
 dest_y=0x87e6298 "",
     list=0, delta=7168, height=16, square=0, n=4, pic=0x847c358,
     at libavcodec/h264.c:935
 #4  mc_part_std (chroma_idc=1, pixel_shift=0, list1=0, list0=8192,
     chroma_avg=0xaadc60 <ff_avg_h264_chroma_mc4_mmxext>,
     chroma_put=0xaad4b0 <ff_put_h264_chroma_mc4_mmx>, qpix_put=0x844bd54,
     x_offset=4, dest_cr=0x8870214 "", dest_cb=0x8853ff4 "",
 dest_y=0x87e6298 "",
     delta=7168, height=16, square=0, n=4, h=0x844bc60) at
 #5  mc_part_420_simple_8 (h=h at entry=0x844bc60, n=n at entry=4,
 square=square at entry=0,
     height=height at entry=16, delta=7168,
     dest_y=dest_y at entry=0x87e6290 "\026\026\026\027\027\027\027\030",
     dest_cb=dest_cb at entry=0x8853ff0 "\200\200\200\200",
     dest_cr=dest_cr at entry=0x8870210 "\177\177\177\177",
 x_offset=x_offset at entry=4,
     y_offset=y_offset at entry=0, qpix_put=qpix_put at entry=0x844bd54,
     chroma_put=0xaad4b0 <ff_put_h264_chroma_mc4_mmx>,
     qpix_avg=qpix_avg at entry=0x844be54,
     chroma_avg=0xaadc60 <ff_avg_h264_chroma_mc4_mmxext>,
     weight_op=weight_op at entry=0x844bc70,
 weight_avg=weight_avg at entry=0x844bc80,
     list0=list0 at entry=8192, list1=0) at libavcodec/h264_mc_template.c:56
 #6  0x0062cdb2 in hl_motion_420_simple_8 (weight_avg=<optimized out>,
     weight_op=<optimized out>, chroma_avg=<optimized out>,
 qpix_avg=<optimized out>,
     chroma_put=<optimized out>, qpix_put=<optimized out>,
 dest_cr=<optimized out>,
     dest_cb=<optimized out>, dest_y=<optimized out>, h=<optimized out>)
     at libavcodec/h264_mc_template.c:98
 #7  hl_decode_mb_simple_8 (h=h at entry=0x844bc60) at
 #8  0x00634d28 in ff_h264_hl_decode_mb (h=0x844bc60) at
 #9  decode_slice (avctx=avctx at entry=0x815e2e0, arg=arg at entry=0x22f7fc)
     at libavcodec/h264.c:4439
 #10 0x0063535f in execute_decode_slices (h=h at entry=0x844bc60,
     context_count=context_count at entry=1) at libavcodec/h264.c:4590
 #11 0x0063d7c9 in decode_nal_units (parse_extradata=0, buf_size=6402,
 buf=0x8531a20 "",
     h=0x844bc60) at libavcodec/h264.c:4942
 #12 decode_frame (avctx=0x815e2e0, data=0x850a500, got_frame=0x22fb2c,
     at libavcodec/h264.c:5079
 #13 0x005eb46f in avcodec_decode_video2 (avctx=<optimized out>,
     picture=<optimized out>, picture at entry=0x850a500,
 got_picture_ptr=<optimized out>,
     got_picture_ptr at entry=0x22fb2c, avpkt=<optimized out>,
 avpkt at entry=0x22fd80)
     at libavcodec/utils.c:1994
 #14 0x0041049f in decode_video (ist=ist at entry=0x8533ac0,
 pkt=pkt at entry=0x22fd80,
     got_output=got_output at entry=0x22fb2c) at ffmpeg.c:1668
 #15 0x00415e4b in output_packet (pkt=0x22fd30, ist=0x8533ac0) at
 #16 process_input (file_index=<optimized out>) at ffmpeg.c:3089
 #17 0x01424d09 in transcode_step () at ffmpeg.c:3185
 #18 transcode () at ffmpeg.c:3237
 #19 main (argc=8, argv=<optimized out>) at ffmpeg.c:3418
 (gdb) disass $pc-32,$pc+32
 Dump of assembler code from 0xabf779 to 0xabf7b9:
    0x00abf779 <ff_emu_edge_vfix21_sse+9>:       adc    $0x8b,%al
    0x00abf77b <ff_emu_edge_vfix21_sse+11>:      push   %esp
    0x00abf77c <ff_emu_edge_vfix21_sse+12>:      and    $0x1c,%al
    0x00abf77e <ff_emu_edge_vfix21_sse+14>:      mov    0x20(%esp),%ebx
    0x00abf782 <ff_emu_edge_vfix21_sse+18>:      mov    0x24(%esp),%esi
    0x00abf786 <ff_emu_edge_vfix21_sse+22>:      sub    %ebx,%esi
    0x00abf788 <ff_emu_edge_vfix21_sse+24>:      sub    %edx,%ebx
    0x00abf78a <ff_emu_edge_vfix21_sse+26>:      test   %edx,%edx
    0x00abf78c <ff_emu_edge_vfix21_sse+28>:      je     0xabf7a5
    0x00abf78e <ff_emu_edge_vfix21_sse+30>:      movups (%ecx),%xmm0
    0x00abf791 <ff_emu_edge_vfix21_sse+33>:      movq   0xd(%ecx),%xmm1
    0x00abf796 <ff_emu_edge_vfix21_sse+38>:      movups %xmm0,(%eax)
 => 0x00abf799 <ff_emu_edge_vfix21_sse+41>:      movq   %xmm1,0xd(%eax)
    0x00abf79e <ff_emu_edge_vfix21_sse+46>:      add    0x10(%esp),%eax
    0x00abf7a2 <ff_emu_edge_vfix21_sse+50>:      dec    %edx
    0x00abf7a3 <ff_emu_edge_vfix21_sse+51>:      jne    0xabf796
    0x00abf7a5 <ff_emu_edge_vfix21_sse+53>:      movups (%ecx),%xmm0
    0x00abf7a8 <ff_emu_edge_vfix21_sse+56>:      movq   0xd(%ecx),%xmm1
    0x00abf7ad <ff_emu_edge_vfix21_sse+61>:      movups %xmm0,(%eax)
    0x00abf7b0 <ff_emu_edge_vfix21_sse+64>:      movq   %xmm1,0xd(%eax)
    0x00abf7b5 <ff_emu_edge_vfix21_sse+69>:      add    0x10(%esp),%eax
 End of assembler dump.

Ticket URL: <https://ffmpeg.org/trac/ffmpeg/ticket/3017>
FFmpeg <http://ffmpeg.org>
FFmpeg issue tracker

More information about the FFmpeg-trac mailing list