[FFmpeg-trac] #7078(avfilter:new): deshake crashes with crop and sse2

FFmpeg trac at avcodec.org
Sat Mar 10 16:26:42 EET 2018


#7078: deshake crashes with crop and sse2
-------------------------------------+-------------------------------------
               Reporter:  cehoyos    |                  Owner:
                   Type:  defect     |                 Status:  new
               Priority:  important  |              Component:  avfilter
                Version:  git-       |               Keywords:  deshake
  master                             |  crash SIGSEGV
             Blocked By:             |               Blocking:
Reproduced by developer:  0          |  Analyzed by developer:  0
-------------------------------------+-------------------------------------
 Alexander found this crash:
 {{{
 (gdb) r -lavfi testsrc2=s=hd1080,crop=1720:1080,deshake -f null -
 Starting program: ffmpeg_g -lavfi testsrc2=s=hd1080,crop=1720:1080,deshake
 -f null -
 [Thread debugging using libthread_db enabled]
 Using host libthread_db library "/lib64/libthread_db.so.1".
 ffmpeg version N-90284-g950170b Copyright (c) 2000-2018 the FFmpeg
 developers
   built with gcc 4.8 (SUSE Linux)
   configuration: --enable-gpl --enable-libx264 --enable-libx265 --enable-
 libvpx --enable-gnutls --enable-libxml2
   libavutil      56.  9.100 / 56.  9.100
   libavcodec     58. 14.100 / 58. 14.100
   libavformat    58. 10.100 / 58. 10.100
   libavdevice    58.  2.100 / 58.  2.100
   libavfilter     7. 12.100 /  7. 12.100
   libswscale      5.  0.102 /  5.  0.102
   libswresample   3.  0.101 /  3.  0.101
   libpostproc    55.  0.100 / 55.  0.100
 Stream mapping:
   deshake -> Stream #0:0 (wrapped_avframe)
 Press [q] to stop, [?] for help
 [New Thread 0x7ffff2980700 (LWP 9116)]
 [New Thread 0x7ffff217f700 (LWP 9117)]
 Output #0, null, to 'pipe:':
   Metadata:
     encoder         : Lavf58.10.100
     Stream #0:0: Video: wrapped_avframe, yuv420p, 1720x1080 [SAR 1:1 DAR
 43:27], q=2-31, 200 kb/s, 25 fps, 25 tbn, 25 tbc (default)
     Metadata:
       encoder         : Lavc58.14.100 wrapped_avframe

 Program received signal SIGSEGV, Segmentation fault.
 0x0000000001072964 in ff_pixelutils_sad_u_16x16_sse2 ()
 (gdb) bt
 #0  0x0000000001072964 in ff_pixelutils_sad_u_16x16_sse2 ()
 #1  0x0000000000507b62 in find_block_motion (mv=<synthetic pointer>,
 stride=1920, cy=16, cx=864,
     src2=0x7ffff17800a4 'Q' <repeats 200 times>..., src1=0x7ffff17800a4
 'Q' <repeats 200 times>...,
     deshake=0x1fb11c0) at libavfilter/vf_deshake.c:136
 #2  find_motion (deshake=deshake at entry=0x1fb11c0, src1=0x7ffff17800a4 'Q'
 <repeats 200 times>...,
     src2=0x7ffff17800a4 'Q' <repeats 200 times>..., width=1720,
 height=1080, stride=1920, t=t at entry=0x7fffffffd200)
     at libavfilter/vf_deshake.c:266
 #3  0x00000000005083ef in filter_frame (link=link at entry=0x1fb0180,
 in=0x1fc1700) at libavfilter/vf_deshake.c:454
 #4  0x00000000004c5745 in ff_filter_frame_framed (frame=0x1fc1700,
 link=0x1fb0180) at libavfilter/avfilter.c:1115
 #5  ff_filter_frame_to_filter (link=0x1fb0180) at
 libavfilter/avfilter.c:1263
 #6  ff_filter_activate_default (filter=<optimized out>) at
 libavfilter/avfilter.c:1312
 #7  ff_filter_activate (filter=<optimized out>) at
 libavfilter/avfilter.c:1473
 #8  0x00000000004c8e4c in ff_filter_graph_run_once (graph=<optimized out>)
 at libavfilter/avfiltergraph.c:1453
 #9  0x00000000004c94f1 in get_frame_internal (samples=0, flags=1,
 frame=0x0, ctx=0x1faf280)
     at libavfilter/buffersink.c:110
 #10 av_buffersink_get_frame_flags (ctx=0x1faf280, frame=frame at entry=0x0,
 flags=flags at entry=1)
     at libavfilter/buffersink.c:121
 #11 0x00000000004c8ba9 in avfilter_graph_request_oldest (graph=0x1faf1c0)
 at libavfilter/avfiltergraph.c:1406
 #12 0x000000000048675e in transcode_from_filter (best_ist=<synthetic
 pointer>, graph=0x1facc00)
     at fftools/ffmpeg.c:4490
 #13 transcode_step () at fftools/ffmpeg.c:4565
 #14 transcode () at fftools/ffmpeg.c:4641
 #15 main (argc=<optimized out>, argv=<optimized out>) at
 fftools/ffmpeg.c:4844
 (gdb) disass $pc-4,$pc+32
 Dump of assembler code from 0x1072960 to 0x1072984:
    0x0000000001072960 <ff_pixelutils_sad_u_16x16_sse2+0>:       movdqu
 (%rdx),%xmm2
 => 0x0000000001072964 <ff_pixelutils_sad_u_16x16_sse2+4>:       psadbw
 (%rdi),%xmm2
    0x0000000001072968 <ff_pixelutils_sad_u_16x16_sse2+8>:       movdqu
 (%rdx,%rcx,1),%xmm1
    0x000000000107296d <ff_pixelutils_sad_u_16x16_sse2+13>:      psadbw
 (%rdi,%rsi,1),%xmm1
    0x0000000001072972 <ff_pixelutils_sad_u_16x16_sse2+18>:      paddw
 %xmm1,%xmm2
    0x0000000001072976 <ff_pixelutils_sad_u_16x16_sse2+22>:      lea
 (%rdi,%rsi,2),%rdi
    0x000000000107297a <ff_pixelutils_sad_u_16x16_sse2+26>:      lea
 (%rdx,%rcx,2),%rdx
    0x000000000107297e <ff_pixelutils_sad_u_16x16_sse2+30>:      movdqu
 (%rdx),%xmm0
    0x0000000001072982 <ff_pixelutils_sad_u_16x16_sse2+34>:      psadbw
 (%rdi),%xmm0
 End of assembler dump.
 (gdb) info all-register
 rax            0xfffffffffffffff0       -16
 rbx            0x780    1920
 rcx            0x780    1920
 rdx            0x7ffff178f414   140737244623892
 rsi            0x780    1920
 rdi            0x7ffff1787c04   140737244593156
 rbp            0x0      0x0
 rsp            0x7fffffffd0a8   0x7fffffffd0a8
 r8             0x357    855
 r9             0x9      9
 r10            0x351    849
 r11            0x352    850
 r12            0x1fb11c0        33231296
 r13            0xfffffff0       4294967280
 r14            0x0      0
 r15            0xfffffff0       4294967280
 rip            0x1072964        0x1072964
 <ff_pixelutils_sad_u_16x16_sse2+4>
 eflags         0x10206  [ PF IF RF ]
 cs             0x33     51
 ss             0x2b     43
 ds             0x0      0
 es             0x0      0
 fs             0x0      0
 gs             0x0      0
 st0            0        (raw 0x00000000000000000000)
 st1            0        (raw 0x00000000000000000000)
 st2            0        (raw 0x00000000000000000000)
 st3            0        (raw 0x00000000000000000000)
 st4            0        (raw 0x00000000000000000000)
 st5            0        (raw 0x00000000000000000000)
 st6            0        (raw 0x00000000000000000000)
 st7            0        (raw 0x00000000000000000000)
 fctrl          0x37f    895
 fstat          0x0      0
 ftag           0xffff   65535
 fiseg          0x0      0
 fioff          0x0      0
 foseg          0x0      0
 fooff          0x0      0
 fop            0x0      0
 xmm0           {v4_float = {0x0, 0x1, 0x0, 0x1}, v2_double = {0x0, 0x0},
 v16_int8 = {0x9a, 0x99, 0x99, 0x99, 0x99,
     0x99, 0xb9, 0x3f, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xb9, 0x3f},
 v8_int16 = {0x999a, 0x9999, 0x9999, 0x3fb9,
     0x999a, 0x9999, 0x9999, 0x3fb9}, v4_int32 = {0x9999999a, 0x3fb99999,
 0x9999999a, 0x3fb99999}, v2_int64 = {
     0x3fb999999999999a, 0x3fb999999999999a}, uint128 =
 0x3fb999999999999a3fb999999999999a}
 xmm1           {v4_float = {0x0, 0x2, 0x0, 0x0}, v2_double = {0x14, 0x0},
 v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0,
     0x0, 0x34, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 =
 {0x0, 0x0, 0x0, 0x4034, 0x0, 0x0, 0x0,
     0x0}, v4_int32 = {0x0, 0x40340000, 0x0, 0x0}, v2_int64 =
 {0x4034000000000000, 0x0},
   uint128 = 0x00000000000000004034000000000000}
 xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x29 <repeats 16 times>},
   v8_int16 = {0x2929, 0x2929, 0x2929, 0x2929, 0x2929, 0x2929, 0x2929,
 0x2929}, v4_int32 = {0x29292929, 0x29292929,
     0x29292929, 0x29292929}, v2_int64 = {0x2929292929292929,
 0x2929292929292929},
   uint128 = 0x29292929292929292929292929292929}
 xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
     0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0}, v8_int16 = {0x0,
 0x0, 0x0, 0xff, 0x0, 0x0, 0xff00, 0x0},
   v4_int32 = {0x0, 0xff0000, 0x0, 0xff00}, v2_int64 = {0xff000000000000,
 0xff0000000000},
   uint128 = 0x0000ff000000000000ff000000000000}
 xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0,
 0x8000000000000000}, v16_int8 = {0x2c, 0x20,
     0x71, 0x3d, 0x32, 0x2d, 0x33, 0x31, 0x2c, 0x20, 0x32, 0x30, 0x30,
 0x20, 0x6b, 0x62}, v8_int16 = {0x202c,
     0x3d71, 0x2d32, 0x3133, 0x202c, 0x3032, 0x2030, 0x626b}, v4_int32 =
 {0x3d71202c, 0x31332d32, 0x3032202c,
     0x626b2030}, v2_int64 = {0x31332d323d71202c, 0x626b20303032202c},
 uint128 = 0x626b20303032202c31332d323d71202c}
 xmm5           {v4_float = {0x0, 0xffffffff, 0x3, 0x3}, v2_double =
 {0xffffffffffffffff, 0x20}, v16_int8 = {0x0,
     0x0, 0x0, 0x0, 0x0, 0x0, 0xf0, 0xbf, 0x40, 0x40, 0x40, 0x40, 0x40,
 0x40, 0x40, 0x40}, v8_int16 = {0x0, 0x0,
     0x0, 0xbff0, 0x4040, 0x4040, 0x4040, 0x4040}, v4_int32 = {0x0,
 0xbff00000, 0x40404040, 0x40404040},
   v2_int64 = {0xbff0000000000000, 0x4040404040404040}, uint128 =
 0x4040404040404040bff0000000000000}
 xmm6           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0xcd, 0xcc, 0xcc, 0x3d, 0xcd,
     0xcc, 0xcc, 0x3d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 =
 {0xcccd, 0x3dcc, 0xcccd, 0x3dcc, 0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x3dcccccd, 0x3dcccccd, 0x0, 0x0},
 v2_int64 = {0x3dcccccd3dcccccd, 0x0},
   uint128 = 0x00000000000000003dcccccd3dcccccd}
 xmm7           {v4_float = {0x3, 0x3, 0x0, 0x0}, v2_double = {0x20, 0x0},
 v16_int8 = {0x0, 0x0, 0x40, 0x40, 0x0,
     0x0, 0x40, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 =
 {0x0, 0x4040, 0x0, 0x4040, 0x0, 0x0, 0x0,
     0x0}, v4_int32 = {0x40400000, 0x40400000, 0x0, 0x0}, v2_int64 =
 {0x4040000040400000, 0x0},
   uint128 = 0x00000000000000004040000040400000}
 xmm8           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm9           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm11          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm12          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0, 0xff,
     0x0 <repeats 14 times>}, v8_int16 = {0xff00, 0x0, 0x0, 0x0, 0x0, 0x0,
 0x0, 0x0}, v4_int32 = {0xff00, 0x0, 0x0,
     0x0}, v2_int64 = {0xff00, 0x0}, uint128 =
 0x0000000000000000000000000000ff00}
 xmm13          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm14          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 xmm15          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
 v16_int8 = {0x0 <repeats 16 times>},
   v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0,
 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
   uint128 = 0x00000000000000000000000000000000}
 mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]
 }}}

--
Ticket URL: <https://trac.ffmpeg.org/ticket/7078>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker


More information about the FFmpeg-trac mailing list