[Ffmpeg-devel] h264 playback crash

grenola ffmpeg
Fri May 5 11:28:52 CEST 2006

Hi there,

I have recently become aware of a bug in ffmpeg's h264 decoding 
routines. The bug is seemingly triggered when one piece of h264 is 
appended to another piece of h264 and the resulting output file fed to 
ffmpeg. The bug has been tested using appended samples created both 
using avidemux and VirtualDub, and the result is the same. The bug also 
affects playback using vlc and mplayer.

The following gdb trace was derived using a CVS snapshot from the 5th of 
May 2006. There follows a similar dump created using mplayer (compiled 
against the same ffmpeg CVS snapshot) which shows the bug a bit more 
clearly (in particular see the very end of this email).

Any workarounds (i.e. a way to append 264 video files that does not 
result in this bug being triggered) would be very helpful.

You may download a crashing testcase here:



(gdb) run
Starting program: /grenola/ffmpeg-cvs-20060505/bin/ffmpeg -i 
/home/ali/Desktop/nate/loller.avi /home/ali/Desktop/nate/loller_enc.avi
(no debugging symbols found)
(no debugging symbols found)
(no debugging symbols found)
(no debugging symbols found)
FFmpeg version CVS, Copyright (c) 2000-2004 Fabrice Bellard
   configuration:  --prefix=/grenola/ffmpeg
   libavutil version: 49.0.0
   libavcodec version: 51.9.0
   libavformat version: 50.4.0
   built on May  5 2006 09:56:09, gcc: 4.0.2 20050808 (prerelease) 
(Ubuntu 4.0.1-4ubuntu9)
Input #0, avi, from '/home/ali/Desktop/nate/loller.avi':
   Duration: 00:00:05.9, start: 0.000000, bitrate: 251 kb/s
   Stream #0.0, 29.97 fps(r): Video: h264, yuv420p, 640x480
   Stream #0.1: Audio: mp3, 48000 Hz, stereo, 128 kb/s
Output #0, avi, to '/home/ali/Desktop/nate/loller_enc.avi':
   Stream #0.0, 29.97 fps(c): Video: mpeg4, yuv420p, 640x480, q=2-31, 
200 kb/s
   Stream #0.1: Audio: mp2, 48000 Hz, stereo, 64 kb/s
Stream mapping:
   Stream #0.0 -> #0.0
   Stream #0.1 -> #0.1
Press [q] to stop encoding
frame=   64 q=2.0 size=     346kB time=2.1 bitrate=1328.3kbits/s 
frame=  146 q=31.0 size=     609kB time=4.9 bitrate=1023.9kbits/s
Program received signal SIGSEGV, Segmentation fault.
0x0000000000599d84 in ff_h261_encode_mb ()
(gdb) bt
#0  0x0000000000599d84 in ff_h261_encode_mb ()
#1  0x000000000059f573 in ff_h261_encode_mb ()
#2  0x00000000005a0973 in ff_h261_encode_mb ()
#3  0x00000000005a1989 in ff_h261_encode_mb ()
#4  0x00000000005a2784 in ff_h261_encode_mb ()
#5  0x00000000004643b0 in avcodec_decode_video ()
#6  0x00000000004153d4 in parse_arg_file ()
#7  0x0000000000417b10 in main ()
(gdb) disass $pc-32 $pc+32
Dump of assembler code from 0x599d64 to 0x599da4:
0x0000000000599d64 <ff_h261_encode_mb+109908>:	add    %al,(%rax)
0x0000000000599d66 <ff_h261_encode_mb+109910>:	cmp 
0x0000000000599d6a <ff_h261_encode_mb+109914>:	jl     0x59a0f0 
0x0000000000599d70 <ff_h261_encode_mb+109920>:	sub 
0x0000000000599d74 <ff_h261_encode_mb+109924>:	mov 
0x0000000000599d79 <ff_h261_encode_mb+109929>:	movslq %r12d,%rdx
0x0000000000599d7c <ff_h261_encode_mb+109932>:	mov    $0x20,%eax
0x0000000000599d81 <ff_h261_encode_mb+109937>:	mov    %esi,(%r10)
0x0000000000599d84 <ff_h261_encode_mb+109940>:	sub    (%rcx,%rdx,4),%eax
0x0000000000599d87 <ff_h261_encode_mb+109943>:	shr    $0x6,%eax
0x0000000000599d8a <ff_h261_encode_mb+109946>:	mov    %ax,(%r15,%rdx,2)
0x0000000000599d8f <ff_h261_encode_mb+109951>:	inc    %r14d
0x0000000000599d92 <ff_h261_encode_mb+109954>:	dec    %ebp
0x0000000000599d94 <ff_h261_encode_mb+109956>:	js     0x599b6c 
0x0000000000599d9a <ff_h261_encode_mb+109962>:	xor    %eax,%eax
0x0000000000599d9c <ff_h261_encode_mb+109964>:	test   %r13d,%r13d
0x0000000000599d9f <ff_h261_encode_mb+109967>:	jne    0x599c45 
End of assembler dump.
(gdb) info all-registers
rax            0x20	32
rbx            0x2b753eb5a010	47782563258384
rcx            0x1b00	6912
rdx            0x0	0
rsi            0xcbd0	52176
rdi            0x0	0
rbp            0x0	0x0
rsp            0x7fffffd67c68	0x7fffffd67c68
r8             0x3f	63
r9             0x2b753eb9bfd9	47782563528665
r10            0x2b753eb9bae8	47782563527400
r11            0x0	0
r12            0x0	0
r13            0x0	0
r14            0x1	1
r15            0x2b753eb9b7e8	47782563526632
rip            0x599d84	0x599d84 <ff_h261_encode_mb+109940>
eflags         0x10202	66050
cs             0x33	51
ss             0x2b	43
ds             0x0	0
es             0x0	0
fs             0x0	0
gs             0x0	0
st0            <invalid float value>	(raw 0xffff0000000000000000)
st1            -nan(0x1212121213131313)	(raw 0xffff1212121213131313)
st2            -nan(0x1313131313131313)	(raw 0xffff1313131313131313)
st3            <invalid float value>	(raw 0xffff0000000000000000)
st4            -nan(0xc0c0c0c0c0c0c0c)	(raw 0xffff0c0c0c0c0c0c0c0c)
st5            -nan(0x101010100000000)	(raw 0xffff0101010100000000)
st6            -nan(0x101010100000000)	(raw 0xffff0101010100000000)
st7            <invalid float value>	(raw 0xffff0000000000000000)
fctrl          0x37f	895
fstat          0x0	0
ftag           0xaaaa	43690
fiseg          0x0	0
fioff          0x0	0
foseg          0x0	0
fooff          0x0	0
fop            0x0	0
xmm0           {f = {0xfffffffe, 0xd, 0x0, 0x0}}	{f = {-2, 13.3261728, 
0, 0}}
xmm1           {f = {0x0, 0x3, 0x0, 0x0}}	{f = {0, 3.48828125, 0, 0}}
xmm2           {f = {0x0, 0x2, 0x0, 0x0}}	{f = {0.988000035, 2.34949994, 
-1.75331087e+38, -nan(0x90cdd)}}
xmm3           {f = {0x0, 0x2, 0x0, 0x0}}	{f = {1.53749397e-15, 
2.31072736, 1.24715563e-43, 1.2751816e-43}}
xmm4           {f = {0x0, 0xd, 0x0, 0xfffff904}}	{f = {-0, 13.1853304, 
-4.57506084e-21, -1788.73804}}
xmm5           {f = {0xc24b42, 0x0, 0x0, 0x0}}	{f = {12733250, 
1.97132904e-26, 2.06927913e-14, -2.28904872e+26}}
xmm6           {f = {0x0, 0x0, 0x0, 0x0}}	{f = {9.18354962e-41, 
9.18354962e-41, 9.18354962e-41, 9.18354962e-41}}
xmm7           {f = {0x0, 0x0, 0x0, 0x0}}	{f = {1.48485789e-39, 
9.2925342e-39, 5.08481607e-39, 9.30358484e-39}}
xmm8           {f = {0x0, 0x0, 0x0, 0x0}}	{f = {-7.4649608e-34, 
0.118344754, 0, 0}}
xmm9           {f = {0x0, 0x1, 0x0, 0x0}}	{f = {0, 1.75, 2.37215456e-12, 
xmm10          {f = {0x0, 0x1, 0x0, 0x0}}	{f = {-3.40421439e+23, 
1.74999988, 4.73870248e-30, 0.672393799}}
xmm11          {f = {0x0, 0xd, 0x73a249, 0xffffffff}}	{f = {0, 
13.0393038, 7578185, -1.29953289}}
xmm12          {f = {0x0, 0x0, 0x0, 0x0}}	{f = {0, 0, 0, 0}}
---Type <return> to continue, or q <return> to quit---
xmm13          {f = {0x0, 0x1, 0x0, 0xffffffff}}	{f = {0, 1.15420532, 
-2.13419931e+15, -1.83225775}}
xmm14          {f = {0x0, 0x26, 0x0, 0x0}}	{f = {0, 38, 0, 0}}
xmm15          {f = {0x0, 0x2, 0x0, 0x0}}	{f = {1.60990027e-20, 
2.47653866, 0, 0}}
mxcsr          0x1fa0	8096


This dump was created using mplayer compiled against the same ffmpeg CVS 
snapshot (5th May):


(gdb) set args /home/ali/Desktop/loller.avi
(gdb) run
Starting program: /grenola/mplayer-cvs20060505-debug/bin/mplayer 
[Thread debugging using libthread_db enabled]
[New Thread 47916107783232 (LWP 15164)]
MPlayer dev-CVS-060505-09:01-4.0.2 (C) 2000-2006 MPlayer Team
CPU: Advanced Micro Devices Athlon 64 Newcastle,Winchester,San 
Diego,Venice; Sempron Palermo (Family: 15, Stepping: 2)
CPUflags:  MMX: 1 MMX2: 1 3DNow: 1 3DNow2: 1 SSE: 1 SSE2: 1
Compiled for x86 CPU with extensions: MMX MMX2 3DNow 3DNowEx SSE SSE2

Linux RTC init error in ioctl (rtc_irqp_set 1024): Permission denied
Try adding "echo 1024 > /proc/sys/dev/rtc/max-user-freq" to your system 
startup scripts.

Playing /home/ali/Desktop/loller.avi.
AVI file format detected.
VIDEO:  [H264]  640x480  24bpp  29.970 fps   99.4 kbps (12.1 kbyte/s)
Opening audio decoder: [mp3lib] MPEG layer-2, layer-3
AUDIO: 48000 Hz, 2 ch, s16le, 128.0 kbit/8.33% (ratio: 16000->192000)
Selected audio codec: [mp3] afm: mp3lib (mp3lib MPEG layer-2, layer-3)
Opening video decoder: [ffmpeg] FFmpeg's libavcodec codec family
Selected video codec: [ffh264] vfm: ffmpeg (FFmpeg H.264)
[AO OSS] audio_setup: Can't open audio device /dev/sound/dsp: No such 
file or directory
alsa-init: using device default
alsa-lib: confmisc.c:560:(snd_determine_driver) could not open control 
for card 2
alsa-lib: conf.c:3479:(_snd_config_evaluate) function 
snd_func_card_driver returned error: No such file or directory
alsa-lib: confmisc.c:392:(snd_func_concat) error evaluating strings
alsa-lib: conf.c:3479:(_snd_config_evaluate) function snd_func_concat 
returned error: No such file or directory
alsa-lib: confmisc.c:955:(snd_func_refer) error evaluating name
alsa-lib: conf.c:3479:(_snd_config_evaluate) function snd_func_refer 
returned error: No such file or directory
alsa-lib: conf.c:3948:(snd_config_expand) Evaluate error: No such file 
or directory
alsa-lib: pcm.c:2090:(snd_pcm_open_noupdate) Unknown PCM default
alsa-init: playback open error: No such file or directory
[AO ARTS] loading the aRts backend "/usr/lib/libartscbackend.la" failed
[AO ESD] esd_open_sound failed: No such file or directory
ao_nas: init(): Can't open nas audio server -> nosound
[AO SDL] Samplerate: 48000Hz Channels: Stereo Format s16le
[New Thread 1082132832 (LWP 15167)]
AO: [sdl] 48000Hz 2ch s16le (2 bytes per sample)
Starting playback...
A:   0.0 V:   0.0 A-V: -0.033 ct:  0.000   1/  1 ??% ??% ??,?% 0 0 [J 
VDec: vo config request - 640 x 480 (preferred colorspace: Planar YV12)
VDec: using Planar YV12 as output csp (no 0)
Movie-Aspect is undefined - no prescaling applied.
VO: [xv] 640x480 => 640x480 Planar YV12
New_Face failed. Maybe the font path is wrong.
Please supply the text font file (~/.mplayer/subfont.ttf).
subtitle font: load_sub_face failed.
A:   0.1 V:   0.1 A-V:  0.040 ct:  0.001   2/  2 ??% ??% ??,?% 0 0

[... snip ...]

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 47916107783232 (LWP 15164)]
0x00000000006d2f5a in decode_cabac_residual (h=0x2b9456aa0010, 
block=0x2b9456ae17e8, cat=5, n=0, scantable=0x2b9456ae21c8 "", 
qmul=0x1b00, max_coeff=64)
     at h264.c:5750
5750	                if( get_cabac_bypass( &h->cabac ) ) block[j] = 
(-qmul[j] + 32) >> 6;
(gdb) bt
#0  0x00000000006d2f5a in decode_cabac_residual (h=0x2b9456aa0010, 
block=0x2b9456ae17e8, cat=5, n=0, scantable=0x2b9456ae21c8 "", 
qmul=0x1b00, max_coeff=64)
     at h264.c:5750
#1  0x00000000006d72dd in decode_mb_cabac (h=0x2b9456aa0010) at h264.c:6224
#2  0x00000000006d8610 in decode_slice (h=Variable "h" is not available.
) at h264.c:6886
#3  0x00000000006d9139 in decode_nal_units (h=0x2b9456aa0010, 
buf=0xc3dc50 "", buf_size=759) at h264.c:7620
#4  0x00000000006d9e04 in decode_frame (avctx=0xc58330, data=0xc58210, 
data_size=0x7fffffe5407c, buf=0xc3dc50 "", buf_size=759) at h264.c:7763
#5  0x00000000005aa2e1 in avcodec_decode_video (avctx=0xc58330, 
picture=0xc58210, got_picture_ptr=0x7fffffe5407c, buf=0xc3dc50 "", 
     at utils.c:946
#6  0x00000000004c37bb in decode (sh=0xc3b590, data=0xc3dc50, len=759, 
flags=0) at vd_ffmpeg.c:819
#7  0x00000000004be4c4 in decode_video (sh_video=0xc3b590, 
start=0xc3dc50 "", in_size=759, drop_frame=0, 
pts=-9.2233720368547758e+18) at dec_video.c:315
#8  0x0000000000444892 in main (argc=0, argv=Variable "argv" is not 
) at mplayer.c:3730
(gdb) info registers
rax            0x20	32
rbx            0x2b9456ae1ae8	47916109404904
rcx            0x1b00	6912
rdx            0x0	0
rsi            0xcbd0	52176
rdi            0x25800	153600
rbp            0x0	0x0
rsp            0x7fffffe53a20	0x7fffffe53a20
r8             0x2b9456ae21c8	47916109406664
r9             0x1b00	6912
r10            0x2	2
r11            0x5	5
r12            0x0	0
r13            0x40	64
r14            0x0	0
r15            0x2b9456aa0010	47916109135888
rip            0x6d2f5a	0x6d2f5a <decode_cabac_residual+842>
eflags         0x10202	66050
cs             0x33	51
ss             0x2b	43
ds             0x0	0
es             0x0	0
fs             0x0	0
gs             0x0	0
(gdb) disass
Dump of assembler code for function decode_cabac_residual:

[... snip ...]

0x00000000006d2f31 <decode_cabac_residual+801>:	test   %sil,%sil
0x00000000006d2f34 <decode_cabac_residual+804>:	mov    %esi,(%rbx)
0x00000000006d2f36 <decode_cabac_residual+806>:	je     0x6d3215 
0x00000000006d2f3c <decode_cabac_residual+812>:	mov    0x4(%rbx),%eax
0x00000000006d2f3f <decode_cabac_residual+815>:	cmp    %eax,%esi
0x00000000006d2f41 <decode_cabac_residual+817>:	jl     0x6d31d5 
0x00000000006d2f47 <decode_cabac_residual+823>:	movslq 0x54(%rsp),%rdx
0x00000000006d2f4c <decode_cabac_residual+828>:	mov    0x8(%rsp),%rcx
0x00000000006d2f51 <decode_cabac_residual+833>:	sub    %eax,%esi
0x00000000006d2f53 <decode_cabac_residual+835>:	mov    %esi,(%rbx)
0x00000000006d2f55 <decode_cabac_residual+837>:	mov    $0x20,%eax
0x00000000006d2f5a <decode_cabac_residual+842>:	sub    (%rcx,%rdx,4),%eax
0x00000000006d2f5d <decode_cabac_residual+845>:	mov    0x20(%rsp),%rcx
0x00000000006d2f62 <decode_cabac_residual+850>:	shr    $0x6,%eax
0x00000000006d2f65 <decode_cabac_residual+853>:	mov    %ax,(%rcx,%rdx,2)
0x00000000006d2f69 <decode_cabac_residual+857>:	incl   0x30(%rsp)
---Type <return> to continue, or q <return> to quit---
0x00000000006d2f6d <decode_cabac_residual+861>:	decl   0x2c(%rsp)
0x00000000006d2f71 <decode_cabac_residual+865>:	js     0x6d2d46 
0x00000000006d2f77 <decode_cabac_residual+871>:	mov    0x34(%rsp),%r10d
0x00000000006d2f7c <decode_cabac_residual+876>:	xor    %eax,%eax
0x00000000006d2f7e <decode_cabac_residual+878>:	test   %r10d,%r10d
0x00000000006d2f81 <decode_cabac_residual+881>:	jne    0x6d2e88 
0x00000000006d2f87 <decode_cabac_residual+887>:	jmpq   0x6d2e77 
0x00000000006d2f8c <decode_cabac_residual+892>:	cmpl   $0x5,0x34(%rsp)
0x00000000006d2f91 <decode_cabac_residual+897>:	mov    $0x4,%eax

[... snip ...]

End of assembler dump.
(gdb) print qmul[j]
Cannot access memory at address 0x1b00
(gdb) print j
$1 = 0
(gdb) print qmul
$2 = (const uint32_t *) 0x1b00

More information about the ffmpeg-devel mailing list