[FFmpeg-trac] #10234(ffmpeg:new): Assertion qmin <= qmax at ratecontrol.c:123
FFmpeg
trac at avcodec.org
Wed Mar 8 12:21:02 EET 2023
#10234: Assertion qmin <= qmax at ratecontrol.c:123
-------------------------------------+-------------------------------------
Reporter: Youngseok | Type: defect
Choi |
Status: new | Priority: normal
Component: ffmpeg | Version: git-
| master
Keywords: | Blocked By:
Blocking: | Reproduced by developer: 0
Analyzed by developer: 0 |
-------------------------------------+-------------------------------------
Hi, we are developing a new fuzz testing feature, and it found a assertion
violation on ffmpeg.
How to reproduce:
{{{
% ./ffmpeg -i <input_file> -f mp4 -lmax 1 e
ffmpeg version N-109968-gcc76e8340d (git-master)
built on Ubuntu 18.04.1 with gcc 7.5.0
}}}
You can download <input_file> from https://github.com/3-24/oss-fuzz-
reports/raw/master/ffmpeg/poc_1/poc_file.
Command output:
{{{
ffmpeg version N-109968-gcc76e8340d Copyright (c) 2000-2023 the FFmpeg
developers
built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)
configuration: --extra-cflags='-fsanitize=address -g -O0' --extra-
cxxflags='-fsanitize=address -g -O0' --extra-ldflags='-fsanitize=address
-g -O0'
libavutil 58. 3.100 / 58. 3.100
libavcodec 60. 6.100 / 60. 6.100
libavformat 60. 4.100 / 60. 4.100
libavdevice 60. 2.100 / 60. 2.100
libavfilter 9. 4.100 / 9. 4.100
libswscale 7. 2.100 / 7. 2.100
libswresample 4. 11.100 / 4. 11.100
[h261 @ 0x617000000080] Format h261 detected only with low score of 25,
misdetection possible!
[h261 @ 0x619000000580] warning: first frame is no keyframe
[h261 @ 0x619000000580] illegal ac vlc code at 6x0
[h261 @ 0x619000000580] Error at MB: 6
Input #0, h261, from 'poc_file':
Duration: N/A, bitrate: N/A
Stream #0:0: Video: h261, yuv420p, 176x144, 29.97 tbr, 1200k tbn
Stream mapping:
Stream #0:0 -> #0:0 (h261 (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
[h261 @ 0x619000001980] warning: first frame is no keyframe
[h261 @ 0x619000001980] illegal ac vlc code at 6x0
[h261 @ 0x619000001980] Error at MB: 6
[mpeg4 @ 0x619000002d80] too many threads/slices (10), reducing to 9
Output #0, mp4, to 'e':
Metadata:
encoder : Lavf60.4.100
Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(progressive),
176x144, q=2-31, 200 kb/s, 29.97 fps, 30k tbn
Metadata:
encoder : Lavc60.6.100 mpeg4
Side data:
cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
Assertion qmin <= qmax failed at libavcodec/ratecontrol.c:123
Aborted (core dumped)
}}}
Backtrace:
{{{
#0 __GI_raise (sig=sig at entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007ffff56207f1 in __GI_abort () at abort.c:79
#2 0x00005555571fef60 in get_qminmax (s=0x625000005100, s=0x625000005100,
s=0x625000005100, pict_type=1, qmax_ret=<synthetic pointer>,
qmin_ret=<synthetic pointer>) at libavcodec/ratecontrol.c:123
#3 ff_rate_estimate_qscale (s=s at entry=0x625000005100, dry_run=<optimized
out>) at libavcodec/ratecontrol.c:885
#4 0x0000555556fd5cb7 in estimate_qp (s=s at entry=0x625000005100,
dry_run=dry_run at entry=0) at libavcodec/mpegvideo_enc.c:3525
#5 0x0000555556fd9666 in encode_picture (s=0x625000005100) at
libavcodec/mpegvideo_enc.c:3721
#6 ff_mpv_encode_picture (avctx=<optimized out>, pkt=<optimized out>,
pic_arg=<optimized out>, got_packet=<optimized out>)
at libavcodec/mpegvideo_enc.c:1801
#7 0x0000555556a8659b in ff_encode_encode_cb
(avctx=avctx at entry=0x619000002d80, avpkt=avpkt at entry=0x610000002640,
frame=0x616000011d80,
got_packet=got_packet at entry=0x7fffffffcb80) at libavcodec/encode.c:223
#8 0x0000555556a872e6 in encode_simple_internal (avpkt=0x610000002640,
avctx=0x619000002d80) at libavcodec/encode.c:309
#9 encode_simple_receive_packet (avpkt=<optimized out>, avctx=<optimized
out>) at libavcodec/encode.c:323
#10 encode_receive_packet_internal (avctx=avctx at entry=0x619000002d80,
avpkt=0x610000002640) at libavcodec/encode.c:357
#11 0x0000555556a87913 in avcodec_send_frame (avctx=0x619000002d80,
frame=0x616000009080) at libavcodec/encode.c:506
#12 0x0000555555c6dd2d in encode_frame (of=<optimized out>,
ost=0x618000000080, frame=<optimized out>) at fftools/ffmpeg.c:904
#13 0x0000555555c719fe in submit_encode_frame (frame=0x616000009080,
ost=0x618000000080, of=0x611000000900) at fftools/ffmpeg.c:985
#14 do_video_out (of=0x611000000900, ost=0x618000000080,
next_picture=<optimized out>) at fftools/ffmpeg.c:1340
#15 0x0000555555c7335e in reap_filters (flush=<optimized out>) at
fftools/ffmpeg.c:1426
#16 0x0000555555c7b01b in transcode_step () at fftools/ffmpeg.c:4002
#17 transcode () at fftools/ffmpeg.c:4039
#18 0x0000555555bed03e in main (argc=8, argv=0x7fffffffe0d8) at
fftools/ffmpeg.c:4177
}}}
Disassembly around pc:
{{{
(gdb) disass $pc-32,$pc+32
Dump of assembler code from 0x7ffff561ee67 to 0x7ffff561eea7:
0x00007ffff561ee67 <__GI_raise+167>: add %dh,%al
0x00007ffff561ee69 <__GI_raise+169>: (bad)
0x00007ffff561ee6a <__GI_raise+170>: pushq 0x3b(%rdi)
0x00007ffff561ee6d <__GI_raise+173>: mov %eax,%r8d
0x00007ffff561ee70 <__GI_raise+176>: mov $0x8,%r10d
0x00007ffff561ee76 <__GI_raise+182>: xor %edx,%edx
0x00007ffff561ee78 <__GI_raise+184>: mov %r9,%rsi
0x00007ffff561ee7b <__GI_raise+187>: mov $0x2,%edi
0x00007ffff561ee80 <__GI_raise+192>: mov $0xe,%eax
0x00007ffff561ee85 <__GI_raise+197>: syscall
=> 0x00007ffff561ee87 <__GI_raise+199>: mov 0x108(%rsp),%rcx
0x00007ffff561ee8f <__GI_raise+207>: xor %fs:0x28,%rcx
0x00007ffff561ee98 <__GI_raise+216>: mov %r8d,%eax
0x00007ffff561ee9b <__GI_raise+219>: jne 0x7ffff561eebc
<__GI_raise+252>
0x00007ffff561ee9d <__GI_raise+221>: add $0x118,%rsp
0x00007ffff561eea4 <__GI_raise+228>: retq
0x00007ffff561eea5 <__GI_raise+229>: nopl (%rax)
}}}
register info:
{{{
rax 0x0 0
rbx 0xec 236
rcx 0x7ffff561ee87 140737310224007
rdx 0x0 0
rsi 0x7fffffffc2a0 140737488339616
rdi 0x2 2
rbp 0x619000002d80 0x619000002d80
rsp 0x7fffffffc2a0 0x7fffffffc2a0
r8 0x0 0
r9 0x7fffffffc2a0 140737488339616
r10 0x8 8
r11 0x246 582
r12 0x625000005100 108095736926464
r13 0x619000002d80 107271103196544
r14 0x1 1
r15 0x7fffffffc5e0 140737488340448
rip 0x7ffff561ee87 0x7ffff561ee87 <__GI_raise+199>
eflags 0x246 [ PF ZF IF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
st0 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st1 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st2 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st3 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st4 <invalid float value> (raw 0xffff0000000000000000)
st5 <invalid float value> (raw 0xffff0005000500050005)
st6 <invalid float value> (raw 0xffff000a000a000a000a)
st7 -nan(0xfffbfffbfffbfffb) (raw 0xfffffffbfffbfffbfffb)
fctrl 0x37f 895
fstat 0x0 0
ftag 0xffff 65535
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
mxcsr 0x1fa8 [ OE PE IM DM ZM OM UM PM ]
ymm0 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 =
{
0xff <repeats 16 times>, 0x0 <repeats 16 times>}, v16_int16 = {0xffff,
0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0}, v8_int32 = {0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xffffffffffffffff,
0xffffffffffffffff, 0x0, 0x0}, v2_int128 =
{0xffffffffffffffffffffffffffffffff, 0x0}}
ymm1 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x25 <repeats 16 times>,
0x0 <repeats 16 times>}, v16_int16 = {0x2525, 0x2525, 0x2525, 0x2525,
0x2525, 0x2525, 0x2525, 0x2525, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v8_int32 = {0x25252525, 0x25252525, 0x25252525, 0x25252525, 0x0, 0x0,
0x0, 0x0}, v4_int64 = {0x2525252525252525, 0x2525252525252525, 0x0, 0x0},
v2_int128 = {0x25252525252525252525252525252525, 0x0}}
ymm2 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 =
{0x0,
0x0, 0xff, 0xff, 0x0, 0x0, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xff, 0xff, 0x0 <repeats 16 times>}, v16_int16 = {0x0, 0xffff, 0x0,
0xffff,
0x0, 0x0, 0x0, 0xffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v8_int32 = {0xffff0000, 0xffff0000, 0x0, 0xffff0000, 0x0, 0x0, 0x0, 0x0},
v4_int64 = {0xffff0000ffff0000, 0xffff000000000000, 0x0, 0x0}, v2_int128
= {0xffff000000000000ffff0000ffff0000, 0x0}}
ymm3 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
---Type <return> to continue, or q <return> to quit---
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0,
0x0}}
ymm4 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 =
{0x0,
0x0, 0x0, 0x0, 0xff <repeats 12 times>, 0x0 <repeats 16 times>},
v16_int16 = {0x0, 0x0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x0, 0xffffffff,
0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0xffffffff00000000,
0xffffffffffffffff, 0x0, 0x0}, v2_int128 =
{0xffffffffffffffffffffffff00000000, 0x0}}
ymm5 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0,
0x0}}
ymm6 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0,
0x0}}
ymm7 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0,
0x0}}
ymm8 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0,
0x0}}
ymm9 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80,
0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17
times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080,
0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 =
{0x800080008000800080008000800080, 0x0}}
ymm10 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0}, v4_double = {0x7fffffffffffffff, 0x0, 0x0, 0x0}, v32_int8 = {0x84,
0x78,
0x84, 0x83, 0x7d, 0x7d, 0x89, 0x7b, 0x87, 0x76, 0x7f, 0x86, 0x83,
0x78, 0x81, 0x81, 0x0 <repeats 16 times>}, v16_int16 = {0x7884, 0x8384,
0x7d7d,
0x7b89, 0x7687, 0x867f, 0x7883, 0x8181, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0}, v8_int32 = {0x83847884, 0x7b897d7d, 0x867f7687, 0x81817883,
0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x7b897d7d83847884, 0x81817883867f7687,
0x0, 0x0}, v2_int128 = {0x81817883867f76877b897d7d83847884, 0x0}}
ymm11 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x87, 0x0, 0x76, 0x0, 0x7f,
0x0,
0x86, 0x0, 0x83, 0x0, 0x78, 0x0, 0x81, 0x0, 0x81, 0x0 <repeats 17
times>}, v16_int16 = {0x87, 0x76, 0x7f, 0x86, 0x83, 0x78, 0x81, 0x81, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x760087, 0x86007f,
0x780083, 0x810081, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x86007f00760087,
0x81008100780083, 0x0, 0x0}, v2_int128 =
{0x810081007800830086007f00760087, 0x0}}
ymm12 {v8_float = {0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0}, v4_double = {0x7fffffffffffffff, 0x0, 0x0, 0x0}, v32_int8 =
{0x7d,
0x83, 0x84, 0x7a, 0x82, 0x81, 0x82, 0x7e, 0x85, 0x7a, 0x7d, 0x82,
0x88, 0x79, 0x7c, 0x85, 0x0 <repeats 16 times>}, v16_int16 = {0x837d,
0x7a84,
0x8182, 0x7e82, 0x7a85, 0x827d, 0x7988, 0x857c, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x7a84837d, 0x7e828182, 0x827d7a85,
0x857c7988, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x7e8281827a84837d,
0x857c7988827d7a85, 0x0, 0x0}, v2_int128 =
{0x857c7988827d7a857e8281827a84837d,
0x0}}
ymm13 {v8_float = {0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x7fffffffffffffff,
0x7fffffffffffffff,
0x0, 0x0}, v32_int8 = {0x78, 0x89, 0x88, 0x74, 0x80, 0x85, 0x84, 0x7b,
0x76, 0x8b, 0x8a, 0x72, 0x7d, 0x87, 0x87, 0x79, 0x0 <repeats 16 times>},
v16_int16 = {0x8978, 0x7488, 0x8580, 0x7b84, 0x8b76, 0x728a, 0x877d,
0x7987, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x74888978,
0x7b848580, 0x728a8b76, 0x7987877d, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x7b84858074888978, 0x7987877d728a8b76, 0x0, 0x0}, v2_int128 = {
0x7987877d728a8b767b84858074888978, 0x0}}
ymm14 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80,
0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17
times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080,
0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 =
{0x800080008000800080008000800080, 0x0}}
ymm15 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80,
0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17
times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080,
0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 =
{0x800080008000800080008000800080, 0x0}}
}}}
--
Ticket URL: <https://trac.ffmpeg.org/ticket/10234>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker
More information about the FFmpeg-trac
mailing list