[FFmpeg-devel] [PATCH 3/3] avcodec/vp9dsp_template: Fix integer overflow(s) in iadst16_1d()
Michael Niedermayer
michael at niedermayer.cc
Mon May 25 01:38:43 EEST 2020
Fixes: signed integer overflow: 1080285923 - -1130879337 cannot be represented in type 'int'
Fixes: 22002/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-6260237310099456
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
libavcodec/vp9dsp_template.c | 152 +++++++++++++++++------------------
1 file changed, 76 insertions(+), 76 deletions(-)
diff --git a/libavcodec/vp9dsp_template.c b/libavcodec/vp9dsp_template.c
index c3273dd726..c6944f5ce3 100644
--- a/libavcodec/vp9dsp_template.c
+++ b/libavcodec/vp9dsp_template.c
@@ -1378,48 +1378,48 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
dctint t0a, t1a, t2a, t3a, t4a, t5a, t6a, t7a;
dctint t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
- t0 = IN(15) * 16364 + IN(0) * 804;
- t1 = IN(15) * 804 - IN(0) * 16364;
- t2 = IN(13) * 15893 + IN(2) * 3981;
- t3 = IN(13) * 3981 - IN(2) * 15893;
- t4 = IN(11) * 14811 + IN(4) * 7005;
- t5 = IN(11) * 7005 - IN(4) * 14811;
- t6 = IN(9) * 13160 + IN(6) * 9760;
- t7 = IN(9) * 9760 - IN(6) * 13160;
- t8 = IN(7) * 11003 + IN(8) * 12140;
- t9 = IN(7) * 12140 - IN(8) * 11003;
- t10 = IN(5) * 8423 + IN(10) * 14053;
- t11 = IN(5) * 14053 - IN(10) * 8423;
- t12 = IN(3) * 5520 + IN(12) * 15426;
- t13 = IN(3) * 15426 - IN(12) * 5520;
- t14 = IN(1) * 2404 + IN(14) * 16207;
- t15 = IN(1) * 16207 - IN(14) * 2404;
-
- t0a = (t0 + t8 + (1 << 13)) >> 14;
- t1a = (t1 + t9 + (1 << 13)) >> 14;
- t2a = (t2 + t10 + (1 << 13)) >> 14;
- t3a = (t3 + t11 + (1 << 13)) >> 14;
- t4a = (t4 + t12 + (1 << 13)) >> 14;
- t5a = (t5 + t13 + (1 << 13)) >> 14;
- t6a = (t6 + t14 + (1 << 13)) >> 14;
- t7a = (t7 + t15 + (1 << 13)) >> 14;
- t8a = (t0 - t8 + (1 << 13)) >> 14;
- t9a = (t1 - t9 + (1 << 13)) >> 14;
- t10a = (t2 - t10 + (1 << 13)) >> 14;
- t11a = (t3 - t11 + (1 << 13)) >> 14;
- t12a = (t4 - t12 + (1 << 13)) >> 14;
- t13a = (t5 - t13 + (1 << 13)) >> 14;
- t14a = (t6 - t14 + (1 << 13)) >> 14;
- t15a = (t7 - t15 + (1 << 13)) >> 14;
-
- t8 = t8a * 16069 + t9a * 3196;
- t9 = t8a * 3196 - t9a * 16069;
- t10 = t10a * 9102 + t11a * 13623;
- t11 = t10a * 13623 - t11a * 9102;
- t12 = t13a * 16069 - t12a * 3196;
- t13 = t13a * 3196 + t12a * 16069;
- t14 = t15a * 9102 - t14a * 13623;
- t15 = t15a * 13623 + t14a * 9102;
+ t0 = IN(15) * 16364U + IN(0) * 804U;
+ t1 = IN(15) * 804U - IN(0) * 16364U;
+ t2 = IN(13) * 15893U + IN(2) * 3981U;
+ t3 = IN(13) * 3981U - IN(2) * 15893U;
+ t4 = IN(11) * 14811U + IN(4) * 7005U;
+ t5 = IN(11) * 7005U - IN(4) * 14811U;
+ t6 = IN(9) * 13160U + IN(6) * 9760U;
+ t7 = IN(9) * 9760U - IN(6) * 13160U;
+ t8 = IN(7) * 11003U + IN(8) * 12140U;
+ t9 = IN(7) * 12140U - IN(8) * 11003U;
+ t10 = IN(5) * 8423U + IN(10) * 14053U;
+ t11 = IN(5) * 14053U - IN(10) * 8423U;
+ t12 = IN(3) * 5520U + IN(12) * 15426U;
+ t13 = IN(3) * 15426U - IN(12) * 5520U;
+ t14 = IN(1) * 2404U + IN(14) * 16207U;
+ t15 = IN(1) * 16207U - IN(14) * 2404U;
+
+ t0a = (dctint)((1U << 13) + t0 + t8 ) >> 14;
+ t1a = (dctint)((1U << 13) + t1 + t9 ) >> 14;
+ t2a = (dctint)((1U << 13) + t2 + t10) >> 14;
+ t3a = (dctint)((1U << 13) + t3 + t11) >> 14;
+ t4a = (dctint)((1U << 13) + t4 + t12) >> 14;
+ t5a = (dctint)((1U << 13) + t5 + t13) >> 14;
+ t6a = (dctint)((1U << 13) + t6 + t14) >> 14;
+ t7a = (dctint)((1U << 13) + t7 + t15) >> 14;
+ t8a = (dctint)((1U << 13) + t0 - t8 ) >> 14;
+ t9a = (dctint)((1U << 13) + t1 - t9 ) >> 14;
+ t10a = (dctint)((1U << 13) + t2 - t10) >> 14;
+ t11a = (dctint)((1U << 13) + t3 - t11) >> 14;
+ t12a = (dctint)((1U << 13) + t4 - t12) >> 14;
+ t13a = (dctint)((1U << 13) + t5 - t13) >> 14;
+ t14a = (dctint)((1U << 13) + t6 - t14) >> 14;
+ t15a = (dctint)((1U << 13) + t7 - t15) >> 14;
+
+ t8 = t8a * 16069U + t9a * 3196U;
+ t9 = t8a * 3196U - t9a * 16069U;
+ t10 = t10a * 9102U + t11a * 13623U;
+ t11 = t10a * 13623U - t11a * 9102U;
+ t12 = t13a * 16069U - t12a * 3196U;
+ t13 = t13a * 3196U + t12a * 16069U;
+ t14 = t15a * 9102U - t14a * 13623U;
+ t15 = t15a * 13623U + t14a * 9102U;
t0 = t0a + t4a;
t1 = t1a + t5a;
@@ -1429,49 +1429,49 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
t5 = t1a - t5a;
t6 = t2a - t6a;
t7 = t3a - t7a;
- t8a = (t8 + t12 + (1 << 13)) >> 14;
- t9a = (t9 + t13 + (1 << 13)) >> 14;
- t10a = (t10 + t14 + (1 << 13)) >> 14;
- t11a = (t11 + t15 + (1 << 13)) >> 14;
- t12a = (t8 - t12 + (1 << 13)) >> 14;
- t13a = (t9 - t13 + (1 << 13)) >> 14;
- t14a = (t10 - t14 + (1 << 13)) >> 14;
- t15a = (t11 - t15 + (1 << 13)) >> 14;
-
- t4a = t4 * 15137 + t5 * 6270;
- t5a = t4 * 6270 - t5 * 15137;
- t6a = t7 * 15137 - t6 * 6270;
- t7a = t7 * 6270 + t6 * 15137;
- t12 = t12a * 15137 + t13a * 6270;
- t13 = t12a * 6270 - t13a * 15137;
- t14 = t15a * 15137 - t14a * 6270;
- t15 = t15a * 6270 + t14a * 15137;
+ t8a = (dctint)((1U << 13) + t8 + t12) >> 14;
+ t9a = (dctint)((1U << 13) + t9 + t13) >> 14;
+ t10a = (dctint)((1U << 13) + t10 + t14) >> 14;
+ t11a = (dctint)((1U << 13) + t11 + t15) >> 14;
+ t12a = (dctint)((1U << 13) + t8 - t12) >> 14;
+ t13a = (dctint)((1U << 13) + t9 - t13) >> 14;
+ t14a = (dctint)((1U << 13) + t10 - t14) >> 14;
+ t15a = (dctint)((1U << 13) + t11 - t15) >> 14;
+
+ t4a = t4 * 15137U + t5 * 6270U;
+ t5a = t4 * 6270U - t5 * 15137U;
+ t6a = t7 * 15137U - t6 * 6270U;
+ t7a = t7 * 6270U + t6 * 15137U;
+ t12 = t12a * 15137U + t13a * 6270U;
+ t13 = t12a * 6270U - t13a * 15137U;
+ t14 = t15a * 15137U - t14a * 6270U;
+ t15 = t15a * 6270U + t14a * 15137U;
out[ 0] = t0 + t2;
out[15] = -(t1 + t3);
t2a = t0 - t2;
t3a = t1 - t3;
- out[ 3] = -((t4a + t6a + (1 << 13)) >> 14);
- out[12] = (t5a + t7a + (1 << 13)) >> 14;
- t6 = (t4a - t6a + (1 << 13)) >> 14;
- t7 = (t5a - t7a + (1 << 13)) >> 14;
+ out[ 3] = -((dctint)((1U << 13) + t4a + t6a) >> 14);
+ out[12] = (dctint)((1U << 13) + t5a + t7a) >> 14;
+ t6 = (dctint)((1U << 13) + t4a - t6a) >> 14;
+ t7 = (dctint)((1U << 13) + t5a - t7a) >> 14;
out[ 1] = -(t8a + t10a);
out[14] = t9a + t11a;
t10 = t8a - t10a;
t11 = t9a - t11a;
- out[ 2] = (t12 + t14 + (1 << 13)) >> 14;
- out[13] = -((t13 + t15 + (1 << 13)) >> 14);
- t14a = (t12 - t14 + (1 << 13)) >> 14;
- t15a = (t13 - t15 + (1 << 13)) >> 14;
-
- out[ 7] = ((t2a + t3a) * -11585 + (1 << 13)) >> 14;
- out[ 8] = ((t2a - t3a) * 11585 + (1 << 13)) >> 14;
- out[ 4] = ((t7 + t6) * 11585 + (1 << 13)) >> 14;
- out[11] = ((t7 - t6) * 11585 + (1 << 13)) >> 14;
- out[ 6] = ((t11 + t10) * 11585 + (1 << 13)) >> 14;
- out[ 9] = ((t11 - t10) * 11585 + (1 << 13)) >> 14;
- out[ 5] = ((t14a + t15a) * -11585 + (1 << 13)) >> 14;
- out[10] = ((t14a - t15a) * 11585 + (1 << 13)) >> 14;
+ out[ 2] = (dctint)((1U << 13) + t12 + t14) >> 14;
+ out[13] = -((dctint)((1U << 13) + t13 + t15) >> 14);
+ t14a = (dctint)((1U << 13) + t12 - t14) >> 14;
+ t15a = (dctint)((1U << 13) + t13 - t15) >> 14;
+
+ out[ 7] = (dctint)(-(t2a + t3a) * 11585U + (1 << 13)) >> 14;
+ out[ 8] = (dctint)( (t2a - t3a) * 11585U + (1 << 13)) >> 14;
+ out[ 4] = (dctint)( (t7 + t6) * 11585U + (1 << 13)) >> 14;
+ out[11] = (dctint)( (t7 - t6) * 11585U + (1 << 13)) >> 14;
+ out[ 6] = (dctint)( (t11 + t10) * 11585U + (1 << 13)) >> 14;
+ out[ 9] = (dctint)( (t11 - t10) * 11585U + (1 << 13)) >> 14;
+ out[ 5] = (dctint)(-(t14a + t15a) * 11585U + (1 << 13)) >> 14;
+ out[10] = (dctint)( (t14a - t15a) * 11585U + (1 << 13)) >> 14;
}
itxfm_wrap(16, 6)
--
2.17.1
More information about the ffmpeg-devel
mailing list