[FFmpeg-cvslog] swscale: fix overflow in 16-bit vertical scaling.
Ronald S. Bultje
git at videolan.org
Sun Jul 10 04:33:40 CEST 2011
ffmpeg | branch: master | Ronald S. Bultje <rsbultje at gmail.com> | Fri Jul 8 14:57:07 2011 -0700| [7d7bacf0f1f43f5cc112977fc3b8438e3e4a4a9b] | committer: Ronald S. Bultje
swscale: fix overflow in 16-bit vertical scaling.
We operated on 31-bits, but with e.g. lanczos scaling, values can
add up to beyond 0x80000000, thus leading to output of zeroes. Drop
one bit of precision fixes this.
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7d7bacf0f1f43f5cc112977fc3b8438e3e4a4a9b
---
libswscale/swscale.c | 18 +++++++++---------
tests/ref/lavfi/pixdesc | 8 ++++----
tests/ref/lavfi/pixfmts_copy | 8 ++++----
tests/ref/lavfi/pixfmts_crop | 8 ++++----
tests/ref/lavfi/pixfmts_hflip | 8 ++++----
tests/ref/lavfi/pixfmts_null | 8 ++++----
tests/ref/lavfi/pixfmts_scale | 16 ++++++++--------
tests/ref/lavfi/pixfmts_vflip | 8 ++++----
8 files changed, 41 insertions(+), 41 deletions(-)
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 527dd80..db4d231 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -195,7 +195,7 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
int i;
uint16_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
*aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
- int shift = 15 + 16 - output_bits;
+ int shift = 15 + 16 - output_bits - 1;
#define output_pixel(pos, val) \
if (big_endian) { \
@@ -212,24 +212,24 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
} \
}
for (i = 0; i < dstW; i++) {
- int val = 1 << (30-output_bits);
+ int val = 1 << (30-output_bits - 1);
int j;
for (j = 0; j < lumFilterSize; j++)
- val += lumSrc[j][i] * lumFilter[j];
+ val += (lumSrc[j][i] * lumFilter[j]) >> 1;
output_pixel(&yDest[i], val);
}
if (uDest) {
for (i = 0; i < chrDstW; i++) {
- int u = 1 << (30-output_bits);
- int v = 1 << (30-output_bits);
+ int u = 1 << (30-output_bits - 1);
+ int v = 1 << (30-output_bits - 1);
int j;
for (j = 0; j < chrFilterSize; j++) {
- u += chrUSrc[j][i] * chrFilter[j];
- v += chrVSrc[j][i] * chrFilter[j];
+ u += (chrUSrc[j][i] * chrFilter[j]) >> 1;
+ v += (chrVSrc[j][i] * chrFilter[j]) >> 1;
}
output_pixel(&uDest[i], u);
@@ -239,11 +239,11 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
if (CONFIG_SWSCALE_ALPHA && aDest) {
for (i = 0; i < dstW; i++) {
- int val = 1 << (30-output_bits);
+ int val = 1 << (30-output_bits - 1);
int j;
for (j = 0; j < lumFilterSize; j++)
- val += alpSrc[j][i] * lumFilter[j];
+ val += (alpSrc[j][i] * lumFilter[j]) >> 1;
output_pixel(&aDest[i], val);
}
diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc
index 76d845a..a82f8ed 100644
--- a/tests/ref/lavfi/pixdesc
+++ b/tests/ref/lavfi/pixdesc
@@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc
yuv420p9be ce880fa07830e5297c22acf6e20555ce
yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a
yuv422p c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be 86ad3447f97969ce095afeef81fa9abf
-yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le f181c8d8436f1233ba566d9bc88005ec
yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf
yuv444p 0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le f2117fc9cf66d3a832183e159ff71803
+yuv444p16be af555dbaa401b142a995566864f47545
+yuv444p16le a803e8016997dad95c5b2a72f54c34d6
yuva420p a29884f3f3dfe1e00b961bc17bef3d47
yuvj420p 32eec78ba51857b16ce9b813a49b7189
yuvj422p 0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy
index 76d845a..a82f8ed 100644
--- a/tests/ref/lavfi/pixfmts_copy
+++ b/tests/ref/lavfi/pixfmts_copy
@@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc
yuv420p9be ce880fa07830e5297c22acf6e20555ce
yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a
yuv422p c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be 86ad3447f97969ce095afeef81fa9abf
-yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le f181c8d8436f1233ba566d9bc88005ec
yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf
yuv444p 0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le f2117fc9cf66d3a832183e159ff71803
+yuv444p16be af555dbaa401b142a995566864f47545
+yuv444p16le a803e8016997dad95c5b2a72f54c34d6
yuva420p a29884f3f3dfe1e00b961bc17bef3d47
yuvj420p 32eec78ba51857b16ce9b813a49b7189
yuvj422p 0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_crop b/tests/ref/lavfi/pixfmts_crop
index fb5c838..e3bb88c 100644
--- a/tests/ref/lavfi/pixfmts_crop
+++ b/tests/ref/lavfi/pixfmts_crop
@@ -29,12 +29,12 @@ yuv420p bfea0188ddd4889787c403caae119cc7
yuv420p16be 8365eff38b8c329aeb95fc605fa229bb
yuv420p16le 5e8dd38d973d5854abe1ad4efad20cc1
yuv422p f2f930a91fe00d4252c4720b5ecd8961
-yuv422p16be 6647fe1c381c148f8207c988c0e22bf0
-yuv422p16le e1548c9dc51202db38a9625c8954203f
+yuv422p16be 93f9b6f33f9529db6de6a9f0ddd70eb5
+yuv422p16le 2e66dcfec54ca6b57aa4bbd9ac234639
yuv440p 2472417d980e395ad6843cbb8b633b29
yuv444p 1f151980486848c96bc5585ced99003e
-yuv444p16be 02d78b564a23df2f68cf6895d3bfe6bf
-yuv444p16le cbea9591b954ea31d6a0cb25a9aed599
+yuv444p16be e7d1ecf0c11a41b5db192f761f55bd3c
+yuv444p16le 3298a0043d982e7cf1a33a1292fa11f0
yuva420p 7536753dfbc7932560fb50c921369a0e
yuvj420p 21f891093006d42d7683b0e1d773a657
yuvj422p 9a43d474c407590ad8f213880586b45e
diff --git a/tests/ref/lavfi/pixfmts_hflip b/tests/ref/lavfi/pixfmts_hflip
index 4a32e35..2084d58 100644
--- a/tests/ref/lavfi/pixfmts_hflip
+++ b/tests/ref/lavfi/pixfmts_hflip
@@ -29,12 +29,12 @@ yuv420p 2d5c80f9ba2ddd85b2aeda3564cc7d64
yuv420p16be 758b0c1e2113b15e7afde48da4e4d024
yuv420p16le 480ccd951dcb806bc875d307e02e50a0
yuv422p 6e728f4eb9eae287c224f396d84be6ea
-yuv422p16be 9dbe0af0eb877987611cf04bfa577202
-yuv422p16le 2d8f37231110177cc5e1b61c8cb4b163
+yuv422p16be 8657d2c8d443940300fdb4028d555631
+yuv422p16le 4ab27609981e50de5b1150125718ae76
yuv440p a99e2b57ed601f39852715c9d675d0d3
yuv444p 947e47f7bb5fdccc659d19b7df2b6fc3
-yuv444p16be 38cdb28061ebb8e7aa3797238615e77f
-yuv444p16le 80d654986d6f3754e924310a045fdb24
+yuv444p16be a5154ce329db0d2caf0bd43f1347dba3
+yuv444p16le 1f703308b90feb048191b3bccc695671
yuva420p d83ec0c01498189f179ec574918185f1
yuvj420p df3aaaec3bb157c3bde5f0365af30f4f
yuvj422p d113871528d510a192797af59df9c05c
diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null
index 76d845a..a82f8ed 100644
--- a/tests/ref/lavfi/pixfmts_null
+++ b/tests/ref/lavfi/pixfmts_null
@@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc
yuv420p9be ce880fa07830e5297c22acf6e20555ce
yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a
yuv422p c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be 86ad3447f97969ce095afeef81fa9abf
-yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le f181c8d8436f1233ba566d9bc88005ec
yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf
yuv444p 0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le f2117fc9cf66d3a832183e159ff71803
+yuv444p16be af555dbaa401b142a995566864f47545
+yuv444p16le a803e8016997dad95c5b2a72f54c34d6
yuva420p a29884f3f3dfe1e00b961bc17bef3d47
yuvj420p 32eec78ba51857b16ce9b813a49b7189
yuvj422p 0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale
index 14c748f..094e524 100644
--- a/tests/ref/lavfi/pixfmts_scale
+++ b/tests/ref/lavfi/pixfmts_scale
@@ -31,19 +31,19 @@ uyvy422 314bd486277111a95d9369b944fa0400
yuv410p 7df8f6d69b56a8dcb6c7ee908e5018b5
yuv411p 1143e7c5cc28fe0922b051b17733bc4c
yuv420p fdad2d8df8985e3d17e73c71f713cb14
-yuv420p10be aad747a7634ba4ed48f149cdfc78792e
-yuv420p10le 46547f19a7f58638dff73657477b11b9
-yuv420p16be d7270efce54eb59c7b01c14157a1b890
-yuv420p16le e85abf00bad940a922b623c91c9026d7
+yuv420p10be af5429f27b9f95bf955e795921c65cdc
+yuv420p10le d0b47e6a8a44e6b5ca0fe4349a4e393b
+yuv420p16be 9688e33e03b8c8275ab2fb1df0f06bee
+yuv420p16le cba8b390ad5e7b8678e419b8ce79c008
yuv420p9be a073b2d93b2a7dce2069ba252bc43175
yuv420p9le b67233c3c7d93763d07d88f697c145e1
yuv422p 918e37701ee7377d16a8a6c119c56a40
-yuv422p16be 5cd8fe1da161d21b65bf75bf4cb50c75
-yuv422p16le 3b545317b1f5e627751525fb2958d88e
+yuv422p16be 2cf502d7d386db1f1b3b946679d897b1
+yuv422p16le 3002a4e47520731dcee5929aff49eb74
yuv440p 461503fdb9b90451020aa3b25ddf041c
yuv444p 81b2eba962d12e8d64f003ac56f6faf2
-yuv444p16be 2bd3e992d1533f8e0978a8e0c7008243
-yuv444p16le daee5a461ae3bc53295c392e423aa1d7
+yuv444p16be b9f051ce7335923fe33efd162e48da1d
+yuv444p16le fa47e317efac988b4a7fa55141c89126
yuva420p 8673a9131fb47de69788863f93a50eb7
yuvj420p 30427bd6caf5bda93a173dbebe759e09
yuvj422p fc8288f64fd149573f73cf8da05d8e6d
diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip
index b440efa..8702eca 100644
--- a/tests/ref/lavfi/pixfmts_vflip
+++ b/tests/ref/lavfi/pixfmts_vflip
@@ -38,12 +38,12 @@ yuv420p16le 0f609e588e5a258644ef85170d70e030
yuv420p9be be40ec975fb2873891643cbbbddbc3b0
yuv420p9le 7e606310d3f5ff12badf911e8f333471
yuv422p d7f5cb44d9b0210d66d6a8762640ab34
-yuv422p16be 8cdfbddf2dd4c44c3efef4ee00170eba
-yuv422p16le a2f421f6a1af950544081c1797de01ae
+yuv422p16be 51d9aa4e78d121c226d919ce97976fe4
+yuv422p16le 12965c54bda8932ca72da194419a9908
yuv440p 876385e96165acf51271b20e5d85a416
yuv444p 9c3c667d1613b72d15bc6d851c5eb8f7
-yuv444p16be bb86de32c67dd49469989ac184b89592
-yuv444p16le 64b899f52d820b14d05ff95954c15790
+yuv444p16be 6502abd75030d462c58d99a8673ec517
+yuv444p16le cd7e88b6d08425450a57555bc86ab210
yuva420p c705d1cf061d8c6580ac690b55f92276
yuvj420p 41fd02b204da0ab62452cd14b595e2e4
yuvj422p 7f6ca9bc1812cde02036d7d29a7cce43
More information about the ffmpeg-cvslog
mailing list