[FFmpeg-devel] [PATCH v3 1/2] lavc/texturedsp: fix premult2straight inversion

[Connor Worley]

This function should convert premultiplied alpha to straight, but does the opposite.

Signed-off-by: Connor Worley <connorbworley at gmail.com>
---
 libavcodec/texturedsp.c | 9 ++++++---
 tests/ref/fate/dds-dxt2 | 2 +-
 tests/ref/fate/dds-dxt4 | 2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/libavcodec/texturedsp.c b/libavcodec/texturedsp.c
index 5fb79937da..721bfc17f5 100644
--- a/libavcodec/texturedsp.c
+++ b/libavcodec/texturedsp.c
@@ -175,9 +175,12 @@ static av_always_inline void premult2straight(uint8_t *src)
     int b = src[2];
     int a = src[3]; /* unchanged */
 
-    src[0] = (uint8_t) r * a / 255;
-    src[1] = (uint8_t) g * a / 255;
-    src[2] = (uint8_t) b * a / 255;
+    if (a == 0)
+        return;
+
+    src[0] = (uint8_t) FFMIN(r * 255 / a, 255);
+    src[1] = (uint8_t) FFMIN(g * 255 / a, 255);
+    src[2] = (uint8_t) FFMIN(b * 255 / a, 255);
 }
 
 /**
diff --git a/tests/ref/fate/dds-dxt2 b/tests/ref/fate/dds-dxt2
index b5bdfbadc3..1744ef41f6 100644
--- a/tests/ref/fate/dds-dxt2
+++ b/tests/ref/fate/dds-dxt2
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 64x64
 #sar 0: 0/1
-0,          0,          0,        1,    16384, 0x11cebeb0
+0,          0,          0,        1,    16384, 0xd7f7241b
diff --git a/tests/ref/fate/dds-dxt4 b/tests/ref/fate/dds-dxt4
index 136dfd8006..f22878da56 100644
--- a/tests/ref/fate/dds-dxt4
+++ b/tests/ref/fate/dds-dxt4
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 64x64
 #sar 0: 0/1
-0,          0,          0,        1,    16384, 0x31aaacd6
+0,          0,          0,        1,    16384, 0xf18d4216
-- 
2.40.1