[FFmpeg-devel] [PATCHv2] all: do standards compliant absdiff computation

[Ganesh Ajjanagadde]

This resolves implementation defined behavior, and also silences -Wabsolute-value in clang 3.5+.
Moreover, the generated asm is identical to before modulo nop padding.

Signed-off-by: Ganesh Ajjanagadde <gajjanagadde at gmail.com>
---
 libavcodec/flashsv2enc.c | 8 +++++---
 libavfilter/vf_hqx.c     | 8 +++++---
 libavfilter/vf_xbr.c     | 7 ++++---
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/libavcodec/flashsv2enc.c b/libavcodec/flashsv2enc.c
index c2c00f4..65db112 100644
--- a/libavcodec/flashsv2enc.c
+++ b/libavcodec/flashsv2enc.c
@@ -412,12 +412,14 @@ static inline unsigned pixel_color15(const uint8_t * src)
 
 static inline unsigned int chroma_diff(unsigned int c1, unsigned int c2)
 {
+#define ABSDIFF(a,b) (abs((int)(a)-(int)(b)))
+
     unsigned int t1 = (c1 & 0x000000ff) + ((c1 & 0x0000ff00) >> 8) + ((c1 & 0x00ff0000) >> 16);
     unsigned int t2 = (c2 & 0x000000ff) + ((c2 & 0x0000ff00) >> 8) + ((c2 & 0x00ff0000) >> 16);
 
-    return abs(t1 - t2) + abs((c1 & 0x000000ff) - (c2 & 0x000000ff)) +
-        abs(((c1 & 0x0000ff00) >> 8) - ((c2 & 0x0000ff00) >> 8)) +
-        abs(((c1 & 0x00ff0000) >> 16) - ((c2 & 0x00ff0000) >> 16));
+    return ABSDIFF(t1, t2) + ABSDIFF(c1 & 0x000000ff, c2 & 0x000000ff) +
+        ABSDIFF((c1 & 0x0000ff00) >> 8 , (c2 & 0x0000ff00) >> 8) +
+        ABSDIFF((c1 & 0x00ff0000) >> 16, (c2 & 0x00ff0000) >> 16);
 }
 
 static inline int pixel_color7_fast(Palette * palette, unsigned c15)
diff --git a/libavfilter/vf_hqx.c b/libavfilter/vf_hqx.c
index fa15d9c..d1e360f 100644
--- a/libavfilter/vf_hqx.c
+++ b/libavfilter/vf_hqx.c
@@ -65,9 +65,11 @@ static av_always_inline int yuv_diff(uint32_t yuv1, uint32_t yuv2)
 #define YMASK 0xff0000
 #define UMASK 0x00ff00
 #define VMASK 0x0000ff
-    return abs((yuv1 & YMASK) - (yuv2 & YMASK)) > (48 << 16) ||
-           abs((yuv1 & UMASK) - (yuv2 & UMASK)) > ( 7 <<  8) ||
-           abs((yuv1 & VMASK) - (yuv2 & VMASK)) > ( 6 <<  0);
+#define ABSDIFF(a,b) (abs((int)(a)-(int)(b)))
+
+    return ABSDIFF(yuv1 & YMASK, yuv2 & YMASK) > (48 << 16) ||
+           ABSDIFF(yuv1 & UMASK, yuv2 & UMASK) > ( 7 <<  8) ||
+           ABSDIFF(yuv1 & VMASK, yuv2 & VMASK) > ( 6 <<  0);
 }
 
 /* (c1*w1 + c2*w2) >> s */
diff --git a/libavfilter/vf_xbr.c b/libavfilter/vf_xbr.c
index 8586608..c92e9a8 100644
--- a/libavfilter/vf_xbr.c
+++ b/libavfilter/vf_xbr.c
@@ -65,13 +65,14 @@ static uint32_t pixel_diff(uint32_t x, uint32_t y, const uint32_t *r2y)
 #define YMASK 0xff0000
 #define UMASK 0x00ff00
 #define VMASK 0x0000ff
+#define ABSDIFF(a,b) (abs((int)(a)-(int)(b)))
 
     uint32_t yuv1 = r2y[x & 0xffffff];
     uint32_t yuv2 = r2y[y & 0xffffff];
 
-    return (abs((yuv1 & YMASK) - (yuv2 & YMASK)) >> 16) +
-           (abs((yuv1 & UMASK) - (yuv2 & UMASK)) >>  8) +
-           abs((yuv1 & VMASK) - (yuv2 & VMASK));
+    return (ABSDIFF(yuv1 & YMASK, yuv2 & YMASK) >> 16) +
+           (ABSDIFF(yuv1 & UMASK, yuv2 & UMASK) >>  8) +
+            ABSDIFF(yuv1 & VMASK, yuv2 & VMASK);
 }
 
 #define ALPHA_BLEND_128_W(a, b) ((((a) & LB_MASK) >> 1) + (((b) & LB_MASK) >> 1))
-- 
2.5.2