[FFmpeg-devel] [PATCH] avfilter/vf_overlay: fix alpha blending when main source has an alpha channel

[Peter Große]

Use alpha value from alpha channel instead of selected component channel.

Signed-off-by: Peter Große <pegro at friiks.de>
---
 libavfilter/vf_overlay.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
index 5c9f590957..cde09f5fb3 100644
--- a/libavfilter/vf_overlay.c
+++ b/libavfilter/vf_overlay.c
@@ -508,7 +508,7 @@ static av_always_inline void blend_plane(AVFilterContext *ctx,
     int dst_hp = AV_CEIL_RSHIFT(dst_h, vsub);
     int yp = y>>vsub;
     int xp = x>>hsub;
-    uint8_t *s, *sp, *d, *dp, *a, *ap;
+    uint8_t *s, *sp, *d, *dp, *a, *ap, *da, *dap;
     int jmax, j, k, kmax;
 
     j = FFMAX(-yp, 0);
@@ -517,12 +517,16 @@ static av_always_inline void blend_plane(AVFilterContext *ctx,
                       + (yp+j)    * dst->linesize[dst_plane]
                       + dst_offset;
     ap = src->data[3] + (j<<vsub) * src->linesize[3];
+    if (main_has_alpha)
+        dap = dst->data[3] + (j<<vsub) * dst->linesize[3];
 
     for (jmax = FFMIN(-yp + dst_hp, src_hp); j < jmax; j++) {
         k = FFMAX(-xp, 0);
         d = dp + (xp+k) * dst_step;
         s = sp + k;
         a = ap + (k<<hsub);
+        if (main_has_alpha)
+            da = dap + (k<<hsub);
 
         for (kmax = FFMIN(-xp + dst_wp, src_wp); k < kmax; k++) {
             int alpha_v, alpha_h, alpha;
@@ -545,16 +549,16 @@ static av_always_inline void blend_plane(AVFilterContext *ctx,
                 // average alpha for color components, improve quality
                 uint8_t alpha_d;
                 if (hsub && vsub && j+1 < src_hp && k+1 < src_wp) {
-                    alpha_d = (d[0] + d[src->linesize[3]] +
-                               d[1] + d[src->linesize[3]+1]) >> 2;
+                    alpha_d = (da[0] + da[src->linesize[3]] +
+                               da[1] + da[src->linesize[3]+1]) >> 2;
                 } else if (hsub || vsub) {
                     alpha_h = hsub && k+1 < src_wp ?
-                        (d[0] + d[1]) >> 1 : d[0];
+                        (da[0] + da[1]) >> 1 : da[0];
                     alpha_v = vsub && j+1 < src_hp ?
-                        (d[0] + d[src->linesize[3]]) >> 1 : d[0];
+                        (da[0] + da[src->linesize[3]]) >> 1 : da[0];
                     alpha_d = (alpha_v + alpha_h) >> 1;
                 } else
-                    alpha_d = d[0];
+                    alpha_d = da[0];
                 alpha = UNPREMULTIPLY_ALPHA(alpha, alpha_d);
             }
             *d = FAST_DIV255(*d * (255 - alpha) + *s * alpha);
-- 
2.13.0