[FFmpeg-cvslog] sws: Check for unsupported combination of ED/FULL_CHR flags

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Sat Jan 19 21:43:54 2013 +0100| [aef5c0b68f31c4f4ff2654a8439dccf511c182df] | committer: Michael Niedermayer

sws: Check for unsupported combination of ED/FULL_CHR flags

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=aef5c0b68f31c4f4ff2654a8439dccf511c182df
---

 libswscale/utils.c |   17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/libswscale/utils.c b/libswscale/utils.c
index b61a4ae..e96f255 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -1000,17 +1000,24 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
         }
     }
 
-    if (flags & SWS_ERROR_DIFFUSION && !(flags & SWS_FULL_CHR_H_INT)) {
-        if(dstFormat == AV_PIX_FMT_BGR4_BYTE ||
-           dstFormat == AV_PIX_FMT_RGB4_BYTE ||
-           dstFormat == AV_PIX_FMT_BGR8 ||
-           dstFormat == AV_PIX_FMT_RGB8) {
+    if(dstFormat == AV_PIX_FMT_BGR4_BYTE ||
+       dstFormat == AV_PIX_FMT_RGB4_BYTE ||
+       dstFormat == AV_PIX_FMT_BGR8 ||
+       dstFormat == AV_PIX_FMT_RGB8) {
+        if (flags & SWS_ERROR_DIFFUSION && !(flags & SWS_FULL_CHR_H_INT)) {
             av_log(c, AV_LOG_DEBUG,
                 "Error diffusion dither is only supported in full chroma interpolation for destination format '%s'\n",
                 av_get_pix_fmt_name(dstFormat));
             flags   |= SWS_FULL_CHR_H_INT;
             c->flags = flags;
         }
+        if (!(flags & SWS_ERROR_DIFFUSION) && (flags & SWS_FULL_CHR_H_INT)) {
+            av_log(c, AV_LOG_DEBUG,
+                "Ordered dither is not supported in full chroma interpolation for destination format '%s'\n",
+                av_get_pix_fmt_name(dstFormat));
+            flags   |= SWS_ERROR_DIFFUSION;
+            c->flags = flags;
+        }
     }
 
     /* reuse chroma for 2 pixels RGB/BGR unless user wants full