[FFmpeg-trac] #7889(avcodec:new): garbled output with transparent GIF

[FFmpeg]

#7889: garbled output with transparent GIF
-------------------------------------+-------------------------------------
             Reporter:  bjorn        |                     Type:  defect
               Status:  new          |                 Priority:  normal
            Component:  avcodec      |                  Version:
             Keywords:  gif,         |  unspecified
  transparent                        |               Blocked By:
             Blocking:               |  Reproduced by developer:  0
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
 Summary of the bug:
 How to reproduce:
 {{{
 $ ./ffmpeg -i source_sticker.gif -filter_complex
 '[0:v]scale=100:-1,split[a][b];[a]palettegen[p];[b][p]paletteuse' out.gif
 On OS X produces various problems:
 - in FFMPeg 4.1.3 it enters an infinite loop.
 - in FFMPEG master, it produces garbled output.
 }}}

 Here's the patch:
 {{{
 diff --git a/libavcodec/gif.c b/libavcodec/gif.c
 index 94c8b1af49..d7768b4426 100644
 --- a/libavcodec/gif.c
 +++ b/libavcodec/gif.c
 @@ -136,7 +136,7 @@ static void gif_crop_translucent(AVCodecContext
 *avctx,
          while (*y_start < y_end) {
              int is_trans = 1;
              for (int i = 0; i < w; i++) {
 -                if (buf[w * *y_start + i] != trans) {
 +                if (buf[linesize * *y_start + i] != trans) {
                      is_trans = 0;
                      break;
                  }
 @@ -148,10 +148,10 @@ static void gif_crop_translucent(AVCodecContext
 *avctx,
          }

          // crop bottom
 -        while (y_end < h) {
 +        while (y_end > *y_start) {
              int is_trans = 1;
              for (int i = 0; i < w; i++) {
 -                if (buf[w * y_end + i] != trans) {
 +                if (buf[linesize * y_end + i] != trans) {
                      is_trans = 0;
                      break;
                  }
 @@ -165,7 +165,7 @@ static void gif_crop_translucent(AVCodecContext
 *avctx,
          while (*x_start < x_end) {
              int is_trans = 1;
              for (int i = *y_start; i < y_end; i++) {
 -                if (buf[w * i + *x_start] != trans) {
 +                if (buf[linesize * i + *x_start] != trans) {
                      is_trans = 0;
                      break;
                  }
 @@ -176,10 +176,10 @@ static void gif_crop_translucent(AVCodecContext
 *avctx,
          }

          // crop right
 -        while (x_end < w) {
 +        while (x_end > *x_start) {
              int is_trans = 1;
              for (int i = *y_start; i < y_end; i++) {
 -                if (buf[w * i + x_end] != trans) {
 +                if (buf[linesize * i + x_end] != trans) {
                      is_trans = 0;
                      break;
                  }
 }}}

--
Ticket URL: <https://trac.ffmpeg.org/ticket/7889>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker