[FFmpeg-devel] [PATCH] Add missing overflow checks in av_image_fill_pointers() and av_image_fill_linesizes().

[Stefano Sabatini]

On date Wednesday 2010-12-01 02:07:03 +0100, Michael Niedermayer encoded:
> On Fri, Nov 26, 2010 at 04:23:47PM +0100, Stefano Sabatini wrote:
> > ---
> >  libavcore/imgutils.c |   17 ++++++++++++++++-
> >  1 files changed, 16 insertions(+), 1 deletions(-)
> > 
> > diff --git a/libavcore/imgutils.c b/libavcore/imgutils.c
> > index 554639f..a2adaa6 100644
> > --- a/libavcore/imgutils.c
> > +++ b/libavcore/imgutils.c
> > @@ -71,6 +71,8 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
> >          return AVERROR(EINVAL);
> >  
> >      if (desc->flags & PIX_FMT_BITSTREAM) {
> > +        if (width > (INT_MAX -7) / (desc->comp[0].step_minus1+1))
> > +            return AVERROR(EINVAL);
> >          linesizes[0] = (width * (desc->comp[0].step_minus1+1) + 7) >> 3;
> >          return 0;
> >      }
> > @@ -78,7 +80,10 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
> >      av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
> >      for (i = 0; i < 4; i++) {
> >          int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0;
> > -        linesizes[i] = max_step[i] * (((width + (1 << s) - 1)) >> s);
> > +        int shifted_w = ((width + (1 << s) - 1)) >> s;
> > +        if (max_step[i] > SIZE_MAX / shifted_w)
> 
> 
> SIZE_MAX ?

Replaced by INT_MAX.

> [...]
> 
> 
> > @@ -98,11 +103,17 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh
> >          return AVERROR(EINVAL);
> >  
> >      data[0] = ptr;
> > +    if (linesizes[0] > INT_MAX / height)
> > +        return AVERROR(EINVAL);
> >      size[0] = linesizes[0] * height;
> >  
> >      if (desc->flags & PIX_FMT_PAL) {
> > +        if (size[0] > INT_MAX -3)
> > +            return AVERROR(EINVAL);
> >          size[0] = (size[0] + 3) & ~3;
> >          data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */
> > +        if (size[0] > INT_MAX - 256 * 4)
> > +            return AVERROR(EINVAL);
> 
> these 3 can be merged, it doesnt matter if our max image size is INT_MAX or
> INT_MAX- 1024

Done.
-- 
FFmpeg = Fundamental and Fast Muttering Pacific Ecumenical Game