[FFmpeg-devel] [PATCH 1/8] avfilter/vf_overlay: fix memory leaks

[Ganesh Ajjanagadde]

On Fri, Dec 4, 2015 at 9:39 AM, Ganesh Ajjanagadde
<gajjanagadde at gmail.com> wrote:
> Recent commits 6aaac24d72a7da631173209841a3944fcb4a3309 and
> 3835554bf8ed78539a3492c239f979c0ab03a15f made progress towards cleaning
> up usage of the formats API, and in particular fixed possible NULL pointer
> dereferences.
>
> This commit addresses the issue of possible resource leaks when some intermediate
> call fails.
>
> Tested with valgrind --leak-check=full --show-leak-kinds=all, and manual simulation
> of malloc/realloc failures.
>
> Fixes: CID 1338327.
>
> Signed-off-by: Ganesh Ajjanagadde <gajjanagadde at gmail.com>
> ---
>  libavfilter/vf_overlay.c | 32 +++++++++++++++++++++++---------
>  1 file changed, 23 insertions(+), 9 deletions(-)
>
> diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
> index 3c61731..68cfb1b 100644
> --- a/libavfilter/vf_overlay.c
> +++ b/libavfilter/vf_overlay.c
> @@ -252,23 +252,31 @@ static int query_formats(AVFilterContext *ctx)
>      switch (s->format) {
>      case OVERLAY_FORMAT_YUV420:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv420)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +            }
>          break;
>      case OVERLAY_FORMAT_YUV422:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv422)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +            }
>          break;
>      case OVERLAY_FORMAT_YUV444:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv444)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +            }
>          break;
>      case OVERLAY_FORMAT_RGB:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_rgb)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +            }
>          break;
>      default:
>          av_assert0(0);
> @@ -277,9 +285,15 @@ static int query_formats(AVFilterContext *ctx)
>      if ((ret = ff_formats_ref(main_formats   , &ctx->inputs[MAIN]->out_formats   )) < 0 ||
>          (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
>          (ret = ff_formats_ref(main_formats   , &ctx->outputs[MAIN]->in_formats   )) < 0)
> -        return ret;
> +            goto fail;
>
>      return 0;
> +fail:
> +    av_freep(&main_formats->formats);
> +    av_freep(&main_formats);
> +    av_freep(&overlay_formats->formats);
> +    av_freep(&overlay_formats);
> +    return ret;
>  }
>
>  static const enum AVPixelFormat alpha_pix_fmts[] = {
> --
> 2.6.3
>

pushed, with the necessary modification described by Clement