[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle video parameters

[wm4]

On Sat, 21 Jan 2017 10:35:50 -0700
Pavel Koshevoy <pkoshevoy at gmail.com> wrote:

> On Sat, Jan 21, 2017 at 10:27 AM,  <pkoshevoy at gmail.com> wrote:
> > From: Pavel Koshevoy <pkoshevoy at gmail.com>
> >
> > Evidently CUVID doesn't support decoding 422 or 444 chroma formats,
> > and only a limited set of resolutions per codec are supported.
> >
> > Unfortunately CUVID silently drops packets for video of unsupported
> > resolution.  However, it will error-out at cuvidCreateDecoder call
> > if the indicated video resolution is not supported.
> >
> > Given that stream resolution and pixel format are typically known as
> > a result of probing it is better to use this information during
> > avcodec_open2 call to fail immediately, rather than proceeding to
> > decode and never receiving any frames from the decoder nor receiving
> > any indication of decode failure.
> > ---
> >  libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++-----
> >  1 file changed, 57 insertions(+), 5 deletions(-)
> >
> > diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
> > index 8fc713d..f9c29a1 100644
> > --- a/libavcodec/cuvid.c
> > +++ b/libavcodec/cuvid.c
> > @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
> >      return 0;
> >  }
> >
> > -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
> > +static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
> > +                                    const CUVIDPARSERPARAMS *cuparseinfo,
> > +                                    cudaVideoChromaFormat probed_chroma_format,
> > +                                    int probed_width,
> > +                                    int probed_height)
> >  {
> >      CuvidContext *ctx = avctx->priv_data;
> >      CUVIDDECODECREATEINFO cuinfo;
> > @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
> >      memset(&cuinfo, 0, sizeof(cuinfo));
> >
> >      cuinfo.CodecType = cuparseinfo->CodecType;
> > -    cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
> > +    cuinfo.ChromaFormat = probed_chroma_format;
> >      cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
> >
> > -    cuinfo.ulWidth = 1280;
> > -    cuinfo.ulHeight = 720;
> > +    cuinfo.ulWidth = probed_width;
> > +    cuinfo.ulHeight = probed_height;
> >      cuinfo.ulTargetWidth = cuinfo.ulWidth;
> >      cuinfo.ulTargetHeight = cuinfo.ulHeight;
> >
> > @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
> >      return 0;
> >  }
> >
> > +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt,
> > +                                               cudaVideoChromaFormat *out)
> > +{
> > +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
> > +    if (!(out && desc &&
> > +          (desc->nb_components == 1 || desc->nb_components == 3) &&
> > +          (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2)))
> > +    {
> > +        return AVERROR(EINVAL);
> > +    }
> > +
> > +    if (desc->nb_components == 1)
> > +    {
> > +        *out = cudaVideoChromaFormat_Monochrome;
> > +    }
> > +    else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR)
> > +    {
> > +        *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 :
> > +                (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 :
> > +                cudaVideoChromaFormat_420);
> > +    }
> > +    else
> > +    {
> > +        return AVERROR(EINVAL);
> > +    }
> > +
> > +    // unfortunately, 420 is the only one that works:
> > +    return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL;
> > +}
> > +
> >  static av_cold int cuvid_decode_init(AVCodecContext *avctx)
> >  {
> >      CuvidContext *ctx = avctx->priv_data;
> > @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
> >      CUcontext cuda_ctx = NULL;
> >      CUcontext dummy;
> >      const AVBitStreamFilter *bsf;
> > +    cudaVideoChromaFormat probed_chroma_format;
> > +    int probed_width;
> > +    int probed_height;
> >      int ret = 0;
> >
> >      enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
> >                                         AV_PIX_FMT_NV12,
> >                                         AV_PIX_FMT_NONE };
> >
> > +    enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt <= 0 ?
> > +                                         AV_PIX_FMT_YUV420P :
> > +                                         avctx->pix_fmt);
> > +
> > +    ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format);
> > +    if (ret < 0) {
> > +        // pixel format is not supported:
> > +        return ret;
> > +    }
> > +    probed_width = avctx->coded_width ? avctx->coded_width : 1280;
> > +    probed_height = avctx->coded_height ? avctx->coded_height : 720;
> > +
> >      // Accelerated transcoding scenarios with 'ffmpeg' require that the
> >      // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
> >      // pix_fmt for non-accelerated transcoding, do not need to be correct
> > @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
> >      if (ret < 0)
> >          goto error;
> >
> > -    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
> > +    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
> > +                                   probed_chroma_format,
> > +                                   probed_width,
> > +                                   probed_height);
> >      if (ret < 0)
> >          goto error;
> >
> > --
> > 2.9.2
> >  
> 
> 
> Despite wm4 objections I am resubmitting this patch, because I can
> find no other reasonable method for rejecting ffmpeg cuvid decoder
> when it can't handle a given video stream.  An unreasonable
> alternative would be to duplicate the work that cuvid.c does
> internally on my application side, and that would be of little benefit
> to me and to ffmpeg -- I might as well use the nvenc APIs directly
> then.  I still see this patch as an improvement.

PS:

Actually, cuvid is a bit different from normal hwaccel and I confused
that (blame Sunday "morning").

In this case, do you just want the cuvid to gracefully exit the ffmpeg.c
process if decoding is not supported? In that case, this would be much
simpler:

1. Return a special error code from cuvid that signals unsupported
   format
2. Make ffmpeg.c check for this error code, and stop transcoding with
   an error if it's encountered