[FFmpeg-devel] [PATCH] Adds decode support for formats other than 420

[Deb Mukherjee]

Handles decoding of new VP9 profiles 1-3 with different color sampling
and bit-depths.

For high bitdepth (profiles 2 and 3) support, we currently need to link
with the highbitdepth branch of libvpx with --enable-experimental
and --enable-vp9-high config options on. But eventually this branch will
be merged into master, whereafter to enable high bitdepth
support you will need to link with libvpx with configure option
--enable-vp9-highbitdepth on.
---
 libavcodec/libvpxdec.c | 78 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 68 insertions(+), 10 deletions(-)

diff --git a/libavcodec/libvpxdec.c b/libavcodec/libvpxdec.c
index 94e1e4d..60e418f 100644
--- a/libavcodec/libvpxdec.c
+++ b/libavcodec/libvpxdec.c
@@ -60,7 +60,60 @@ static av_cold int vpx_init(AVCodecContext *avctx,
     return 0;
 }
 
-static int vp8_decode(AVCodecContext *avctx,
+// returns 0 on success, AVERROR_INVALIDDATA otherwise
+static int set_pix_fmt(AVCodecContext *avctx, struct vpx_image *img) {
+    if (avctx->codec_id == AV_CODEC_ID_VP8) {
+        if (img->fmt != VPX_IMG_FMT_I420)
+            return 0;
+    }
+    switch (img->fmt) {
+        case VPX_IMG_FMT_I420:
+            avctx->pix_fmt = AV_PIX_FMT_YUV420P;
+            return 0;
+        case VPX_IMG_FMT_I422:
+            avctx->pix_fmt = AV_PIX_FMT_YUV422P;
+            return 0;
+        case VPX_IMG_FMT_I444:
+            avctx->pix_fmt = AV_PIX_FMT_YUV444P;
+            return 0;
+#ifdef VPX_IMG_FMT_HIGHBITDEPTH
+        case VPX_IMG_FMT_I42016:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV420P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV420P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+        case VPX_IMG_FMT_I42216:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV422P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV422P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+        case VPX_IMG_FMT_I44416:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV444P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV444P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+#endif
+        default:
+            return AVERROR_INVALIDDATA;
+    }
+}
+
+static int vpx_decode(AVCodecContext *avctx,
                       void *data, int *got_frame, AVPacket *avpkt)
 {
     VP8Context *ctx = avctx->priv_data;
@@ -82,10 +135,15 @@ static int vp8_decode(AVCodecContext *avctx,
     }
 
     if ((img = vpx_codec_get_frame(&ctx->decoder, &iter))) {
-        if (img->fmt != VPX_IMG_FMT_I420) {
-            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d)\n",
-                   img->fmt);
-            return AVERROR_INVALIDDATA;
+        if ((ret = set_pix_fmt(avctx, img)) < 0) {
+#ifdef VPX_IMG_FMT_HIGHBITDEPTH
+            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d) / bit_depth (%d)\n",
+                   img->fmt, img->bit_depth);
+#else
+            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d) / bit_depth (%d)\n",
+                   img->fmt, 8);
+#endif
+            return ret;
         }
 
         if ((int) img->d_w != avctx->width || (int) img->d_h != avctx->height) {
@@ -104,7 +162,7 @@ static int vp8_decode(AVCodecContext *avctx,
     return avpkt->size;
 }
 
-static av_cold int vp8_free(AVCodecContext *avctx)
+static av_cold int vpx_free(AVCodecContext *avctx)
 {
     VP8Context *ctx = avctx->priv_data;
     vpx_codec_destroy(&ctx->decoder);
@@ -124,8 +182,8 @@ AVCodec ff_libvpx_vp8_decoder = {
     .id             = AV_CODEC_ID_VP8,
     .priv_data_size = sizeof(VP8Context),
     .init           = vp8_init,
-    .close          = vp8_free,
-    .decode         = vp8_decode,
+    .close          = vpx_free,
+    .decode         = vpx_decode,
     .capabilities   = CODEC_CAP_AUTO_THREADS | CODEC_CAP_DR1,
 };
 #endif /* CONFIG_LIBVPX_VP8_DECODER */
@@ -143,8 +201,8 @@ AVCodec ff_libvpx_vp9_decoder = {
     .id             = AV_CODEC_ID_VP9,
     .priv_data_size = sizeof(VP8Context),
     .init           = vp9_init,
-    .close          = vp8_free,
-    .decode         = vp8_decode,
+    .close          = vpx_free,
+    .decode         = vpx_decode,
     .capabilities   = CODEC_CAP_AUTO_THREADS | CODEC_CAP_DR1,
     .init_static_data = ff_vp9_init_static,
 };
-- 
2.1.0.rc2.206.gedb03e5