[FFmpeg-devel] [PATCH] vaapi_encode_h265: Ensure that ref pics are always in the RPS

[Mark Thompson]

When making a new P-frame when B-frames are present the previous P-frame
is normally in the DPB because it will be referred to by subsequent
B-frames.  However, this is not true if there are no B-frames, or in edge
cases where a GOP ends with two P-frames.  Fix this by adding the direct
ref pics to the RPS explicitly.

Fixes #7699.
---
 libavcodec/vaapi_encode_h265.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 1d40e06667..19e7104e9e 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -902,6 +902,7 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
 
     if (pic->type != PICTURE_TYPE_IDR) {
         H265RawSTRefPicSet *rps;
+        const VAAPIEncodeH265Picture *strp;
         int rps_poc[MAX_DPB_SIZE];
         int rps_used[MAX_DPB_SIZE];
         int i, j, poc, rps_pics;
@@ -912,16 +913,24 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
         memset(rps, 0, sizeof(*rps));
 
         rps_pics = 0;
+        for (i = 0; i < pic->nb_refs; i++) {
+            strp = pic->refs[i]->priv_data;
+            rps_poc[rps_pics]  = strp->pic_order_cnt;
+            rps_used[rps_pics] = 1;
+            ++rps_pics;
+        }
         for (i = 0; i < pic->nb_dpb_pics; i++) {
-            VAAPIEncodeH265Picture *strp;
             if (pic->dpb[i] == pic)
                 continue;
+            for (j = 0; j < pic->nb_refs; j++) {
+                if (pic->dpb[i] == pic->refs[j])
+                    break;
+            }
+            if (j < pic->nb_refs)
+                continue;
             strp = pic->dpb[i]->priv_data;
             rps_poc[rps_pics]  = strp->pic_order_cnt;
             rps_used[rps_pics] = 0;
-            for (j = 0; j < pic->nb_refs; j++)
-                if (pic->dpb[i] == pic->refs[j])
-                    rps_used[rps_pics] = 1;
             ++rps_pics;
         }
 
-- 
2.19.2