[FFmpeg-devel] [PATCH] lavc/vvc: Fix emulation prevention byte handling

[post at frankplowman.com]

From: Frank Plowman <post at frankplowman.com>

nal->skipped_bytes_pos contains the positions of errors relative to the
start of the slice header, whereas the position they were tested against
is relative to the start of the slice data, i.e. one byte after the end
of the slice header.

Patch fixes this by storing the size of the slice header in H266RawSlice
and adding it to the position given by the GetBitContext before
comparing to skipped_bytes_pos.  This fixes AVERROR_INVALIDDATAs for
various valid bitstreams, such as the LMCS_B_Dolby_2 conformance
bitstream.

Signed-off-by: Frank Plowman <post at frankplowman.com>
---
 libavcodec/cbs_h2645.c  | 1 +
 libavcodec/cbs_h266.h   | 1 +
 libavcodec/vvc/vvcdec.c | 9 +++++----
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index c48a06b241..2fb249bcd3 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -1155,6 +1155,7 @@ static int cbs_h266_read_nal_unit(CodedBitstreamContext *ctx,
                     return err;
             }
 
+            slice->header_size = pos / 8;
             slice->data_size = len - pos / 8;
             slice->data_ref  = av_buffer_ref(unit->data_ref);
             if (!slice->data_ref)
diff --git a/libavcodec/cbs_h266.h b/libavcodec/cbs_h266.h
index 8b3ad391b1..73d94157d4 100644
--- a/libavcodec/cbs_h266.h
+++ b/libavcodec/cbs_h266.h
@@ -843,6 +843,7 @@ typedef struct H266RawSlice {
 
     uint8_t     *data;
     AVBufferRef *data_ref;
+    size_t       header_size;
     size_t       data_size;
     int          data_bit_start;
 } H266RawSlice;
diff --git a/libavcodec/vvc/vvcdec.c b/libavcodec/vvc/vvcdec.c
index 540a05f8cf..83ee472ce6 100644
--- a/libavcodec/vvc/vvcdec.c
+++ b/libavcodec/vvc/vvcdec.c
@@ -451,8 +451,9 @@ static int slices_realloc(VVCFrameContext *fc)
 }
 
 static void ep_init_cabac_decoder(SliceContext *sc, const int index,
-    const H2645NAL *nal, GetBitContext *gb)
+    const H2645NAL *nal, GetBitContext *gb, const CodedBitstreamUnit *unit)
 {
+    const H266RawSlice *slice     = unit->content_ref;
     const H266RawSliceHeader *rsh = sc->sh.r;
     EntryPoint *ep                = sc->eps + index;
     int size;
@@ -461,10 +462,10 @@ static void ep_init_cabac_decoder(SliceContext *sc, const int index,
         int skipped = 0;
         int64_t start =  (gb->index >> 3);
         int64_t end = start + rsh->sh_entry_point_offset_minus1[index] + 1;
-        while (skipped < nal->skipped_bytes && nal->skipped_bytes_pos[skipped] <= start) {
+        while (skipped < nal->skipped_bytes && nal->skipped_bytes_pos[skipped] <= start + slice->header_size) {
             skipped++;
         }
-        while (skipped < nal->skipped_bytes && nal->skipped_bytes_pos[skipped] < end) {
+        while (skipped < nal->skipped_bytes && nal->skipped_bytes_pos[skipped] <= end + slice->header_size) {
             end--;
             skipped++;
         }
@@ -506,7 +507,7 @@ static int slice_init_entry_points(SliceContext *sc,
             fc->tab.slice_idx[rs] = sc->slice_idx;
         }
 
-        ep_init_cabac_decoder(sc, i, nal, &gb);
+        ep_init_cabac_decoder(sc, i, nal, &gb, unit);
 
         if (i + 1 < sc->nb_eps)
             ctu_addr = sh->entry_point_start_ctu[i];
-- 
2.43.0