FFmpeg
vvc_mp4toannexb_bsf.c
Go to the documentation of this file.
1 /*
2  * H.266/VVC MP4 to Annex B byte stream format filter
3  * Copyright (c) 2022, Thomas Siedel
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <string.h>
23 
24 #include "libavutil/intreadwrite.h"
25 #include "libavutil/mem.h"
26 
27 #include "bsf.h"
28 #include "bsf_internal.h"
29 #include "bytestream.h"
30 #include "defs.h"
31 #include "vvc.h"
32 
33 #define MIN_VVCC_LENGTH 23
34 
35 typedef struct VVCBSFContext {
36  uint8_t length_size;
39 
41 {
42  GetByteContext gb;
43  int length_size, num_arrays, i, j;
44  int ret = 0;
45  int temp = 0;
46  int ptl_present;
47 
48  uint8_t *new_extradata = NULL;
49  size_t new_extradata_size = 0;
50 
51  int max_picture_width = 0;
52  int max_picture_height = 0;
53  int avg_frame_rate = 0;
54 
55  bytestream2_init(&gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
56  temp = bytestream2_get_byte(&gb);
57  length_size = ((temp & 6) >> 1) + 1;
58  ptl_present = temp & 1;
59  if (ptl_present) {
60  int num_bytes_constraint_info;
61  int general_profile_idc;
62  int general_tier_flag;
63  int general_level_idc;
64  int ptl_frame_only_constraint_flag;
65  int ptl_multi_layer_enabled_flag;
66  int ptl_num_sub_profiles;
67  int temp3, temp4, temp5;
68  int temp2 = bytestream2_get_be16(&gb);
69  int ols_idx = (temp2 >> 7) & 0x1ff;
70  int num_sublayers = (temp2 >> 4) & 0x7;
71  int constant_frame_rate = (temp2 >> 2) & 0x3;
72  int chroma_format_idc = temp2 & 0x3;
73  int bit_depth_minus8 = (bytestream2_get_byte(&gb) >> 5) & 0x7;
75  "bit_depth_minus8 %d chroma_format_idc %d\n", bit_depth_minus8,
76  chroma_format_idc);
77  av_log(ctx, AV_LOG_DEBUG, "constant_frame_rate %d, ols_idx %d\n",
78  constant_frame_rate, ols_idx);
79  // VvcPTLRecord(num_sublayers) native_ptl
80  temp3 = bytestream2_get_byte(&gb);
81  num_bytes_constraint_info = (temp3) & 0x3f;
82  temp4 = bytestream2_get_byte(&gb);
83  general_profile_idc = (temp4 >> 1) & 0x7f;
84  general_tier_flag = (temp4) & 1;
85  general_level_idc = bytestream2_get_byte(&gb);
87  "general_profile_idc %d, general_tier_flag %d, general_level_idc %d, num_sublayers %d num_bytes_constraint_info %d\n",
88  general_profile_idc, general_tier_flag, general_level_idc,
89  num_sublayers, num_bytes_constraint_info);
90 
91  temp5 = bytestream2_get_byte(&gb);
92  ptl_frame_only_constraint_flag = (temp5 >> 7) & 0x1;
93  ptl_multi_layer_enabled_flag = (temp5 >> 6) & 0x1;
94  for (i = 0; i < num_bytes_constraint_info - 1; i++) {
95  // unsigned int(8*num_bytes_constraint_info - 2) general_constraint_info;
96  bytestream2_get_byte(&gb);
97  }
98 
100  "ptl_multi_layer_enabled_flag %d, ptl_frame_only_constraint_flag %d\n",
101  ptl_multi_layer_enabled_flag, ptl_frame_only_constraint_flag);
102 
103  if (num_sublayers > 1) {
104  int temp6 = bytestream2_get_byte(&gb);
105  uint8_t ptl_sublayer_level_present_flag[8] = { 0 };
106  //uint8_t sublayer_level_idc[8] = {0};
107  for (i = num_sublayers - 2; i >= 0; i--) {
108  ptl_sublayer_level_present_flag[i] =
109  (temp6 >> (7 - (num_sublayers - 2 - i))) & 0x01;
110  }
111  // for (j=num_sublayers; j<=8 && num_sublayers > 1; j++)
112  // bit(1) ptl_reserved_zero_bit = 0;
113  for (i = num_sublayers - 2; i >= 0; i--) {
114  if (ptl_sublayer_level_present_flag[i]) {
115  //sublayer_level_idc[i] = bytestream2_get_byte(&gb);
116  }
117  }
118  }
119 
120  ptl_num_sub_profiles = bytestream2_get_byte(&gb);
121  for (j = 0; j < ptl_num_sub_profiles; j++) {
122  // unsigned int(32) general_sub_profile_idc[j];
123  bytestream2_get_be16(&gb);
124  bytestream2_get_be16(&gb);
125  }
126 
127  max_picture_width = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_width;
128  max_picture_height = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_height;
129  avg_frame_rate = bytestream2_get_be16(&gb); // unsigned int(16) avg_frame_rate; }
131  "max_picture_width %d, max_picture_height %d, avg_frame_rate %d\n",
132  max_picture_width, max_picture_height, avg_frame_rate);
133  }
134 
135  num_arrays = bytestream2_get_byte(&gb);
136 
137  for (i = 0; i < num_arrays; i++) {
138  int cnt;
139  int type = bytestream2_get_byte(&gb) & 0x1f;
140 
141  if (type == VVC_OPI_NUT || type == VVC_DCI_NUT)
142  cnt = 1;
143  else
144  cnt = bytestream2_get_be16(&gb);
145 
146  av_log(ctx, AV_LOG_DEBUG, "nalu_type %d cnt %d\n", type, cnt);
147 
148  if (!(type == VVC_OPI_NUT || type == VVC_DCI_NUT ||
152  "Invalid NAL unit type in extradata: %d\n", type);
154  goto fail;
155  }
156 
157  for (j = 0; j < cnt; j++) {
158  int nalu_len = bytestream2_get_be16(&gb);
159 
160  if (4 + AV_INPUT_BUFFER_PADDING_SIZE + nalu_len >
161  SIZE_MAX - new_extradata_size) {
163  goto fail;
164  }
165  ret = av_reallocp(&new_extradata, new_extradata_size + nalu_len + 4
167  if (ret < 0)
168  goto fail;
169 
170  AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
171  bytestream2_get_buffer(&gb, new_extradata + new_extradata_size + 4,
172  nalu_len);
173  new_extradata_size += 4 + nalu_len;
174  memset(new_extradata + new_extradata_size, 0,
176  }
177  }
178 
179  av_freep(&ctx->par_out->extradata);
180  ctx->par_out->extradata = new_extradata;
181  ctx->par_out->extradata_size = new_extradata_size;
182 
183  if (!new_extradata_size)
184  av_log(ctx, AV_LOG_WARNING, "No parameter sets in the extradata\n");
185 
186  return length_size;
187  fail:
188  av_freep(&new_extradata);
189  return ret;
190 }
191 
193 {
195  int ret;
196 
197  if (ctx->par_in->extradata_size < MIN_VVCC_LENGTH ||
198  AV_RB24(ctx->par_in->extradata) == 1 ||
199  AV_RB32(ctx->par_in->extradata) == 1) {
201  "The input looks like it is Annex B already\n");
202  } else {
204  if (ret < 0)
205  return ret;
206  s->length_size = ret;
207  s->extradata_parsed = 1;
208  }
209 
210  return 0;
211 }
212 
214 {
216  AVPacket *in;
217  GetByteContext gb;
218 
219  int is_irap = 0;
220  int added_extra = 0;
221  int i, ret = 0;
222 
223  ret = ff_bsf_get_packet(ctx, &in);
224  if (ret < 0)
225  return ret;
226 
227  if (!s->extradata_parsed) {
228  av_packet_move_ref(out, in);
229  av_packet_free(&in);
230  return 0;
231  }
232 
233  bytestream2_init(&gb, in->data, in->size);
234 
235  /* check if this packet contains an IRAP. The extradata will need to be added before any potential PH_NUT */
236  while (bytestream2_get_bytes_left(&gb)) {
237  uint32_t nalu_size = 0;
238  int nalu_type;
239 
240  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
242  goto fail;
243  }
244 
245  for (i = 0; i < s->length_size; i++)
246  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
247 
248  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
250  goto fail;
251  }
252 
253  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
254  is_irap = nalu_type >= VVC_IDR_W_RADL && nalu_type <= VVC_RSV_IRAP_11;
255  if (is_irap) {
256  break;
257  }
258  bytestream2_seek(&gb, nalu_size, SEEK_CUR);
259  }
260 
261  bytestream2_seek(&gb, 0, SEEK_SET);
262  while (bytestream2_get_bytes_left(&gb)) {
263  uint32_t nalu_size = 0;
264  int nalu_type;
265  int add_extradata, extra_size, prev_size;
266 
267  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
269  goto fail;
270  }
271 
272  for (i = 0; i < s->length_size; i++)
273  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
274 
275  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
277  goto fail;
278  }
279 
280  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
281 
282  /* prepend extradata to IRAP frames */
283  add_extradata = is_irap && nalu_type != VVC_AUD_NUT && !added_extra;
284  extra_size = add_extradata * ctx->par_out->extradata_size;
285  added_extra |= add_extradata;
286 
287  if (FFMIN(INT_MAX, SIZE_MAX) < 4ULL + nalu_size + extra_size) {
289  goto fail;
290  }
291 
292  prev_size = out->size;
293 
294  ret = av_grow_packet(out, 4 + nalu_size + extra_size);
295  if (ret < 0)
296  goto fail;
297 
298  if (extra_size)
299  memcpy(out->data + prev_size, ctx->par_out->extradata, extra_size);
300  AV_WB32(out->data + prev_size + extra_size, 1);
301  bytestream2_get_buffer(&gb, out->data + prev_size + 4 + extra_size,
302  nalu_size);
303  }
304 
306  if (ret < 0)
307  goto fail;
308 
309  fail:
310  if (ret < 0)
312  av_packet_free(&in);
313 
314  return ret;
315 }
316 
317 static const enum AVCodecID codec_ids[] = {
319 };
320 
322  .p.name = "vvc_mp4toannexb",
323  .p.codec_ids = codec_ids,
324  .priv_data_size = sizeof(VVCBSFContext),
327 };
av_packet_unref
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:426
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
VVC_RSV_IRAP_11
@ VVC_RSV_IRAP_11
Definition: vvc.h:40
bsf_internal.h
out
FILE * out
Definition: movenc.c:54
GetByteContext
Definition: bytestream.h:33
VVC_PPS_NUT
@ VVC_PPS_NUT
Definition: vvc.h:45
AVBitStreamFilter::name
const char * name
Definition: bsf.h:112
av_grow_packet
int av_grow_packet(AVPacket *pkt, int grow_by)
Increase packet size, correctly zeroing padding.
Definition: avpacket.c:121
ff_vvc_mp4toannexb_bsf
const FFBitStreamFilter ff_vvc_mp4toannexb_bsf
Definition: vvc_mp4toannexb_bsf.c:321
bytestream2_seek
static av_always_inline int bytestream2_seek(GetByteContext *g, int offset, int whence)
Definition: bytestream.h:212
AVPacket::data
uint8_t * data
Definition: packet.h:515
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
ff_bsf_get_packet
int ff_bsf_get_packet(AVBSFContext *ctx, AVPacket **pkt)
Called by the bitstream filters to get the next packet for filtering.
Definition: bsf.c:235
av_packet_free
void av_packet_free(AVPacket **pkt)
Free the packet, if the packet is reference counted, it will be unreferenced first.
Definition: avpacket.c:74
AVBSFContext
The bitstream filter state.
Definition: bsf.h:68
VVC_AUD_NUT
@ VVC_AUD_NUT
Definition: vvc.h:49
bsf.h
VVC_SUFFIX_SEI_NUT
@ VVC_SUFFIX_SEI_NUT
Definition: vvc.h:53
fail
#define fail()
Definition: checkasm.h:177
VVC_VPS_NUT
@ VVC_VPS_NUT
Definition: vvc.h:43
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
VVC_PREFIX_SEI_NUT
@ VVC_PREFIX_SEI_NUT
Definition: vvc.h:52
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
intreadwrite.h
s
#define s(width, name)
Definition: cbs_vp9.c:198
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts_bsf.c:365
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
VVC_DCI_NUT
@ VVC_DCI_NUT
Definition: vvc.h:42
NULL
#define NULL
Definition: coverity.c:32
FFBitStreamFilter
Definition: bsf_internal.h:27
bytestream2_get_buffer
static av_always_inline unsigned int bytestream2_get_buffer(GetByteContext *g, uint8_t *dst, unsigned int size)
Definition: bytestream.h:267
av_packet_move_ref
void av_packet_move_ref(AVPacket *dst, AVPacket *src)
Move every field in src to dst and reset src.
Definition: avpacket.c:483
bytestream2_get_bytes_left
static av_always_inline int bytestream2_get_bytes_left(GetByteContext *g)
Definition: bytestream.h:158
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
VVCBSFContext::length_size
uint8_t length_size
Definition: vvc_mp4toannexb_bsf.c:36
AV_WB32
#define AV_WB32(p, v)
Definition: intreadwrite.h:417
FFBitStreamFilter::p
AVBitStreamFilter p
The public AVBitStreamFilter.
Definition: bsf_internal.h:31
VVC_IDR_W_RADL
@ VVC_IDR_W_RADL
Definition: vvc.h:36
AVPacket::size
int size
Definition: packet.h:516
av_reallocp
int av_reallocp(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory through a pointer to a pointer.
Definition: mem.c:186
AV_RB32
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_RB32
Definition: bytestream.h:96
AV_CODEC_ID_VVC
@ AV_CODEC_ID_VVC
Definition: codec_id.h:250
vvc_mp4toannexb_init
static int vvc_mp4toannexb_init(AVBSFContext *ctx)
Definition: vvc_mp4toannexb_bsf.c:192
vvc_extradata_to_annexb
static int vvc_extradata_to_annexb(AVBSFContext *ctx)
Definition: vvc_mp4toannexb_bsf.c:40
av_packet_copy_props
int av_packet_copy_props(AVPacket *dst, const AVPacket *src)
Copy only "properties" fields from src to dst.
Definition: avpacket.c:389
AV_CODEC_ID_NONE
@ AV_CODEC_ID_NONE
Definition: codec_id.h:50
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:245
VVC_OPI_NUT
@ VVC_OPI_NUT
Definition: vvc.h:41
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
ret
ret
Definition: filter_design.txt:187
VVC_SPS_NUT
@ VVC_SPS_NUT
Definition: vvc.h:44
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
vvc.h
MIN_VVCC_LENGTH
#define MIN_VVCC_LENGTH
Definition: vvc_mp4toannexb_bsf.c:33
defs.h
temp
else temp
Definition: vf_mcdeint.c:263
mem.h
AVPacket
This structure stores compressed data.
Definition: packet.h:492
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
VVCBSFContext::extradata_parsed
int extradata_parsed
Definition: vvc_mp4toannexb_bsf.c:37
bytestream.h
bytestream2_init
static av_always_inline void bytestream2_init(GetByteContext *g, const uint8_t *buf, int buf_size)
Definition: bytestream.h:137
vvc_mp4toannexb_filter
static int vvc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
Definition: vvc_mp4toannexb_bsf.c:213
codec_ids
static enum AVCodecID codec_ids[]
Definition: vvc_mp4toannexb_bsf.c:317
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:61
AV_RB24
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_WB32 unsigned int_TMPL AV_RB24
Definition: bytestream.h:97
VVCBSFContext
Definition: vvc_mp4toannexb_bsf.c:35
AVFormatContext::priv_data
void * priv_data
Format private data.
Definition: avformat.h:1220