[FFmpeg-devel] [PATCH 2/2] vaapi_encode_h26[45]: Reject bitrate targets higher than 2^31

[Mark Thompson]

---
I'll go with Matthieu's suggestion to use INT32_MAX instead of 1 << 31.

 libavcodec/vaapi_encode_h264.c | 6 ++++++
 libavcodec/vaapi_encode_h265.c | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index dc7774b..2e2052b 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -731,6 +731,12 @@ static av_cold int vaapi_encode_h264_init_constant_bitrate(AVCodecContext *avctx
     int hrd_buffer_size;
     int hrd_initial_buffer_fullness;

+    if (avctx->bit_rate > INT32_MAX) {
+        av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or "
+               "higher is not supported.\n");
+        return AVERROR(EINVAL);
+    }
+
     if (avctx->rc_buffer_size)
         hrd_buffer_size = avctx->rc_buffer_size;
     else
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 17cd900..e4d21f0 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -1158,6 +1158,12 @@ static av_cold int vaapi_encode_h265_init_constant_bitrate(AVCodecContext *avctx
     int hrd_buffer_size;
     int hrd_initial_buffer_fullness;

+    if (avctx->bit_rate > INT32_MAX) {
+        av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or "
+               "higher is not supported.\n");
+        return AVERROR(EINVAL);
+    }
+
     if (avctx->rc_buffer_size)
         hrd_buffer_size = avctx->rc_buffer_size;
     else
-- 
2.8.1