[FFmpeg-devel] [PATCH] avcodec/libvpxenc: add VP8 support for ROI-based encoding
Mark Thompson
sw at jkqxz.net
Thu Feb 28 00:03:47 EET 2019
On 27/02/2019 10:22, Gyan wrote:
> On 27-02-2019 01:57 PM, Guo, Yejun wrote:
>>
>>> -----Original Message-----
>>> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces at ffmpeg.org] On Behalf
>>> Of Guo, Yejun
>>> Sent: Thursday, February 28, 2019 12:13 AM
>>> To: ffmpeg-devel at ffmpeg.org
>>> Subject: [FFmpeg-devel] [PATCH] avcodec/libvpxenc: add VP8 support for
>>> ROI-based encoding
>>>
>>> Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
>>> ---
>>> libavcodec/libvpxenc.c | 132
>>> +++++++++++++++++++++++++++++++++++++++++++++++++
>>> 1 file changed, 132 insertions(+)
>>>
>>> diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
>>> index c823b8a..e3de547 100644
>>> --- a/libavcodec/libvpxenc.c
>>> +++ b/libavcodec/libvpxenc.c
>>> @@ -1057,6 +1057,135 @@ static int queue_frames(AVCodecContext *avctx,
>>> AVPacket *pkt_out)
>>> return size;
>>> }
>>>
>>> +static int vp8_encode_set_roi(AVCodecContext *avctx, const AVFrame
>>> *frame)
>>> +{
>>> + /* range of vpx_roi_map_t.delta_q[i] is [-63, 63] */
>>> +#define MAX_DELTA_Q 63
>>> +
>>> + AVRegionOfInterest *roi = NULL;
>>> + vpx_roi_map_t roi_map;
>>> + AVFrameSideData *sd = av_frame_get_side_data(frame,
>>> AV_FRAME_DATA_REGIONS_OF_INTEREST);
>>> + VPxContext *ctx = avctx->priv_data;
>>> + vpx_active_map_t active_map = { 0, 0, 0 };
>>> + int segment_id;
>>> + int zero_delta_q = 0;
>>> +
>>> + /* record the mapping from delta_q to "segment id + 1".
>>> + * delta_q is shift with MAX_DELTA_Q, and so the range is [0,
>>> 2*MAX_DELTA_Q].
>>> + * add 1 to segment id, so no mapping if the value of array element is zero.
>>> + */
>>> + int segment_mapping[2 * MAX_DELTA_Q + 1] = {0};
>>> +
>>> + active_map.rows = (frame->height + 15) / 16;
>>> + active_map.cols = (frame->width + 15) / 16;
>>> +
>>> + if (!sd) {
>>> + if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP,
>>> &active_map)) {
>>> + log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec
>>> control.\n");
>>> + return AVERROR_INVALIDDATA;
>>> + }
>>> + return 0;
>>> + }
>>> +
>>> + active_map.active_map = av_malloc(active_map.rows * active_map.cols);
>>> + if (!active_map.active_map) {
>>> + av_log(avctx, AV_LOG_ERROR,
>>> + "active_map alloc (%d bytes) failed.\n",
>>> + active_map.rows * active_map.cols);
>>> + return AVERROR(ENOMEM);
>>> + }
>>> + memset(active_map.active_map, 1, active_map.rows * active_map.cols);
>>> +
>>> + /* segment id 0 in roi_map is reserved for the areas not covered by
>>> AVRegionOfInterest.
>>> + * segment id 0 in roi_map is also for the areas with
>>> AVRegionOfInterest.qoffset near 0.
>>> + */
>>> + segment_id = 0;
>>> + segment_mapping[zero_delta_q + MAX_DELTA_Q] = segment_id + 1;
>>> + segment_id++;
>>> + memset(&roi_map, 0, sizeof(roi_map));
>>> + roi_map.delta_q[segment_id] = zero_delta_q;
>>> +
>>> + roi_map.rows = active_map.rows;
>>> + roi_map.cols = active_map.cols;
>>> + roi_map.roi_map = av_mallocz(roi_map.rows * roi_map.cols);
>>> + if (!roi_map.roi_map) {
>>> + av_free(active_map.active_map);
>>> + av_log(avctx, AV_LOG_ERROR,
>>> + "roi_map alloc (%d bytes) failed.\n",
>>> + roi_map.rows * roi_map.cols);
>>> + return AVERROR(ENOMEM);
>>> + }
>>> +
>>> + roi = (AVRegionOfInterest*)sd->data;
>>> + while ((uint8_t*)roi < sd->data + sd->size) {
>>> + int qoffset;
>>> + int mapping_index;
>>> + int mapping_value;
>>> + int starty = FFMIN(roi_map.rows, roi->top / 16);
>>> + int endy = FFMIN(roi_map.rows, (roi->bottom + 15) / 16);
>>> + int startx = FFMIN(roi_map.cols, roi->left / 16);
>>> + int endx = FFMIN(roi_map.cols, (roi->right + 15) / 16);
>>> +
>>> + if (roi->self_size == 0) {
>>> + av_free(active_map.active_map);
>>> + av_free(roi_map.roi_map);
>>> + av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.self_size must be
>>> set to sizeof(AVRegionOfInterest).\n");
>>> + return AVERROR(EINVAL);
>>> + }
>>> +
>>> + if (roi->qoffset.den == 0) {
>>> + av_free(active_map.active_map);
>>> + av_free(roi_map.roi_map);
>>> + av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must
>>> not be zero.\n");
>>> + return AVERROR(EINVAL);
>>> + }
>>> +
>>> + qoffset = (int)(roi->qoffset.num * 1.0f / roi->qoffset.den *
>>> MAX_DELTA_Q);
>>> + qoffset = av_clip(qoffset, -MAX_DELTA_Q, MAX_DELTA_Q);
>>> +
>>> + mapping_index = qoffset + MAX_DELTA_Q;
>>> + if (!segment_mapping[mapping_index]) {
>>> + if (segment_id > 3) {
>>> + av_log(ctx, AV_LOG_WARNING,
>>> + "ROI only support 4 segments (and segment 0 is reserved for
>>> non-ROIs), skipping this one.\n");
>>> + roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
>>> + continue;
>>> + }
>>> +
>>> + segment_mapping[mapping_index] = segment_id + 1;
>>> + roi_map.delta_q[segment_id] = qoffset;
>>> + segment_id++;
>>> + }
>>> +
>>> + mapping_value = segment_mapping[mapping_index];
>>> +
>>> + for (int y = starty; y < endy; y++)
>>> + for (int x = startx; x < endx; x++)
>>> + roi_map.roi_map[x + y * roi_map.cols] = mapping_value - 1;
>>> +
>>> + roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
>>> + }
>>> +
>>> + if (vpx_codec_control(&ctx->encoder, VP8E_SET_ROI_MAP, &roi_map))
>>> {
>>> + av_free(active_map.active_map);
>>> + av_free(roi_map.roi_map);
>>> + log_encoder_error(avctx, "Failed to set VP8E_SET_ROI_MAP codec
>>> control.\n");
>>> + return AVERROR_INVALIDDATA;
>>> + }
>>> +
>>> + if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP,
>>> &active_map)) {
>>> + av_free(active_map.active_map);
>>> + av_free(roi_map.roi_map);
>>> + log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec
>>> control.\n");
>>> + return AVERROR_INVALIDDATA;
>>> + }
>>> +
>>> + av_free(active_map.active_map);
>>> + av_free(roi_map.roi_map);
>>> +
>>> + return 0;
>>> +}
>>> +
>>> static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
>>> const AVFrame *frame, int *got_packet)
>>> {
>>> @@ -1113,6 +1242,9 @@ static int vpx_encode(AVCodecContext *avctx,
>>> AVPacket *pkt,
>>> flags |= strtoul(en->value, NULL, 10);
>>> }
>>> }
>>> +
>>> + if (avctx->codec_id == AV_CODEC_ID_VP8)
>>> + vp8_encode_set_roi(avctx, frame);
>>> }
>>>
>>> res = vpx_codec_encode(&ctx->encoder, rawimg, timestamp,
>> to verify this patch, please cherry-pick https://github.com/guoyejun/ffmpeg/commit/7ba191ad7da5cfd1e6241d0a549c3c6b88c6b2f8,
>> and run command like:
>> ./ffmpeg -i .../path_to_1920x1080_video_file -vf scale=1920:1080 -c:v libvpx -b:v 5M -y tmp.webm
>>
>
> Looks like, at present, the only way to effect ROI is via side data, and no filter or other in-tree mechanism at present can convey or generate it.
>
> Are there any near-term plans to add some? If not, may I suggest that you add a private option for supporting encoders, for users to input ROIs?
I had a filter doing exactly this for testing - it adds the ROI side-data to frames, and can be applied multiple times to add multiple regions. I've included it the set just sent, though I'm not sure how useful it actually is for non-test cases. See <https://lists.ffmpeg.org/pipermail/ffmpeg-devel/2019-February/240533.html>.
Thanks,
- Mark
More information about the ffmpeg-devel
mailing list