October 2022 - ffmpeg-devel - lists.ffmpeg.org

[PATCH v2 00/10] Add support for H266/VVC
by Thomas Siedel 03 Nov '22

03 Nov '22

This patch set adds H266/VVC support. This includes parsing, muxing, demuxing, decoding and encoding. Decoding is done using the external library VVdeC (https://github.com/fraunhoferhhi/vvdec.git) and can be enabled with --enable-libvvdec. Encoding is done using the external library VVenC (https://github.com/fraunhoferhhi/vvenc.git) and can be enabled with --enable-libvvenc. Changes since v1: PATCH 03/10 - mv libavcodec/Makefile cbs changes into patch 3 to make it compilable - cleanup cbs_h266_unit_types (adapt cbs_h266_unit_types to current syntax) PATCH 05/10 libavformat/vvc.c - change gci_general_constraints from unsigned __int128 to uint8_t[9] - change uint8_t *general_constraint_info to general_constraint_info[9] - set correct constraints information in general_constraint_info - vvcc_write fix indent of log output PATCH 07/10 libavcodec/vvc_parse_extradata.c - remove warning for sublayer_level_idc PATCH 09/10 libavcodec/libvvenc.c - set correct hdr mode, depending on color primaries Thomas Siedel (10): avcodec: add enum types for H266/VVC avcodec: add cbs for H266/VVC avcodec: add bitstream parser for H266/VVC avcodec: add MP4 to annexb support for H266/VVC avformat: add demuxer and probe support for H266/VVC avformat: add muxer support for H266/VVC avcodec: add external decoder libvvdec for H266/VVC avcodec: add external encoder libvvenc for H266/VVC avformat: add ts stream types for H266/VVC avcodec: increase minor version for H266/VVC configure | 16 +- libavcodec/Makefile | 6 + libavcodec/allcodecs.c | 2 + libavcodec/bitstream_filters.c | 2 + libavcodec/cbs.c | 6 + libavcodec/cbs_h2645.c | 373 ++- libavcodec/cbs_h266.h | 791 +++++++ libavcodec/cbs_h266_syntax_template.c | 3010 +++++++++++++++++++++++++ libavcodec/cbs_internal.h | 1 + libavcodec/cbs_sei.c | 29 + libavcodec/h2645_parse.c | 71 +- libavcodec/h266_metadata_bsf.c | 145 ++ libavcodec/libvvdec.c | 511 +++++ libavcodec/libvvenc.c | 442 ++++ libavcodec/parsers.c | 1 + libavcodec/version.h | 2 +- libavcodec/vvc.h | 142 ++ libavcodec/vvc_mp4toannexb_bsf.c | 318 +++ libavcodec/vvc_paramset.c | 972 ++++++++ libavcodec/vvc_paramset.h | 429 ++++ libavcodec/vvc_parse_extradata.c | 241 ++ libavcodec/vvc_parse_extradata.h | 36 + libavcodec/vvc_parser.c | 588 +++++ libavformat/Makefile | 8 +- libavformat/allformats.c | 2 + libavformat/demux.c | 7 +- libavformat/isom.c | 1 + libavformat/isom_tags.c | 3 + libavformat/mov.c | 6 + libavformat/movenc.c | 41 +- libavformat/mpeg.c | 3 + libavformat/mpeg.h | 1 + libavformat/mpegts.c | 2 + libavformat/mpegts.h | 1 + libavformat/mpegtsenc.c | 65 + libavformat/rawenc.c | 23 + libavformat/vvc.c | 919 ++++++++ libavformat/vvc.h | 99 + libavformat/vvcdec.c | 61 + 39 files changed, 9365 insertions(+), 11 deletions(-) create mode 100644 libavcodec/cbs_h266.h create mode 100644 libavcodec/cbs_h266_syntax_template.c create mode 100644 libavcodec/h266_metadata_bsf.c create mode 100644 libavcodec/libvvdec.c create mode 100644 libavcodec/libvvenc.c create mode 100644 libavcodec/vvc.h create mode 100644 libavcodec/vvc_mp4toannexb_bsf.c create mode 100644 libavcodec/vvc_paramset.c create mode 100644 libavcodec/vvc_paramset.h create mode 100644 libavcodec/vvc_parse_extradata.c create mode 100644 libavcodec/vvc_parse_extradata.h create mode 100644 libavcodec/vvc_parser.c create mode 100644 libavformat/vvc.c create mode 100644 libavformat/vvc.h create mode 100644 libavformat/vvcdec.c -- 2.25.1

3 16

[PATCH v2 1/2] ffmpeg: shift start time correction to ffmpeg_opt
by Gyan Doshi 03 Nov '22

03 Nov '22

In preparation for applying start time correction that accounts for all factors such as copyts, input_ts_offset ..etc --- fftools/ffmpeg.c | 22 ---------------------- fftools/ffmpeg_opt.c | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index e57486fd4a..0fe582be3b 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3194,28 +3194,6 @@ static int transcode_init(void) input_streams[j + ifile->ist_index]->start = av_gettime_relative(); } - // Correct starttime based on the enabled streams - for (i = 0; i < nb_input_files; i++) { - InputFile *ifile = input_files[i]; - AVFormatContext *is = ifile->ctx; - int64_t new_start_time = INT64_MAX; - - if (is->start_time == AV_NOPTS_VALUE || - !(is->iformat->flags & AVFMT_TS_DISCONT)) - continue; - - for (int j = 0; j < is->nb_streams; j++) { - AVStream *st = is->streams[j]; - if(st->discard == AVDISCARD_ALL || st->start_time == AV_NOPTS_VALUE) - continue; - new_start_time = FFMIN(new_start_time, av_rescale_q(st->start_time, st->time_base, AV_TIME_BASE_Q)); - } - if (new_start_time > is->start_time) { - av_log(is, AV_LOG_VERBOSE, "Correcting start time by %"PRId64"\n", new_start_time - is->start_time); - ifile->ts_offset = -new_start_time; - } - } - /* init input streams */ for (i = 0; i < nb_input_streams; i++) if ((ret = init_input_stream(i, error, sizeof(error))) < 0) diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 9245e02813..28b7d4dc27 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -216,6 +216,31 @@ int parse_and_set_vsync(const char *arg, int *vsync_var, int file_idx, int st_id return 0; } +static void correct_input_start_times(void) +{ + // Correct starttime based on the enabled streams + for (int i = 0; i < nb_input_files; i++) { + InputFile *ifile = input_files[i]; + AVFormatContext *is = ifile->ctx; + int64_t new_start_time = INT64_MAX; + + if (is->start_time == AV_NOPTS_VALUE || + !(is->iformat->flags & AVFMT_TS_DISCONT)) + continue; + + for (int j = 0; j < is->nb_streams; j++) { + AVStream *st = is->streams[j]; + if(st->discard == AVDISCARD_ALL || st->start_time == AV_NOPTS_VALUE) + continue; + new_start_time = FFMIN(new_start_time, av_rescale_q(st->start_time, st->time_base, AV_TIME_BASE_Q)); + } + if (new_start_time > is->start_time) { + av_log(is, AV_LOG_VERBOSE, "Correcting start time by %"PRId64"\n", new_start_time - is->start_time); + ifile->ts_offset = -new_start_time; + } + } +} + static int apply_sync_offsets(void) { for (int i = 0; i < nb_input_files; i++) { @@ -1909,6 +1934,8 @@ int ffmpeg_parse_options(int argc, char **argv) goto fail; } + correct_input_start_times(); + check_filter_outputs(); fail: -- 2.36.1

1 5

[PATCH v2 0/4] swscale rgbaf32 input/output support
by mindmark＠gmail.com 02 Nov '22

02 Nov '22

From: Mark Reid <mindmark(a)gmail.com> This patch series adds swscale input/output support for the packed rgb float formats. A few of the filters also needed support the larger 96/128 bit packed pixel sizes. I also plan to eventually add lossless unscaled conversions between the planer and packed formats. changes since v1 * output correct alpha is src doesn't have alpha Mark Reid (4): swscale/input: add rgbaf32 input support avfilter/vf_hflip: add support for packed rgb float formats avfilter/vf_transpose: add support for packed rgb float formats swscale/output: add rgbaf32 output support libavfilter/vf_hflip_init.h | 25 ++++ libavfilter/vf_transpose.c | 44 ++++++ libswscale/input.c | 172 +++++++++++++++++++++++ libswscale/output.c | 92 ++++++++++++ libswscale/swscale_unscaled.c | 4 +- libswscale/tests/floatimg_cmp.c | 4 +- libswscale/utils.c | 12 +- libswscale/yuv2rgb.c | 2 + tests/ref/fate/filter-pixdesc-rgbaf32be | 1 + tests/ref/fate/filter-pixdesc-rgbaf32le | 1 + tests/ref/fate/filter-pixdesc-rgbf32be | 1 + tests/ref/fate/filter-pixdesc-rgbf32le | 1 + tests/ref/fate/filter-pixfmts-copy | 4 + tests/ref/fate/filter-pixfmts-crop | 4 + tests/ref/fate/filter-pixfmts-field | 4 + tests/ref/fate/filter-pixfmts-fieldorder | 4 + tests/ref/fate/filter-pixfmts-hflip | 4 + tests/ref/fate/filter-pixfmts-il | 4 + tests/ref/fate/filter-pixfmts-null | 4 + tests/ref/fate/filter-pixfmts-scale | 4 + tests/ref/fate/filter-pixfmts-transpose | 4 + tests/ref/fate/filter-pixfmts-vflip | 4 + tests/ref/fate/sws-floatimg-cmp | 16 +++ 23 files changed, 411 insertions(+), 4 deletions(-) create mode 100644 tests/ref/fate/filter-pixdesc-rgbaf32be create mode 100644 tests/ref/fate/filter-pixdesc-rgbaf32le create mode 100644 tests/ref/fate/filter-pixdesc-rgbf32be create mode 100644 tests/ref/fate/filter-pixdesc-rgbf32le -- 2.31.1.windows.1

4 7

[PATCH] Avoid possible integer overflow in yuv420 to rgba64 templates by saturating.
by Drew Dunne 02 Nov '22

02 Nov '22

--- libswscale/output.c | 96 ++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/libswscale/output.c b/libswscale/output.c index 0e1c1225a0..8c8f62682a 100644 --- a/libswscale/output.c +++ b/libswscale/output.c @@ -1109,20 +1109,20 @@ yuv2rgba64_X_c_template(SwsContext *c, const int16_t *lumFilter, B = U * c->yuv2rgb_u2b_coeff; // 8 bits: 30 - 22 = 8 bits, 16 bits: 30 bits - 14 = 16 bits - output_pixel(&dest[0], av_clip_uintp2(R_B + Y1, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y1, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y1, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y1), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y1), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y1), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[6], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[6], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14); dest += 8; } else { - output_pixel(&dest[3], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[3], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); dest += 6; } } @@ -1175,20 +1175,20 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], A2 += 1 << 13; } - output_pixel(&dest[0], av_clip_uintp2(R_B + Y1, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y1, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y1, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y1), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y1), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y1), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[6], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[6], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14); dest += 8; } else { - output_pixel(&dest[3], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[3], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); dest += 6; } } @@ -1232,20 +1232,20 @@ yuv2rgba64_1_c_template(SwsContext *c, const int32_t *buf0, G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff; B = U * c->yuv2rgb_u2b_coeff; - output_pixel(&dest[0], av_clip_uintp2(R_B + Y1, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y1, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y1, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y1), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y1), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y1), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[6], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[6], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14); dest += 8; } else { - output_pixel(&dest[3], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[3], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); dest += 6; } } @@ -1278,20 +1278,20 @@ yuv2rgba64_1_c_template(SwsContext *c, const int32_t *buf0, G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff; B = U * c->yuv2rgb_u2b_coeff; - output_pixel(&dest[0], av_clip_uintp2(R_B + Y1, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y1, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y1, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y1), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y1), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y1), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[6], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[6], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14); dest += 8; } else { - output_pixel(&dest[3], av_clip_uintp2(R_B + Y2, 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2( G + Y2, 30) >> 14); - output_pixel(&dest[5], av_clip_uintp2(B_R + Y2, 30) >> 14); + output_pixel(&dest[3], av_clip_uintp2(av_sat_add32(R_B, Y2), 30) >> 14); + output_pixel(&dest[4], av_clip_uintp2(av_sat_add32( G, Y2), 30) >> 14); + output_pixel(&dest[5], av_clip_uintp2(av_sat_add32(B_R, Y2), 30) >> 14); dest += 6; } } @@ -1351,9 +1351,9 @@ yuv2rgba64_full_X_c_template(SwsContext *c, const int16_t *lumFilter, B = U * c->yuv2rgb_u2b_coeff; // 8bit: 30 - 22 = 8bit, 16bit: 30bit - 14 = 16bit - output_pixel(&dest[0], av_clip_uintp2(R_B + Y, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A, 30) >> 14); dest += 4; @@ -1404,9 +1404,9 @@ yuv2rgba64_full_2_c_template(SwsContext *c, const int32_t *buf[2], A += 1 << 13; } - output_pixel(&dest[0], av_clip_uintp2(R_B + Y, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A, 30) >> 14); dest += 4; @@ -1448,9 +1448,9 @@ yuv2rgba64_full_1_c_template(SwsContext *c, const int32_t *buf0, G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff; B = U * c->yuv2rgb_u2b_coeff; - output_pixel(&dest[0], av_clip_uintp2(R_B + Y, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A, 30) >> 14); dest += 4; @@ -1481,9 +1481,9 @@ yuv2rgba64_full_1_c_template(SwsContext *c, const int32_t *buf0, G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff; B = U * c->yuv2rgb_u2b_coeff; - output_pixel(&dest[0], av_clip_uintp2(R_B + Y, 30) >> 14); - output_pixel(&dest[1], av_clip_uintp2( G + Y, 30) >> 14); - output_pixel(&dest[2], av_clip_uintp2(B_R + Y, 30) >> 14); + output_pixel(&dest[0], av_clip_uintp2(av_sat_add32(R_B, Y), 30) >> 14); + output_pixel(&dest[1], av_clip_uintp2(av_sat_add32( G, Y), 30) >> 14); + output_pixel(&dest[2], av_clip_uintp2(av_sat_add32(B_R, Y), 30) >> 14); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A, 30) >> 14); dest += 4; -- 2.38.0.135.g90850a2211-goog

2 3

[PATCH] avfilter/src_movie: support unknown channel layouts
by Paul B Mahol 01 Nov '22

01 Nov '22

Patch attached.

3 4

[PATCH 0/3] sw_scale: Provide neon implementation for hscale
by Hubert Mazur 01 Nov '22

01 Nov '22

This patchset contains arm64 neon implementation of hscale functions. Fixed minor style issues and declared C function wrappers as static. This patchset do not contain the patch for checkasm tool, as the previous one did. The reason behind it was failing tests on x86 arch but not on aarch64 or loongarch. Probably the hscale functions on x86 have some bugs. Currently the checkasm tool does not check the validity of hscale functions on x86 at all. Implementation of hscale for x86 should be fixed anyway. As it comes to aarch64 the tests were passing. Attaching a link to fate result in patchwork. After fixing x86 the patch for checkasm could be merged. https://patchwork.ffmpeg.org/project/ffmpeg/patch/20221017130715.30896-3-hu… Hubert Mazur (3): sw_scale: Add specializations for hscale 8 to 19 sw_scale: Add specializations for hscale 16 to 15 sw_scale: Add specializations for hscale 16 to 19 libswscale/aarch64/hscale.S | 1100 ++++++++++++++++++++++++++++++++++ libswscale/aarch64/swscale.c | 140 ++++- libswscale/swscale.c | 1 - 3 files changed, 1236 insertions(+), 5 deletions(-) -- 2.37.1

2 4

[PATCH] Revert "avfilter/vf_palette(gen|use): support palettes with alpha"
by Clément Bœsch 01 Nov '22

01 Nov '22

This reverts commit dea673d0d548c864ec85f9260d8900d944ef7a2a. This change cannot work for several reasons, the most obvious ones are: - the alpha is being part of the scoring of the color difference, even though we can not interpret the alpha as part of the perception of the color (we don't even know if it's premultiplied or postmultiplied) - the colors are averaged with their alpha value which simply cannot work The command proposed in the original thread of the patch actually produces a completely broken file: ffmpeg -y -loglevel verbose -i fate-suite/apng/o_sample.png -filter_complex "split[split1][split2];[split1]palettegen=max_colors=254:use_alpha=1[pal1];[split2][pal1]paletteuse=use_alpha=1" -frames:v 1 out.png We can see that many color pixels are off, but more importantly some colors have a random alpha value: https://imgur.com/eFQ2UK7 I don't see any easy fix for this unfortunately, the approach appears to be flawed by design. --- doc/filters.texi | 8 -- libavfilter/vf_palettegen.c | 135 +++++++--------------- libavfilter/vf_paletteuse.c | 225 +++++++++++++++--------------------- 3 files changed, 138 insertions(+), 230 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index bcd19cf931..f8b78ca919 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -18269,9 +18269,6 @@ Compute new histogram for each frame. @end table Default value is @var{full}. -@item use_alpha -Create a palette of colors with alpha components. -Setting this, will automatically disable 'reserve_transparent'. @end table The filter also exports the frame metadata @code{lavfi.color_quant_ratio} @@ -18350,11 +18347,6 @@ will be treated as completely opaque, and values below this threshold will be treated as completely transparent. The option must be an integer value in the range [0,255]. Default is @var{128}. - -@item use_alpha -Apply the palette by taking alpha values into account. Only useful with -palettes that are containing multiple colors with alpha components. -Setting this will automatically disable 'alpha_treshold'. @end table @subsection Examples diff --git a/libavfilter/vf_palettegen.c b/libavfilter/vf_palettegen.c index 27f74fd147..d335ef91e6 100644 --- a/libavfilter/vf_palettegen.c +++ b/libavfilter/vf_palettegen.c @@ -59,7 +59,7 @@ enum { }; #define NBITS 5 -#define HIST_SIZE (1<<(4*NBITS)) +#define HIST_SIZE (1<<(3*NBITS)) typedef struct PaletteGenContext { const AVClass *class; @@ -67,7 +67,6 @@ typedef struct PaletteGenContext { int max_colors; int reserve_transparent; int stats_mode; - int use_alpha; AVFrame *prev_frame; // previous frame used for the diff stats_mode struct hist_node histogram[HIST_SIZE]; // histogram/hashtable of the colors @@ -89,7 +88,6 @@ static const AVOption palettegen_options[] = { { "full", "compute full frame histograms", 0, AV_OPT_TYPE_CONST, {.i64=STATS_MODE_ALL_FRAMES}, INT_MIN, INT_MAX, FLAGS, "mode" }, { "diff", "compute histograms only for the part that differs from previous frame", 0, AV_OPT_TYPE_CONST, {.i64=STATS_MODE_DIFF_FRAMES}, INT_MIN, INT_MAX, FLAGS, "mode" }, { "single", "compute new histogram for each frame", 0, AV_OPT_TYPE_CONST, {.i64=STATS_MODE_SINGLE_FRAMES}, INT_MIN, INT_MAX, FLAGS, "mode" }, - { "use_alpha", "create a palette including alpha values", OFFSET(use_alpha), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS }, { NULL } }; @@ -115,16 +113,15 @@ static int cmp_##name(const void *pa, const void *pb) \ { \ const struct color_ref * const *a = pa; \ const struct color_ref * const *b = pb; \ - return (int)((*a)->color >> (8 * (3 - (pos))) & 0xff) \ - - (int)((*b)->color >> (8 * (3 - (pos))) & 0xff); \ + return (int)((*a)->color >> (8 * (2 - (pos))) & 0xff) \ + - (int)((*b)->color >> (8 * (2 - (pos))) & 0xff); \ } -DECLARE_CMP_FUNC(a, 0) -DECLARE_CMP_FUNC(r, 1) -DECLARE_CMP_FUNC(g, 2) -DECLARE_CMP_FUNC(b, 3) +DECLARE_CMP_FUNC(r, 0) +DECLARE_CMP_FUNC(g, 1) +DECLARE_CMP_FUNC(b, 2) -static const cmp_func cmp_funcs[] = {cmp_a, cmp_r, cmp_g, cmp_b}; +static const cmp_func cmp_funcs[] = {cmp_r, cmp_g, cmp_b}; /** * Simple color comparison for sorting the final palette @@ -146,17 +143,6 @@ static av_always_inline int diff(const uint32_t a, const uint32_t b) return dr*dr + dg*dg + db*db; } -static av_always_inline int diff_alpha(const uint32_t a, const uint32_t b) -{ - const uint8_t c1[] = {a >> 24 & 0xff, a >> 16 & 0xff, a >> 8 & 0xff, a & 0xff}; - const uint8_t c2[] = {b >> 24 & 0xff, b >> 16 & 0xff, b >> 8 & 0xff, b & 0xff}; - const int da = c1[0] - c2[0]; - const int dr = c1[1] - c2[1]; - const int dg = c1[2] - c2[2]; - const int db = c1[3] - c2[3]; - return da*da + dr*dr + dg*dg + db*db; -} - /** * Find the next box to split: pick the one with the highest variance */ @@ -178,10 +164,7 @@ static int get_next_box_id_to_split(PaletteGenContext *s) for (i = 0; i < box->len; i++) { const struct color_ref *ref = s->refs[box->start + i]; - if (s->use_alpha) - variance += (int64_t)diff_alpha(ref->color, box->color) * ref->count; - else - variance += (int64_t)diff(ref->color, box->color) * ref->count; + variance += diff(ref->color, box->color) * ref->count; } box->variance = variance; } @@ -201,31 +184,24 @@ static int get_next_box_id_to_split(PaletteGenContext *s) * specified box. Takes into account the weight of each color. */ static uint32_t get_avg_color(struct color_ref * const *refs, - const struct range_box *box, int use_alpha) + const struct range_box *box) { int i; const int n = box->len; - uint64_t a = 0, r = 0, g = 0, b = 0, div = 0; + uint64_t r = 0, g = 0, b = 0, div = 0; for (i = 0; i < n; i++) { const struct color_ref *ref = refs[box->start + i]; - if (use_alpha) - a += (ref->color >> 24 & 0xff) * ref->count; - r += (ref->color >> 16 & 0xff) * ref->count; - g += (ref->color >> 8 & 0xff) * ref->count; - b += (ref->color & 0xff) * ref->count; + r += (ref->color >> 16 & 0xff) * ref->count; + g += (ref->color >> 8 & 0xff) * ref->count; + b += (ref->color & 0xff) * ref->count; div += ref->count; } - if (use_alpha) - a = a / div; r = r / div; g = g / div; b = b / div; - if (use_alpha) - return a<<24 | r<<16 | g<<8 | b; - return 0xffU<<24 | r<<16 | g<<8 | b; } @@ -244,8 +220,8 @@ static void split_box(PaletteGenContext *s, struct range_box *box, int n) av_assert0(box->len >= 1); av_assert0(new_box->len >= 1); - box->color = get_avg_color(s->refs, box, s->use_alpha); - new_box->color = get_avg_color(s->refs, new_box, s->use_alpha); + box->color = get_avg_color(s->refs, box); + new_box->color = get_avg_color(s->refs, new_box); box->variance = -1; new_box->variance = -1; } @@ -275,7 +251,7 @@ static void write_palette(AVFilterContext *ctx, AVFrame *out) pal += pal_linesize; } - if (s->reserve_transparent && !s->use_alpha) { + if (s->reserve_transparent) { av_assert0(s->nb_boxes < 256); pal[out->width - pal_linesize - 1] = AV_RB32(&s->transparency_color) >> 8; } @@ -343,49 +319,40 @@ static AVFrame *get_palette_frame(AVFilterContext *ctx) box = &s->boxes[box_id]; box->len = s->nb_refs; box->sorted_by = -1; - box->color = get_avg_color(s->refs, box, s->use_alpha); + box->color = get_avg_color(s->refs, box); box->variance = -1; s->nb_boxes = 1; while (box && box->len > 1) { - int i, ar, rr, gr, br, longest; + int i, rr, gr, br, longest; uint64_t median, box_weight = 0; /* compute the box weight (sum all the weights of the colors in the * range) and its boundings */ - uint8_t min[4] = {0xff, 0xff, 0xff, 0xff}; - uint8_t max[4] = {0x00, 0x00, 0x00, 0x00}; + uint8_t min[3] = {0xff, 0xff, 0xff}; + uint8_t max[3] = {0x00, 0x00, 0x00}; for (i = box->start; i < box->start + box->len; i++) { const struct color_ref *ref = s->refs[i]; const uint32_t rgb = ref->color; - const uint8_t a = rgb >> 24 & 0xff, r = rgb >> 16 & 0xff, g = rgb >> 8 & 0xff, b = rgb & 0xff; - min[0] = FFMIN(a, min[0]); max[0] = FFMAX(a, max[0]); - min[1] = FFMIN(r, min[1]); max[1] = FFMAX(r, max[1]); - min[2] = FFMIN(g, min[2]); max[2] = FFMAX(g, max[2]); - min[3] = FFMIN(b, min[3]); max[3] = FFMAX(b, max[3]); + const uint8_t r = rgb >> 16 & 0xff, g = rgb >> 8 & 0xff, b = rgb & 0xff; + min[0] = FFMIN(r, min[0]), max[0] = FFMAX(r, max[0]); + min[1] = FFMIN(g, min[1]), max[1] = FFMAX(g, max[1]); + min[2] = FFMIN(b, min[2]), max[2] = FFMAX(b, max[2]); box_weight += ref->count; } /* define the axis to sort by according to the widest range of colors */ - ar = max[0] - min[0]; - rr = max[1] - min[1]; - gr = max[2] - min[2]; - br = max[3] - min[3]; - longest = 2; // pick green by default (the color the eye is the most sensitive to) - if (s->use_alpha) { - if (ar >= rr && ar >= br && ar >= gr) longest = 0; - if (br >= rr && br >= gr && br >= ar) longest = 3; - if (rr >= gr && rr >= br && rr >= ar) longest = 1; - if (gr >= rr && gr >= br && gr >= ar) longest = 2; // prefer green again - } else { - if (br >= rr && br >= gr) longest = 3; - if (rr >= gr && rr >= br) longest = 1; - if (gr >= rr && gr >= br) longest = 2; // prefer green again - } - - ff_dlog(ctx, "box #%02X [%6d..%-6d] (%6d) w:%-6"PRIu64" ranges:[%2x %2x %2x %2x] sort by %c (already sorted:%c) ", + rr = max[0] - min[0]; + gr = max[1] - min[1]; + br = max[2] - min[2]; + longest = 1; // pick green by default (the color the eye is the most sensitive to) + if (br >= rr && br >= gr) longest = 2; + if (rr >= gr && rr >= br) longest = 0; + if (gr >= rr && gr >= br) longest = 1; // prefer green again + + ff_dlog(ctx, "box #%02X [%6d..%-6d] (%6d) w:%-6"PRIu64" ranges:[%2x %2x %2x] sort by %c (already sorted:%c) ", box_id, box->start, box->start + box->len - 1, box->len, box_weight, - ar, rr, gr, br, "argb"[longest], box->sorted_by == longest ? 'y' : 'n'); + rr, gr, br, "rgb"[longest], box->sorted_by == longest ? 'y':'n'); /* sort the range by its longest axis if it's not already sorted */ if (box->sorted_by != longest) { @@ -427,27 +394,22 @@ static AVFrame *get_palette_frame(AVFilterContext *ctx) * It keeps the NBITS least significant bit of each component to make it * "random" even if the scene doesn't have much different colors. */ -static inline unsigned color_hash(uint32_t color, int use_alpha) +static inline unsigned color_hash(uint32_t color) { const uint8_t r = color >> 16 & ((1<<NBITS)-1); const uint8_t g = color >> 8 & ((1<<NBITS)-1); const uint8_t b = color & ((1<<NBITS)-1); - if (use_alpha) { - const uint8_t a = color >> 24 & ((1 << NBITS) - 1); - return a << (NBITS * 3) | r << (NBITS * 2) | g << NBITS | b; - } - return r << (NBITS * 2) | g << NBITS | b; } /** * Locate the color in the hash table and increment its counter. */ -static int color_inc(struct hist_node *hist, uint32_t color, int use_alpha) +static int color_inc(struct hist_node *hist, uint32_t color) { int i; - const unsigned hash = color_hash(color, use_alpha); + const unsigned hash = color_hash(color); struct hist_node *node = &hist[hash]; struct color_ref *e; @@ -472,7 +434,7 @@ static int color_inc(struct hist_node *hist, uint32_t color, int use_alpha) * Update histogram when pixels differ from previous frame. */ static int update_histogram_diff(struct hist_node *hist, - const AVFrame *f1, const AVFrame *f2, int use_alpha) + const AVFrame *f1, const AVFrame *f2) { int x, y, ret, nb_diff_colors = 0; @@ -483,7 +445,7 @@ static int update_histogram_diff(struct hist_node *hist, for (x = 0; x < f1->width; x++) { if (p[x] == q[x]) continue; - ret = color_inc(hist, p[x], use_alpha); + ret = color_inc(hist, p[x]); if (ret < 0) return ret; nb_diff_colors += ret; @@ -495,7 +457,7 @@ static int update_histogram_diff(struct hist_node *hist, /** * Simple histogram of the frame. */ -static int update_histogram_frame(struct hist_node *hist, const AVFrame *f, int use_alpha) +static int update_histogram_frame(struct hist_node *hist, const AVFrame *f) { int x, y, ret, nb_diff_colors = 0; @@ -503,7 +465,7 @@ static int update_histogram_frame(struct hist_node *hist, const AVFrame *f, int const uint32_t *p = (const uint32_t *)(f->data[0] + y*f->linesize[0]); for (x = 0; x < f->width; x++) { - ret = color_inc(hist, p[x], use_alpha); + ret = color_inc(hist, p[x]); if (ret < 0) return ret; nb_diff_colors += ret; @@ -519,8 +481,8 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) { AVFilterContext *ctx = inlink->dst; PaletteGenContext *s = ctx->priv; - int ret = s->prev_frame ? update_histogram_diff(s->histogram, s->prev_frame, in, s->use_alpha) - : update_histogram_frame(s->histogram, in, s->use_alpha); + int ret = s->prev_frame ? update_histogram_diff(s->histogram, s->prev_frame, in) + : update_histogram_frame(s->histogram, in); if (ret > 0) s->nb_refs += ret; @@ -579,16 +541,6 @@ static int config_output(AVFilterLink *outlink) return 0; } -static int init(AVFilterContext *ctx) -{ - PaletteGenContext* s = ctx->priv; - - if (s->use_alpha && s->reserve_transparent) - s->reserve_transparent = 0; - - return 0; -} - static av_cold void uninit(AVFilterContext *ctx) { int i; @@ -621,7 +573,6 @@ const AVFilter ff_vf_palettegen = { .name = "palettegen", .description = NULL_IF_CONFIG_SMALL("Find the optimal palette for a given stream."), .priv_size = sizeof(PaletteGenContext), - .init = init, .uninit = uninit, FILTER_INPUTS(palettegen_inputs), FILTER_OUTPUTS(palettegen_outputs), diff --git a/libavfilter/vf_paletteuse.c b/libavfilter/vf_paletteuse.c index a6b5d5a5fa..cb18329bb7 100644 --- a/libavfilter/vf_paletteuse.c +++ b/libavfilter/vf_paletteuse.c @@ -29,6 +29,7 @@ #include "libavutil/opt.h" #include "libavutil/qsort.h" #include "avfilter.h" +#include "filters.h" #include "framesync.h" #include "internal.h" @@ -63,7 +64,7 @@ struct color_node { }; #define NBITS 5 -#define CACHE_SIZE (1<<(4*NBITS)) +#define CACHE_SIZE (1<<(3*NBITS)) struct cached_color { uint32_t color; @@ -88,7 +89,6 @@ typedef struct PaletteUseContext { uint32_t palette[AVPALETTE_COUNT]; int transparency_index; /* index in the palette of transparency. -1 if there is no transparency in the palette. */ int trans_thresh; - int use_alpha; int palette_loaded; int dither; int new; @@ -108,7 +108,7 @@ typedef struct PaletteUseContext { } PaletteUseContext; #define OFFSET(x) offsetof(PaletteUseContext, x) -#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM static const AVOption paletteuse_options[] = { { "dither", "select dithering mode", OFFSET(dither), AV_OPT_TYPE_INT, {.i64=DITHERING_SIERRA2_4A}, 0, NB_DITHERING-1, FLAGS, "dithering_mode" }, { "bayer", "ordered 8x8 bayer dithering (deterministic)", 0, AV_OPT_TYPE_CONST, {.i64=DITHERING_BAYER}, INT_MIN, INT_MAX, FLAGS, "dithering_mode" }, @@ -121,7 +121,6 @@ static const AVOption paletteuse_options[] = { { "rectangle", "process smallest different rectangle", 0, AV_OPT_TYPE_CONST, {.i64=DIFF_MODE_RECTANGLE}, INT_MIN, INT_MAX, FLAGS, "diff_mode" }, { "new", "take new palette for each output frame", OFFSET(new), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS }, { "alpha_threshold", "set the alpha threshold for transparency", OFFSET(trans_thresh), AV_OPT_TYPE_INT, {.i64=128}, 0, 255, FLAGS }, - { "use_alpha", "use alpha channel for mapping", OFFSET(use_alpha), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS }, /* following are the debug options, not part of the official API */ { "debug_kdtree", "save Graphviz graph of the kdtree in specified file", OFFSET(dot_filename), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS }, @@ -163,41 +162,37 @@ static av_always_inline uint32_t dither_color(uint32_t px, int er, int eg, | av_clip_uint8((px & 0xff) + ((eb * scale) / (1<<shift))); } -static av_always_inline int diff(const uint8_t *c1, const uint8_t *c2, const PaletteUseContext *s) +static av_always_inline int diff(const uint8_t *c1, const uint8_t *c2, const int trans_thresh) { // XXX: try L*a*b with CIE76 (dL*dL + da*da + db*db) - const int da = c1[0] - c2[0]; const int dr = c1[1] - c2[1]; const int dg = c1[2] - c2[2]; const int db = c1[3] - c2[3]; - if (s->use_alpha) - return da*da + dr*dr + dg*dg + db*db; - - if (c1[0] < s->trans_thresh && c2[0] < s->trans_thresh) { + if (c1[0] < trans_thresh && c2[0] < trans_thresh) { return 0; - } else if (c1[0] >= s->trans_thresh && c2[0] >= s->trans_thresh) { + } else if (c1[0] >= trans_thresh && c2[0] >= trans_thresh) { return dr*dr + dg*dg + db*db; } else { return 255*255 + 255*255 + 255*255; } } -static av_always_inline uint8_t colormap_nearest_bruteforce(const PaletteUseContext *s, const uint8_t *argb) +static av_always_inline uint8_t colormap_nearest_bruteforce(const uint32_t *palette, const uint8_t *argb, const int trans_thresh) { int i, pal_id = -1, min_dist = INT_MAX; for (i = 0; i < AVPALETTE_COUNT; i++) { - const uint32_t c = s->palette[i]; + const uint32_t c = palette[i]; - if (s->use_alpha || c >> 24 >= s->trans_thresh) { // ignore transparent entry + if (c >> 24 >= trans_thresh) { // ignore transparent entry const uint8_t palargb[] = { - s->palette[i]>>24 & 0xff, - s->palette[i]>>16 & 0xff, - s->palette[i]>> 8 & 0xff, - s->palette[i] & 0xff, + palette[i]>>24 & 0xff, + palette[i]>>16 & 0xff, + palette[i]>> 8 & 0xff, + palette[i] & 0xff, }; - const int d = diff(palargb, argb, s); + const int d = diff(palargb, argb, trans_thresh); if (d < min_dist) { pal_id = i; min_dist = d; @@ -213,17 +208,17 @@ struct nearest_color { int dist_sqd; }; -static void colormap_nearest_node(const PaletteUseContext *s, - const struct color_node *map, +static void colormap_nearest_node(const struct color_node *map, const int node_pos, const uint8_t *target, + const int trans_thresh, struct nearest_color *nearest) { const struct color_node *kd = map + node_pos; - const int split = kd->split; + const int s = kd->split; int dx, nearer_kd_id, further_kd_id; const uint8_t *current = kd->val; - const int current_to_target = diff(target, current, s); + const int current_to_target = diff(target, current, trans_thresh); if (current_to_target < nearest->dist_sqd) { nearest->node_pos = node_pos; @@ -231,23 +226,23 @@ static void colormap_nearest_node(const PaletteUseContext *s, } if (kd->left_id != -1 || kd->right_id != -1) { - dx = target[split] - current[split]; + dx = target[s] - current[s]; if (dx <= 0) nearer_kd_id = kd->left_id, further_kd_id = kd->right_id; else nearer_kd_id = kd->right_id, further_kd_id = kd->left_id; if (nearer_kd_id != -1) - colormap_nearest_node(s, map, nearer_kd_id, target, nearest); + colormap_nearest_node(map, nearer_kd_id, target, trans_thresh, nearest); if (further_kd_id != -1 && dx*dx < nearest->dist_sqd) - colormap_nearest_node(s, map, further_kd_id, target, nearest); + colormap_nearest_node(map, further_kd_id, target, trans_thresh, nearest); } } -static av_always_inline uint8_t colormap_nearest_recursive(const PaletteUseContext *s, const struct color_node *node, const uint8_t *rgb) +static av_always_inline uint8_t colormap_nearest_recursive(const struct color_node *node, const uint8_t *rgb, const int trans_thresh) { struct nearest_color res = {.dist_sqd = INT_MAX, .node_pos = -1}; - colormap_nearest_node(s, node, 0, rgb, &res); + colormap_nearest_node(node, 0, rgb, trans_thresh, &res); return node[res.node_pos].palette_id; } @@ -256,7 +251,7 @@ struct stack_node { int dx2; }; -static av_always_inline uint8_t colormap_nearest_iterative(const PaletteUseContext *s, const struct color_node *root, const uint8_t *target) +static av_always_inline uint8_t colormap_nearest_iterative(const struct color_node *root, const uint8_t *target, const int trans_thresh) { int pos = 0, best_node_id = -1, best_dist = INT_MAX, cur_color_id = 0; struct stack_node nodes[16]; @@ -266,7 +261,7 @@ static av_always_inline uint8_t colormap_nearest_iterative(const PaletteUseConte const struct color_node *kd = &root[cur_color_id]; const uint8_t *current = kd->val; - const int current_to_target = diff(target, current, s); + const int current_to_target = diff(target, current, trans_thresh); /* Compare current color node to the target and update our best node if * it's actually better. */ @@ -328,10 +323,10 @@ end: return root[best_node_id].palette_id; } -#define COLORMAP_NEAREST(s, search, root, target) \ - search == COLOR_SEARCH_NNS_ITERATIVE ? colormap_nearest_iterative(s, root, target) : \ - search == COLOR_SEARCH_NNS_RECURSIVE ? colormap_nearest_recursive(s, root, target) : \ - colormap_nearest_bruteforce(s, target) +#define COLORMAP_NEAREST(search, palette, root, target, trans_thresh) \ + search == COLOR_SEARCH_NNS_ITERATIVE ? colormap_nearest_iterative(root, target, trans_thresh) : \ + search == COLOR_SEARCH_NNS_RECURSIVE ? colormap_nearest_recursive(root, target, trans_thresh) : \ + colormap_nearest_bruteforce(palette, target, trans_thresh) /** * Check if the requested color is in the cache already. If not, find it in the @@ -368,13 +363,13 @@ static av_always_inline int color_get(PaletteUseContext *s, uint32_t color, if (!e) return AVERROR(ENOMEM); e->color = color; - e->pal_entry = COLORMAP_NEAREST(s, search_method, s->map, argb_elts); + e->pal_entry = COLORMAP_NEAREST(search_method, s->palette, s->map, argb_elts, s->trans_thresh); return e->pal_entry; } static av_always_inline int get_dst_color_err(PaletteUseContext *s, - uint32_t c, int *ea, int *er, int *eg, int *eb, + uint32_t c, int *er, int *eg, int *eb, const enum color_search_method search_method) { const uint8_t a = c >> 24 & 0xff; @@ -387,9 +382,8 @@ static av_always_inline int get_dst_color_err(PaletteUseContext *s, return dstx; dstc = s->palette[dstx]; if (dstx == s->transparency_index) { - *ea =*er = *eg = *eb = 0; + *er = *eg = *eb = 0; } else { - *ea = (int)a - (int)(dstc >> 24 & 0xff); *er = (int)r - (int)(dstc >> 16 & 0xff); *eg = (int)g - (int)(dstc >> 8 & 0xff); *eb = (int)b - (int)(dstc & 0xff); @@ -413,7 +407,7 @@ static av_always_inline int set_frame(PaletteUseContext *s, AVFrame *out, AVFram for (y = y_start; y < h; y++) { for (x = x_start; x < w; x++) { - int ea, er, eg, eb; + int er, eg, eb; if (dither == DITHERING_BAYER) { const int d = s->ordered_dither[(y & 7)<<3 | (x & 7)]; @@ -433,7 +427,7 @@ static av_always_inline int set_frame(PaletteUseContext *s, AVFrame *out, AVFram } else if (dither == DITHERING_HECKBERT) { const int right = x < w - 1, down = y < h - 1; - const int color = get_dst_color_err(s, src[x], &ea, &er, &eg, &eb, search_method); + const int color = get_dst_color_err(s, src[x], &er, &eg, &eb, search_method); if (color < 0) return color; @@ -445,7 +439,7 @@ static av_always_inline int set_frame(PaletteUseContext *s, AVFrame *out, AVFram } else if (dither == DITHERING_FLOYD_STEINBERG) { const int right = x < w - 1, down = y < h - 1, left = x > x_start; - const int color = get_dst_color_err(s, src[x], &ea, &er, &eg, &eb, search_method); + const int color = get_dst_color_err(s, src[x], &er, &eg, &eb, search_method); if (color < 0) return color; @@ -459,7 +453,7 @@ static av_always_inline int set_frame(PaletteUseContext *s, AVFrame *out, AVFram } else if (dither == DITHERING_SIERRA2) { const int right = x < w - 1, down = y < h - 1, left = x > x_start; const int right2 = x < w - 2, left2 = x > x_start + 1; - const int color = get_dst_color_err(s, src[x], &ea, &er, &eg, &eb, search_method); + const int color = get_dst_color_err(s, src[x], &er, &eg, &eb, search_method); if (color < 0) return color; @@ -478,7 +472,7 @@ static av_always_inline int set_frame(PaletteUseContext *s, AVFrame *out, AVFram } else if (dither == DITHERING_SIERRA2_4A) { const int right = x < w - 1, down = y < h - 1, left = x > x_start; - const int color = get_dst_color_err(s, src[x], &ea, &er, &eg, &eb, search_method); + const int color = get_dst_color_err(s, src[x], &er, &eg, &eb, search_method); if (color < 0) return color; @@ -561,7 +555,8 @@ static int disp_tree(const struct color_node *node, const char *fname) return 0; } -static int debug_accuracy(const PaletteUseContext *s) +static int debug_accuracy(const struct color_node *node, const uint32_t *palette, const int trans_thresh, + const enum color_search_method search_method) { int r, g, b, ret = 0; @@ -569,26 +564,19 @@ static int debug_accuracy(const PaletteUseContext *s) for (g = 0; g < 256; g++) { for (b = 0; b < 256; b++) { const uint8_t argb[] = {0xff, r, g, b}; - const int r1 = COLORMAP_NEAREST(s, s->color_search_method, s->map, argb); - const int r2 = colormap_nearest_bruteforce(s, argb); + const int r1 = COLORMAP_NEAREST(search_method, palette, node, argb, trans_thresh); + const int r2 = colormap_nearest_bruteforce(palette, argb, trans_thresh); if (r1 != r2) { - const uint32_t c1 = s->palette[r1]; - const uint32_t c2 = s->palette[r2]; - const uint8_t a1 = s->use_alpha ? c1>>24 & 0xff : 0xff; - const uint8_t a2 = s->use_alpha ? c2>>24 & 0xff : 0xff; - const uint8_t palargb1[] = { a1, c1>>16 & 0xff, c1>> 8 & 0xff, c1 & 0xff }; - const uint8_t palargb2[] = { a2, c2>>16 & 0xff, c2>> 8 & 0xff, c2 & 0xff }; - const int d1 = diff(palargb1, argb, s); - const int d2 = diff(palargb2, argb, s); + const uint32_t c1 = palette[r1]; + const uint32_t c2 = palette[r2]; + const uint8_t palargb1[] = { 0xff, c1>>16 & 0xff, c1>> 8 & 0xff, c1 & 0xff }; + const uint8_t palargb2[] = { 0xff, c2>>16 & 0xff, c2>> 8 & 0xff, c2 & 0xff }; + const int d1 = diff(palargb1, argb, trans_thresh); + const int d2 = diff(palargb2, argb, trans_thresh); if (d1 != d2) { - if (s->use_alpha) - av_log(NULL, AV_LOG_ERROR, - "/!\\ %02X%02X%02X: %d ! %d (%08"PRIX32" ! %08"PRIX32") / dist: %d ! %d\n", - r, g, b, r1, r2, c1, c2, d1, d2); - else - av_log(NULL, AV_LOG_ERROR, - "/!\\ %02X%02X%02X: %d ! %d (%06"PRIX32" ! %06"PRIX32") / dist: %d ! %d\n", - r, g, b, r1, r2, c1 & 0xffffff, c2 & 0xffffff, d1, d2); + av_log(NULL, AV_LOG_ERROR, + "/!\\ %02X%02X%02X: %d ! %d (%06"PRIX32" ! %06"PRIX32") / dist: %d ! %d\n", + r, g, b, r1, r2, c1 & 0xffffff, c2 & 0xffffff, d1, d2); ret = 1; } } @@ -604,8 +592,8 @@ struct color { }; struct color_rect { - uint8_t min[4]; - uint8_t max[4]; + uint8_t min[3]; + uint8_t max[3]; }; typedef int (*cmp_func)(const void *, const void *); @@ -626,47 +614,43 @@ DECLARE_CMP_FUNC(b, 3) static const cmp_func cmp_funcs[] = {cmp_a, cmp_r, cmp_g, cmp_b}; -static int get_next_color(const uint8_t *color_used, const PaletteUseContext *s, +static int get_next_color(const uint8_t *color_used, const uint32_t *palette, + const int trans_thresh, int *component, const struct color_rect *box) { - int wa, wr, wg, wb; + int wr, wg, wb; int i, longest = 0; unsigned nb_color = 0; struct color_rect ranges; struct color tmp_pal[256]; cmp_func cmpf; - ranges.min[0] = ranges.min[1] = ranges.min[2] = ranges.min[3]= 0xff; - ranges.max[0] = ranges.max[1] = ranges.max[2] = ranges.max[3]= 0x00; + ranges.min[0] = ranges.min[1] = ranges.min[2] = 0xff; + ranges.max[0] = ranges.max[1] = ranges.max[2] = 0x00; for (i = 0; i < AVPALETTE_COUNT; i++) { - const uint32_t c = s->palette[i]; + const uint32_t c = palette[i]; const uint8_t a = c >> 24 & 0xff; const uint8_t r = c >> 16 & 0xff; const uint8_t g = c >> 8 & 0xff; const uint8_t b = c & 0xff; - if (!s->use_alpha && a < s->trans_thresh) { + if (a < trans_thresh) { continue; } - if (color_used[i] || (a != 0xff && !s->use_alpha) || - r < box->min[1] || g < box->min[2] || b < box->min[3] || - r > box->max[1] || g > box->max[2] || b > box->max[3]) + if (color_used[i] || (a != 0xff) || + r < box->min[0] || g < box->min[1] || b < box->min[2] || + r > box->max[0] || g > box->max[1] || b > box->max[2]) continue; - if (s->use_alpha && (a < box->min[0] || a > box->max[0])) - continue; - - if (a < ranges.min[0]) ranges.min[0] = a; - if (r < ranges.min[1]) ranges.min[1] = r; - if (g < ranges.min[2]) ranges.min[2] = g; - if (b < ranges.min[3]) ranges.min[3] = b; + if (r < ranges.min[0]) ranges.min[0] = r; + if (g < ranges.min[1]) ranges.min[1] = g; + if (b < ranges.min[2]) ranges.min[2] = b; - if (a > ranges.max[0]) ranges.max[0] = a; - if (r > ranges.max[1]) ranges.max[1] = r; - if (g > ranges.max[2]) ranges.max[2] = g; - if (b > ranges.max[3]) ranges.max[3] = b; + if (r > ranges.max[0]) ranges.max[0] = r; + if (g > ranges.max[1]) ranges.max[1] = g; + if (b > ranges.max[2]) ranges.max[2] = b; tmp_pal[nb_color].value = c; tmp_pal[nb_color].pal_id = i; @@ -678,22 +662,12 @@ static int get_next_color(const uint8_t *color_used, const PaletteUseContext *s, return -1; /* define longest axis that will be the split component */ - wa = ranges.max[0] - ranges.min[0]; - wr = ranges.max[1] - ranges.min[1]; - wg = ranges.max[2] - ranges.min[2]; - wb = ranges.max[3] - ranges.min[3]; - - if (s->use_alpha) { - if (wa >= wr && wa >= wb && wa >= wg) longest = 0; - if (wr >= wg && wr >= wb && wr >= wa) longest = 1; - if (wg >= wr && wg >= wb && wg >= wa) longest = 2; - if (wb >= wr && wb >= wg && wb >= wa) longest = 3; - } else { - if (wr >= wg && wr >= wb) longest = 1; - if (wg >= wr && wg >= wb) longest = 2; - if (wb >= wr && wb >= wg) longest = 3; - } - + wr = ranges.max[0] - ranges.min[0]; + wg = ranges.max[1] - ranges.min[1]; + wb = ranges.max[2] - ranges.min[2]; + if (wr >= wg && wr >= wb) longest = 1; + if (wg >= wr && wg >= wb) longest = 2; + if (wb >= wr && wb >= wg) longest = 3; cmpf = cmp_funcs[longest]; *component = longest; @@ -706,7 +680,8 @@ static int get_next_color(const uint8_t *color_used, const PaletteUseContext *s, static int colormap_insert(struct color_node *map, uint8_t *color_used, int *nb_used, - const PaletteUseContext *s, + const uint32_t *palette, + const int trans_thresh, const struct color_rect *box) { uint32_t c; @@ -714,14 +689,14 @@ static int colormap_insert(struct color_node *map, int node_left_id = -1, node_right_id = -1; struct color_node *node; struct color_rect box1, box2; - const int pal_id = get_next_color(color_used, s, &component, box); + const int pal_id = get_next_color(color_used, palette, trans_thresh, &component, box); if (pal_id < 0) return -1; /* create new node with that color */ cur_id = (*nb_used)++; - c = s->palette[pal_id]; + c = palette[pal_id]; node = &map[cur_id]; node->split = component; node->palette_id = pal_id; @@ -734,13 +709,13 @@ static int colormap_insert(struct color_node *map, /* get the two boxes this node creates */ box1 = box2 = *box; - box1.max[component] = node->val[component]; - box2.min[component] = FFMIN(node->val[component] + 1, 255); + box1.max[component-1] = node->val[component]; + box2.min[component-1] = FFMIN(node->val[component] + 1, 255); - node_left_id = colormap_insert(map, color_used, nb_used, s, &box1); + node_left_id = colormap_insert(map, color_used, nb_used, palette, trans_thresh, &box1); - if (box2.min[component] <= box2.max[component]) - node_right_id = colormap_insert(map, color_used, nb_used, s, &box2); + if (box2.min[component-1] <= box2.max[component-1]) + node_right_id = colormap_insert(map, color_used, nb_used, palette, trans_thresh, &box2); node->left_id = node_left_id; node->right_id = node_right_id; @@ -755,13 +730,6 @@ static int cmp_pal_entry(const void *a, const void *b) return c1 - c2; } -static int cmp_pal_entry_alpha(const void *a, const void *b) -{ - const int c1 = *(const uint32_t *)a; - const int c2 = *(const uint32_t *)b; - return c1 - c2; -} - static void load_colormap(PaletteUseContext *s) { int i, nb_used = 0; @@ -769,13 +737,12 @@ static void load_colormap(PaletteUseContext *s) uint32_t last_color = 0; struct color_rect box; - if (!s->use_alpha && s->transparency_index >= 0) { + if (s->transparency_index >= 0) { FFSWAP(uint32_t, s->palette[s->transparency_index], s->palette[255]); } /* disable transparent colors and dups */ - qsort(s->palette, AVPALETTE_COUNT-(s->transparency_index >= 0), sizeof(*s->palette), - s->use_alpha ? cmp_pal_entry_alpha : cmp_pal_entry); + qsort(s->palette, AVPALETTE_COUNT-(s->transparency_index >= 0), sizeof(*s->palette), cmp_pal_entry); for (i = 0; i < AVPALETTE_COUNT; i++) { const uint32_t c = s->palette[i]; @@ -784,22 +751,22 @@ static void load_colormap(PaletteUseContext *s) continue; } last_color = c; - if (!s->use_alpha && c >> 24 < s->trans_thresh) { + if (c >> 24 < s->trans_thresh) { color_used[i] = 1; // ignore transparent color(s) continue; } } - box.min[0] = box.min[1] = box.min[2] = box.min[3] = 0x00; - box.max[0] = box.max[1] = box.max[2] = box.max[3] = 0xff; + box.min[0] = box.min[1] = box.min[2] = 0x00; + box.max[0] = box.max[1] = box.max[2] = 0xff; - colormap_insert(s->map, color_used, &nb_used, s, &box); + colormap_insert(s->map, color_used, &nb_used, s->palette, s->trans_thresh, &box); if (s->dot_filename) disp_tree(s->map, s->dot_filename); if (s->debug_accuracy) { - if (!debug_accuracy(s)) + if (!debug_accuracy(s->map, s->palette, s->trans_thresh, s->color_search_method)) av_log(NULL, AV_LOG_INFO, "Accuracy check passed\n"); } } @@ -813,18 +780,16 @@ static void debug_mean_error(PaletteUseContext *s, const AVFrame *in1, uint8_t *src2 = in2->data[0]; const int src1_linesize = in1->linesize[0] >> 2; const int src2_linesize = in2->linesize[0]; - const float div = in1->width * in1->height * (s->use_alpha ? 4 : 3); + const float div = in1->width * in1->height * 3; unsigned mean_err = 0; for (y = 0; y < in1->height; y++) { for (x = 0; x < in1->width; x++) { const uint32_t c1 = src1[x]; const uint32_t c2 = palette[src2[x]]; - const uint8_t a1 = s->use_alpha ? c1>>24 & 0xff : 0xff; - const uint8_t a2 = s->use_alpha ? c2>>24 & 0xff : 0xff; - const uint8_t argb1[] = {a1, c1 >> 16 & 0xff, c1 >> 8 & 0xff, c1 & 0xff}; - const uint8_t argb2[] = {a2, c2 >> 16 & 0xff, c2 >> 8 & 0xff, c2 & 0xff}; - mean_err += diff(argb1, argb2, s); + const uint8_t argb1[] = {0xff, c1 >> 16 & 0xff, c1 >> 8 & 0xff, c1 & 0xff}; + const uint8_t argb2[] = {0xff, c2 >> 16 & 0xff, c2 >> 8 & 0xff, c2 & 0xff}; + mean_err += diff(argb1, argb2, s->trans_thresh); } src1 += src1_linesize; src2 += src2_linesize; @@ -1024,7 +989,7 @@ static void load_palette(PaletteUseContext *s, const AVFrame *palette_frame) for (y = 0; y < palette_frame->height; y++) { for (x = 0; x < palette_frame->width; x++) { s->palette[i] = p[x]; - if (!s->use_alpha && p[x]>>24 < s->trans_thresh) { + if (p[x]>>24 < s->trans_thresh) { s->transparency_index = i; // we are assuming at most one transparent color in palette } i++; -- 2.38.1

4 17

[PATCH v6] libavfilter/x86/vf_convolution: add sobel filter optimization and unit test with intel AVX512 VNNI
by bin.wang＠intel.com 01 Nov '22

01 Nov '22

From: bwang30 <bin.wang(a)intel.com> This commit enabled assembly code with intel AVX512 VNNI and added unit test for sobel filter sobel_c: 4537 sobel_avx512icl 2136 Signed-off-by: bwang30 <bin.wang(a)intel.com> --- libavfilter/convolution.h | 74 +++++++++++++ libavfilter/vf_convolution.c | 91 +++------------- libavfilter/x86/vf_convolution.asm | 147 ++++++++++++++++++++++++++ libavfilter/x86/vf_convolution_init.c | 18 ++++ tests/checkasm/Makefile | 1 + tests/checkasm/checkasm.c | 3 + tests/checkasm/checkasm.h | 1 + tests/checkasm/vf_convolution.c | 103 ++++++++++++++++++ 8 files changed, 360 insertions(+), 78 deletions(-) create mode 100644 tests/checkasm/vf_convolution.c diff --git a/libavfilter/convolution.h b/libavfilter/convolution.h index 88aabe9a20..e44bfb5da8 100644 --- a/libavfilter/convolution.h +++ b/libavfilter/convolution.h @@ -21,6 +21,7 @@ #ifndef AVFILTER_CONVOLUTION_H #define AVFILTER_CONVOLUTION_H #include "avfilter.h" +#include "libavutil/intreadwrite.h" enum MatrixMode { MATRIX_SQUARE, @@ -61,4 +62,77 @@ typedef struct ConvolutionContext { } ConvolutionContext; void ff_convolution_init_x86(ConvolutionContext *s); +void ff_sobel_init_x86(ConvolutionContext *s, int depth, int nb_planes); + +static void setup_3x3(int radius, const uint8_t *c[], const uint8_t *src, int stride, + int x, int w, int y, int h, int bpc) +{ + int i; + + for (i = 0; i < 9; i++) { + int xoff = FFABS(x + ((i % 3) - 1)); + int yoff = FFABS(y + (i / 3) - 1); + + xoff = xoff >= w ? 2 * w - 1 - xoff : xoff; + yoff = yoff >= h ? 2 * h - 1 - yoff : yoff; + + c[i] = src + xoff * bpc + yoff * stride; + } +} + +static void filter_sobel(uint8_t *dst, int width, + float scale, float delta, const int *const matrix, + const uint8_t *c[], int peak, int radius, + int dstride, int stride, int size) +{ + const uint8_t *c0 = c[0], *c1 = c[1], *c2 = c[2]; + const uint8_t *c3 = c[3], *c5 = c[5]; + const uint8_t *c6 = c[6], *c7 = c[7], *c8 = c[8]; + int x; + + for (x = 0; x < width; x++) { + float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 + + c6[x] * 1 + c7[x] * 2 + c8[x] * 1; + float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 + + c5[x] * 2 + c6[x] * -1 + c8[x] * 1; + + dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta); + } +} + +static void filter16_sobel(uint8_t *dstp, int width, + float scale, float delta, const int *const matrix, + const uint8_t *c[], int peak, int radius, + int dstride, int stride, int size) +{ + uint16_t *dst = (uint16_t *)dstp; + int x; + + for (x = 0; x < width; x++) { + float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 + + AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1; + float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 + + AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1; + + dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak); + } +} + +static av_unused void ff_sobel_init(ConvolutionContext *s, int depth, int nb_planes) +{ + for (int i = 0; i < 4; i++) { + s->filter[i] = filter_sobel; + s->copy[i] = !((1 << i) & s->planes); + s->size[i] = 3; + s->setup[i] = setup_3x3; + s->rdiv[i] = s->scale; + s->bias[i] = s->delta; + } + if (s->depth > 8) + for (int i = 0; i < 4; i++) + s->filter[i] = filter16_sobel; +#if ARCH_X86_64 + ff_sobel_init_x86(s, depth, nb_planes); +#endif +} #endif diff --git a/libavfilter/vf_convolution.c b/libavfilter/vf_convolution.c index 9a9c099e6d..7762fa2a05 100644 --- a/libavfilter/vf_convolution.c +++ b/libavfilter/vf_convolution.c @@ -139,24 +139,6 @@ static void filter16_roberts(uint8_t *dstp, int width, } } -static void filter16_sobel(uint8_t *dstp, int width, - float scale, float delta, const int *const matrix, - const uint8_t *c[], int peak, int radius, - int dstride, int stride, int size) -{ - uint16_t *dst = (uint16_t *)dstp; - int x; - - for (x = 0; x < width; x++) { - float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 + - AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1; - float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 + - AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1; - - dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak); - } -} - static void filter16_scharr(uint8_t *dstp, int width, float scale, float delta, const int *const matrix, const uint8_t *c[], int peak, int radius, @@ -261,26 +243,6 @@ static void filter_roberts(uint8_t *dst, int width, } } -static void filter_sobel(uint8_t *dst, int width, - float scale, float delta, const int *const matrix, - const uint8_t *c[], int peak, int radius, - int dstride, int stride, int size) -{ - const uint8_t *c0 = c[0], *c1 = c[1], *c2 = c[2]; - const uint8_t *c3 = c[3], *c5 = c[5]; - const uint8_t *c6 = c[6], *c7 = c[7], *c8 = c[8]; - int x; - - for (x = 0; x < width; x++) { - float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 + - c6[x] * 1 + c7[x] * 2 + c8[x] * 1; - float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 + - c5[x] * 2 + c6[x] * -1 + c8[x] * 1; - - dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta); - } -} - static void filter_scharr(uint8_t *dst, int width, float scale, float delta, const int *const matrix, const uint8_t *c[], int peak, int radius, @@ -552,22 +514,6 @@ static void filter_column(uint8_t *dst, int height, } } -static void setup_3x3(int radius, const uint8_t *c[], const uint8_t *src, int stride, - int x, int w, int y, int h, int bpc) -{ - int i; - - for (i = 0; i < 9; i++) { - int xoff = FFABS(x + ((i % 3) - 1)); - int yoff = FFABS(y + (i / 3) - 1); - - xoff = xoff >= w ? 2 * w - 1 - xoff : xoff; - yoff = yoff >= h ? 2 * h - 1 - yoff : yoff; - - c[i] = src + xoff * bpc + yoff * stride; - } -} - static void setup_5x5(int radius, const uint8_t *c[], const uint8_t *src, int stride, int x, int w, int y, int h, int bpc) { @@ -708,6 +654,18 @@ static int param_init(AVFilterContext *ctx) const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); int p, i; + s->depth = desc->comp[0].depth; + s->max = (1 << s->depth) - 1; + + s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w); + s->planewidth[0] = s->planewidth[3] = inlink->w; + s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h); + s->planeheight[0] = s->planeheight[3] = inlink->h; + + s->nb_planes = av_pix_fmt_count_planes(inlink->format); + s->nb_threads = ff_filter_get_nb_threads(ctx); + s->bpc = (s->depth + 7) / 8; + if (!strcmp(ctx->filter->name, "convolution")) { for (i = 0; i < 4; i++) { int *matrix = (int *)s->matrix[i]; @@ -804,14 +762,7 @@ static int param_init(AVFilterContext *ctx) s->bias[i] = s->delta; } } else if (!strcmp(ctx->filter->name, "sobel")) { - for (i = 0; i < 4; i++) { - s->filter[i] = filter_sobel; - s->copy[i] = !((1 << i) & s->planes); - s->size[i] = 3; - s->setup[i] = setup_3x3; - s->rdiv[i] = s->scale; - s->bias[i] = s->delta; - } + ff_sobel_init(s, s->depth, s->nb_planes); } else if (!strcmp(ctx->filter->name, "kirsch")) { for (i = 0; i < 4; i++) { s->filter[i] = filter_kirsch; @@ -832,18 +783,6 @@ static int param_init(AVFilterContext *ctx) } } - s->depth = desc->comp[0].depth; - s->max = (1 << s->depth) - 1; - - s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w); - s->planewidth[0] = s->planewidth[3] = inlink->w; - s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h); - s->planeheight[0] = s->planeheight[3] = inlink->h; - - s->nb_planes = av_pix_fmt_count_planes(inlink->format); - s->nb_threads = ff_filter_get_nb_threads(ctx); - s->bpc = (s->depth + 7) / 8; - if (!strcmp(ctx->filter->name, "convolution")) { if (s->depth > 8) { for (p = 0; p < s->nb_planes; p++) { @@ -870,10 +809,6 @@ static int param_init(AVFilterContext *ctx) if (s->depth > 8) for (p = 0; p < s->nb_planes; p++) s->filter[p] = filter16_roberts; - } else if (!strcmp(ctx->filter->name, "sobel")) { - if (s->depth > 8) - for (p = 0; p < s->nb_planes; p++) - s->filter[p] = filter16_sobel; } else if (!strcmp(ctx->filter->name, "kirsch")) { if (s->depth > 8) for (p = 0; p < s->nb_planes; p++) diff --git a/libavfilter/x86/vf_convolution.asm b/libavfilter/x86/vf_convolution.asm index 754d4d1064..c912d56752 100644 --- a/libavfilter/x86/vf_convolution.asm +++ b/libavfilter/x86/vf_convolution.asm @@ -22,6 +22,18 @@ SECTION_RODATA half: dd 0.5 +data_p1: dd 1 +data_n1: dd -1 +data_p2: dd 2 +data_n2: dd -2 + +ALIGN 64 +sobel_perm: db 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51 + db 4, 20, 36, 52, 5, 21, 37, 53, 6, 22, 38, 54, 7, 23, 39, 55 + db 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59 + db 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63 +sobel_mulA: db -1, 1, -2, 2 +sobel_mulB: db 1, -1, 2, -2 SECTION .text @@ -154,3 +166,138 @@ cglobal filter_3x3, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c INIT_XMM sse4 FILTER_3X3 %endif + +%macro SOBEL_MUL 2 + movzx ptrd, byte [c%1q + xq] + imul ptrd, [%2] + add rd, ptrd +%endmacro + +%macro SOBEL_ADD 1 + movzx ptrd, byte [c%1q + xq] + add rd, ptrd +%endmacro + +; void filter_sobel_avx512(uint8_t *dst, int width, +; float scale, float delta, const int *const matrix, +; const uint8_t *c[], int peak, int radius, +; int dstride, int stride) +%macro FILTER_SOBEL 0 +%if UNIX64 +cglobal filter_sobel, 4, 15, 7, dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x +%else +cglobal filter_sobel, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x +%endif +%if WIN64 + SWAP xmm0, xmm2 + SWAP xmm1, xmm3 + mov r2q, matrixmp + mov r3q, ptrmp + DEFINE_ARGS dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x +%endif + movsxdifnidn widthq, widthd + VBROADCASTSS m0, xmm0 + VBROADCASTSS m1, xmm1 + pxor m6, m6 + mov c0q, [ptrq + 0*gprsize] + mov c1q, [ptrq + 1*gprsize] + mov c2q, [ptrq + 2*gprsize] + mov c3q, [ptrq + 3*gprsize] + mov c4q, [ptrq + 4*gprsize] + mov c5q, [ptrq + 5*gprsize] + mov c6q, [ptrq + 6*gprsize] + mov c7q, [ptrq + 7*gprsize] + mov c8q, [ptrq + 8*gprsize] + + xor xq, xq + cmp widthq, mmsize/4 + jl .loop2 + + mov rq, widthq + and rq, mmsize/4-1 + sub widthq, rq + + mova m6, [sobel_perm] +.loop1: + movu xm3, [c2q + xq] + pmovzxbd m5, [c0q + xq] + vinserti32x4 ym3, [c6q + xq], 1 + pmovzxbd m4, [c8q + xq] + vinserti32x4 m2, m3, [c1q + xq], 2 + vinserti32x4 m3, [c5q + xq], 2 + vinserti32x4 m2, [c7q + xq], 3 + vinserti32x4 m3, [c3q + xq], 3 + vpermb m2, m6, m2 + psubd m4, m5 + vpermb m3, m6, m3 + mova m5, m4 + vpdpbusd m4, m2, [sobel_mulA] {1to16} + vpdpbusd m5, m3, [sobel_mulB] {1to16} + + cvtdq2ps m4, m4 + mulps m4, m4 + + cvtdq2ps m5, m5 + VFMADD231PS m4, m5, m5 + + sqrtps m4, m4 + fmaddps m4, m4, m0, m1 + cvttps2dq m4, m4 + vpmovusdb [dstq + xq], m4 + + add xq, mmsize/4 + cmp xq, widthq + jl .loop1 + + add widthq, rq + cmp xq, widthq + jge .end + +.loop2: + xor rd, rd + pxor m4, m4 + + ;Gx + SOBEL_MUL 0, data_n1 + SOBEL_MUL 1, data_n2 + SOBEL_MUL 2, data_n1 + SOBEL_ADD 6 + SOBEL_MUL 7, data_p2 + SOBEL_ADD 8 + + cvtsi2ss xmm4, rd + mulss xmm4, xmm4 + + xor rd, rd + ;Gy + SOBEL_MUL 0, data_n1 + SOBEL_ADD 2 + SOBEL_MUL 3, data_n2 + SOBEL_MUL 5, data_p2 + SOBEL_MUL 6, data_n1 + SOBEL_ADD 8 + + cvtsi2ss xmm5, rd + fmaddss xmm4, xmm5, xmm5, xmm4 + + sqrtps xmm4, xmm4 + fmaddss xmm4, xmm4, xmm0, xmm1 ;sum = sum * rdiv + bias + cvttps2dq xmm4, xmm4 ; trunc to integer + packssdw xmm4, xmm4 + packuswb xmm4, xmm4 + movd rd, xmm4 + mov [dstq + xq], rb + + add xq, 1 + cmp xq, widthq + jl .loop2 +.end: + RET +%endmacro + +%if ARCH_X86_64 +%if HAVE_AVX512ICL_EXTERNAL +INIT_ZMM avx512icl +FILTER_SOBEL +%endif +%endif diff --git a/libavfilter/x86/vf_convolution_init.c b/libavfilter/x86/vf_convolution_init.c index b78a47d02b..bff10ca1a4 100644 --- a/libavfilter/x86/vf_convolution_init.c +++ b/libavfilter/x86/vf_convolution_init.c @@ -29,6 +29,11 @@ void ff_filter_3x3_sse4(uint8_t *dst, int width, const uint8_t *c[], int peak, int radius, int dstride, int stride, int size); +void ff_filter_sobel_avx512icl(uint8_t *dst, int width, + float scale, float delta, const int *const matrix, + const uint8_t *c[], int peak, int radius, + int dstride, int stride, int size); + av_cold void ff_convolution_init_x86(ConvolutionContext *s) { #if ARCH_X86_64 @@ -44,3 +49,16 @@ av_cold void ff_convolution_init_x86(ConvolutionContext *s) } #endif } + +av_cold void ff_sobel_init_x86(ConvolutionContext *s, int depth, int nb_planes) +{ +#if ARCH_X86_64 + int cpu_flags = av_get_cpu_flags(); + for (int i = 0; i < nb_planes; i++) { + if (depth == 8) { + if (EXTERNAL_AVX512ICL(cpu_flags)) + s->filter[i] = ff_filter_sobel_avx512icl; + } + } +#endif +} diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index 62d6616faf..59614bc08d 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -46,6 +46,7 @@ AVFILTEROBJS-$(CONFIG_GBLUR_FILTER) += vf_gblur.o AVFILTEROBJS-$(CONFIG_HFLIP_FILTER) += vf_hflip.o AVFILTEROBJS-$(CONFIG_THRESHOLD_FILTER) += vf_threshold.o AVFILTEROBJS-$(CONFIG_NLMEANS_FILTER) += vf_nlmeans.o +AVFILTEROBJS-$(CONFIG_CONVOLUTION_FILTER) += vf_convolution.o CHECKASMOBJS-$(CONFIG_AVFILTER) += $(AVFILTEROBJS-yes) diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index 421bd096c5..ec438c9652 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -197,6 +197,9 @@ static const struct { #if CONFIG_THRESHOLD_FILTER { "vf_threshold", checkasm_check_vf_threshold }, #endif + #if CONFIG_CONVOLUTION_FILTER + { "vf_convolution", checkasm_check_vf_convolution }, + #endif #endif #if CONFIG_SWSCALE { "sw_gbrp", checkasm_check_sw_gbrp }, diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h index ee9151410e..be9f504aed 100644 --- a/tests/checkasm/checkasm.h +++ b/tests/checkasm/checkasm.h @@ -86,6 +86,7 @@ void checkasm_check_vf_eq(void); void checkasm_check_vf_gblur(void); void checkasm_check_vf_hflip(void); void checkasm_check_vf_threshold(void); +void checkasm_check_vf_convolution(void); void checkasm_check_vp8dsp(void); void checkasm_check_vp9dsp(void); void checkasm_check_videodsp(void); diff --git a/tests/checkasm/vf_convolution.c b/tests/checkasm/vf_convolution.c new file mode 100644 index 0000000000..86820b9a1b --- /dev/null +++ b/tests/checkasm/vf_convolution.c @@ -0,0 +1,103 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <string.h> +#include "checkasm.h" +#include "libavfilter/avfilter.h" +#include "libavfilter/convolution.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/mem_internal.h" + +#define WIDTH 512 +#define HEIGHT 512 +#define SRC_STRIDE 512 +#define PIXELS (WIDTH * HEIGHT) + +#define randomize_buffers(buf, size) \ + do { \ + int j; \ + uint8_t *tmp_buf = (uint8_t *)buf;\ + for (j = 0; j< size; j++) \ + tmp_buf[j] = rnd() & 0xFF; \ + } while (0) + +static void check_sobel(const char * report_name) +{ + LOCAL_ALIGNED_32(uint8_t, src, [PIXELS]); + LOCAL_ALIGNED_32(uint8_t, dst_ref, [PIXELS]); + LOCAL_ALIGNED_32(uint8_t, dst_new, [PIXELS]); + const int height = WIDTH; + const int width = HEIGHT; + const int stride = SRC_STRIDE; + const int dstride = SRC_STRIDE; + int mode = 0; + const uint8_t *c[49]; + const int radius = 1; + const int bpc = 1; + const int step = mode == MATRIX_COLUMN ? 16 : 1; + const int slice_start = 0; + const int slice_end = height; + int y; + const int sizew = mode == MATRIX_COLUMN ? height : width; + float scale = 2; + float delta = 10; + + ConvolutionContext s; + + s.scale = scale; + s.delta = delta; + s.depth = 8; + s.nb_planes = 3; + ff_sobel_init(&s, s.depth, s.nb_planes); + + declare_func(void, uint8_t *dst, int width, float scale, float delta, const int *const matrix, + const uint8_t *c[], int peak, int radius, int dstride, int stride, int size); + + memset(dst_ref, 0, PIXELS); + memset(dst_new, 0, PIXELS); + randomize_buffers(src, PIXELS); + + if (check_func(s.filter[0], "%s", report_name)) { + for (y = slice_start; y < slice_end; y += step) { + const int xoff = mode == MATRIX_COLUMN ? (y - slice_start) * bpc : radius * bpc; + const int yoff = mode == MATRIX_COLUMN ? radius * dstride : 0; + + s.setup[0](radius, c, src, stride, radius, width, y, height, bpc); + call_ref(dst_ref + yoff + xoff, sizew - 2 * radius, + scale, delta, NULL, c, 0, radius, + dstride, stride, slice_end - step); + call_new(dst_new + yoff + xoff, sizew - 2 * radius, + scale, delta, NULL, c, 0, radius, + dstride, stride, slice_end - step); + if (memcmp(dst_ref + yoff + xoff, dst_new + yoff + xoff, slice_end - step)) + fail(); + bench_new(dst_new + yoff + xoff, sizew - 2 * radius, + scale, delta, NULL, c, 0, radius, + dstride, stride, slice_end - step); + if (mode != MATRIX_COLUMN) + dst_ref += dstride; + } + } + +} + +void checkasm_check_vf_convolution(void) +{ + check_sobel("sobel"); + report("convolution:sobel"); +} -- 2.27.0

2 1

[PATCH] avcodec/av1: fixed an vaapi decoding corruption issue
by Ruijing Dong 01 Nov '22

01 Nov '22

[problem] When decoding an av1 bitstream, it shows image corruption in the middle of the bitstream around key frames. [analysis] In av1_spec.pdf page 38/669, there is a sentence below: if ( frame_type == KEY_FRAME && show_frame ) { for ( i = 0; i < NUM_REF_FRAMES; i++) { RefValid[ i ] = 0 ...... } ...... } This shows that the condition of invalidating current DPB frames should be the coming frame_type is KEY_FRAME plus show_frame is equal to 1. Otherwise, some of the frames in sequence after KEY_FRAME still refer to the reference frames before KEY_FRAME, and if these before KEY_FRAME reference frames were invalidated, these frames could not find their reference frames, and it could cause image corruption. [solution] Add show_frame flag as another condition to co-determine when to invalidate DPB reference frames. Mesa fix is in https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19386 cc: Fei Wang <fei.w.wang(a)intel.com> Signed-off-by: Ruijing Dong <ruijing.dong(a)amd.com> --- libavcodec/vaapi_av1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/vaapi_av1.c b/libavcodec/vaapi_av1.c index 63374c31c9..d0339b2705 100644 --- a/libavcodec/vaapi_av1.c +++ b/libavcodec/vaapi_av1.c @@ -274,7 +274,7 @@ static int vaapi_av1_start_frame(AVCodecContext *avctx, }; for (int i = 0; i < AV1_NUM_REF_FRAMES; i++) { - if (pic_param.pic_info_fields.bits.frame_type == AV1_FRAME_KEY) + if (pic_param.pic_info_fields.bits.frame_type == AV1_FRAME_KEY && frame_header->show_frame) pic_param.ref_frame_map[i] = VA_INVALID_ID; else pic_param.ref_frame_map[i] = ctx->ref_tab[i].valid ? -- 2.25.1

2 1

[PATCH] avcodec/atrac3plus: reorder channels to match the output layout
by James Almer 01 Nov '22

01 Nov '22

The order in which the channels are coded in the bitstream do not always follow the native, bitmask-based order of channels both signaled by the WAV container and forced by this same decoder. This is the case with layouts containing an LFE channel, as it's always coded last. Fixes ticket #9964. Signed-off-by: James Almer <jamrial(a)gmail.com> --- 6.1 and 7.1 untested, but much like 5.1 they were wrong before this change. libavcodec/atrac3plusdec.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/libavcodec/atrac3plusdec.c b/libavcodec/atrac3plusdec.c index ee71645a3c..8c3d20af76 100644 --- a/libavcodec/atrac3plusdec.c +++ b/libavcodec/atrac3plusdec.c @@ -65,6 +65,7 @@ typedef struct ATRAC3PContext { int num_channel_blocks; ///< number of channel blocks uint8_t channel_blocks[5]; ///< channel configuration descriptor + AVChannelLayout coded_ch_layout; ///< order of channels as coded in the bitstream } ATRAC3PContext; static av_cold int atrac3p_decode_close(AVCodecContext *avctx) @@ -74,6 +75,8 @@ static av_cold int atrac3p_decode_close(AVCodecContext *avctx) av_freep(&ctx->ch_units); av_freep(&ctx->fdsp); + av_channel_layout_uninit(&ctx->coded_ch_layout); + ff_mdct_end(&ctx->mdct_ctx); ff_mdct_end(&ctx->ipqf_dct_ctx); @@ -84,6 +87,7 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, AVCodecContext *avctx) { int channels = avctx->ch_layout.nb_channels; + int ret; memset(ctx->channel_blocks, 0, sizeof(ctx->channel_blocks)); av_channel_layout_uninit(&avctx->ch_layout); @@ -92,17 +96,20 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_MONO; ctx->num_channel_blocks = 1; ctx->channel_blocks[0] = CH_UNIT_MONO; + ctx->coded_ch_layout = avctx->ch_layout; break; case 2: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO; ctx->num_channel_blocks = 1; ctx->channel_blocks[0] = CH_UNIT_STEREO; + ctx->coded_ch_layout = avctx->ch_layout; break; case 3: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_SURROUND; ctx->num_channel_blocks = 2; ctx->channel_blocks[0] = CH_UNIT_STEREO; ctx->channel_blocks[1] = CH_UNIT_MONO; + ctx->coded_ch_layout = avctx->ch_layout; break; case 4: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_4POINT0; @@ -110,6 +117,7 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, ctx->channel_blocks[0] = CH_UNIT_STEREO; ctx->channel_blocks[1] = CH_UNIT_MONO; ctx->channel_blocks[2] = CH_UNIT_MONO; + ctx->coded_ch_layout = avctx->ch_layout; break; case 6: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1_BACK; @@ -118,6 +126,9 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, ctx->channel_blocks[1] = CH_UNIT_MONO; ctx->channel_blocks[2] = CH_UNIT_STEREO; ctx->channel_blocks[3] = CH_UNIT_MONO; + ret = av_channel_layout_from_string(&ctx->coded_ch_layout, "FL+FR+FC+BL+BR+LFE"); + if (ret < 0) + return ret; break; case 7: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_6POINT1_BACK; @@ -127,6 +138,9 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, ctx->channel_blocks[2] = CH_UNIT_STEREO; ctx->channel_blocks[3] = CH_UNIT_MONO; ctx->channel_blocks[4] = CH_UNIT_MONO; + ret = av_channel_layout_from_string(&ctx->coded_ch_layout, "FL+FR+FC+BL+BR+BC+LFE"); + if (ret < 0) + return ret; break; case 8: avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_7POINT1; @@ -136,6 +150,9 @@ static av_cold int set_channel_params(ATRAC3PContext *ctx, ctx->channel_blocks[2] = CH_UNIT_STEREO; ctx->channel_blocks[3] = CH_UNIT_STEREO; ctx->channel_blocks[4] = CH_UNIT_MONO; + ret = av_channel_layout_from_string(&ctx->coded_ch_layout, "FL+FR+FC+BL+BR+SL+SR+LFE"); + if (ret < 0) + return ret; break; default: av_log(avctx, AV_LOG_ERROR, @@ -377,10 +394,12 @@ static int atrac3p_decode_frame(AVCodecContext *avctx, AVFrame *frame, reconstruct_frame(ctx, &ctx->ch_units[ch_block], channels_to_process, avctx); - for (i = 0; i < channels_to_process; i++) - memcpy(samples_p[out_ch_index + i], ctx->outp_buf[i], + for (i = 0; i < channels_to_process; i++) { + enum AVChannel ch = av_channel_layout_channel_from_index(&ctx->coded_ch_layout, out_ch_index + i); + int idx = av_channel_layout_index_from_channel(&frame->ch_layout, ch); + memcpy(samples_p[idx], ctx->outp_buf[i], ATRAC3P_FRAME_SAMPLES * sizeof(**samples_p)); - + } ch_block++; out_ch_index += channels_to_process; } -- 2.38.1

2 7