January 2022 - ffmpeg-devel - lists.ffmpeg.org

[PATCH] avformat/utils: fix logic error in ff_mkdir_p
by Peter Ross 13 Feb '22

13 Feb '22

Fix ticket# 9605 Signed-off-by: Peter Ross <pross(a)xvid.org> --- libavformat/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/utils.c b/libavformat/utils.c index e643821fc9..1164cebe68 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -1135,7 +1135,7 @@ int ff_mkdir_p(const char *path) } } - if ((*(pos - 1) != '/') || (*(pos - 1) != '\\')) { + if ((*(pos - 1) != '/') && (*(pos - 1) != '\\')) { ret = mkdir(temp, 0755); } -- 2.34.1 -- Peter (A907 E02F A6E5 0CD2 34CD 20D2 6760 79C5 AC40 DD6B)

1 1

[PATCH] Fix setsockopt IP_MULTICAST_TTL on OpenBSD
by Brad Smith 12 Feb '22

12 Feb '22

Fix setsockopt() usage on OpenBSD with IP_MULTICAST_TTL. The field type should be an unsigned char on anything but Linux. diff --git a/libavformat/udp.c b/libavformat/udp.c index 180d96a988..29aa865fff 100644 --- a/libavformat/udp.c +++ b/libavformat/udp.c @@ -163,7 +163,13 @@ static int udp_set_multicast_ttl(int sockfd, int mcastTTL, { #ifdef IP_MULTICAST_TTL if (addr->sa_family == AF_INET) { - if (setsockopt(sockfd, IPPROTO_IP, IP_MULTICAST_TTL, &mcastTTL, sizeof(mcastTTL)) < 0) { +#ifdef __linux__ + int ttl = mcastTTL; +#else + unsigned char ttl = mcastTTL; +#endif + + if (setsockopt(sockfd, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) { ff_log_net_error(NULL, AV_LOG_ERROR, "setsockopt(IP_MULTICAST_TTL)"); return ff_neterrno(); }

6 36

[PATCH 1/4] libswscale: Re-factor ff_shuffle_filter_coefficients.
by Alan Kelly 09 Feb '22

09 Feb '22

Make the code more readable, follow the style guide and propagate memory allocation errors. --- libswscale/swscale_internal.h | 2 +- libswscale/utils.c | 68 ++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 3a78d95ba6..26d28d42e6 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -1144,5 +1144,5 @@ void ff_sws_slice_worker(void *priv, int jobnr, int threadnr, #define MAX_LINES_AHEAD 4 //shuffle filter and filterPos for hyScale and hcScale filters in avx2 -void ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW); +int ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW); #endif /* SWSCALE_SWSCALE_INTERNAL_H */ diff --git a/libswscale/utils.c b/libswscale/utils.c index c5ea8853d5..52f07e1661 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -278,39 +278,47 @@ static const FormatEntry format_entries[] = { [AV_PIX_FMT_P416LE] = { 1, 1 }, }; -void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSize, int16_t *filter, int dstW){ +int ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, + int filterSize, int16_t *filter, + int dstW) +{ #if ARCH_X86_64 - int i, j, k, l; + int i = 0, j = 0, k = 0; int cpu_flags = av_get_cpu_flags(); + if (!filter || dstW % 16 != 0) return 0; if (EXTERNAL_AVX2_FAST(cpu_flags) && !(cpu_flags & AV_CPU_FLAG_SLOW_GATHER)) { - if ((c->srcBpc == 8) && (c->dstBpc <= 14)){ - if (dstW % 16 == 0){ - if (filter != NULL){ - for (i = 0; i < dstW; i += 8){ - FFSWAP(int, filterPos[i + 2], filterPos[i+4]); - FFSWAP(int, filterPos[i + 3], filterPos[i+5]); - } - if (filterSize > 4){ - int16_t *tmp2 = av_malloc(dstW * filterSize * 2); - memcpy(tmp2, filter, dstW * filterSize * 2); - for (i = 0; i < dstW; i += 16){//pixel - for (k = 0; k < filterSize / 4; ++k){//fcoeff - for (j = 0; j < 16; ++j){//inner pixel - for (l = 0; l < 4; ++l){//coeff - int from = i * filterSize + j * filterSize + k * 4 + l; - int to = (i) * filterSize + j * 4 + l + k * 64; - filter[to] = tmp2[from]; - } - } - } - } - av_free(tmp2); - } - } - } + if ((c->srcBpc == 8) && (c->dstBpc <= 14)) { + int16_t *filterCopy = NULL; + if (filterSize > 4) { + if (!FF_ALLOC_TYPED_ARRAY(filterCopy, dstW * filterSize)) + return AVERROR(ENOMEM); + memcpy(filterCopy, filter, dstW * filterSize * sizeof(int16_t)); + } + // Do not swap filterPos for pixels which won't be processed by + // the main loop. + for (i = 0; i + 8 <= dstW; i += 8) { + FFSWAP(int, filterPos[i + 2], filterPos[i + 4]); + FFSWAP(int, filterPos[i + 3], filterPos[i + 5]); + } + if (filterSize > 4) { + // 16 pixels are processed at a time. + for (i = 0; i + 16 <= dstW; i += 16) { + // 4 filter coeffs are processed at a time. + for (k = 0; k + 4 <= filterSize; k += 4) { + for (j = 0; j < 16; ++j) { + int from = (i + j) * filterSize + k; + int to = i * filterSize + j * 4 + k * 16; + memcpy(&filter[to], &filterCopy[from], 4 * sizeof(int16_t)); + } + } + } + } + if (filterCopy) + av_free(filterCopy); } } #endif + return 0; } int sws_isSupportedInput(enum AVPixelFormat pix_fmt) @@ -1836,7 +1844,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter, get_local_pos(c, 0, 0, 0), get_local_pos(c, 0, 0, 0))) < 0) goto fail; - ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW); + if ((ret = ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW)) != 0) + goto nomem; if ((ret = initFilter(&c->hChrFilter, &c->hChrFilterPos, &c->hChrFilterSize, c->chrXInc, c->chrSrcW, c->chrDstW, filterAlign, 1 << 14, @@ -1846,7 +1855,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter, get_local_pos(c, c->chrSrcHSubSample, c->src_h_chr_pos, 0), get_local_pos(c, c->chrDstHSubSample, c->dst_h_chr_pos, 0))) < 0) goto fail; - ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW); + if ((ret = ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW)) != 0) + goto nomem; } } // initialize horizontal stuff -- 2.34.1.575.g55b058a8bb-goog

2 6

[PATCH] avformat/hls Implement support for using AVSEEK_FLAG_BACKWARD when seeking
by Gustav Grusell 09 Feb '22

09 Feb '22

Before, seeking in hls streams would always seek to the next keyframe after the given timestamp. With this fix, if AVSEEK_FLAG_BACKWARD is set, seeking will be to the first keyframe of the segment containing the given timestamp. This fixes #6850. --- libavformat/hls.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libavformat/hls.c b/libavformat/hls.c index 8fc6924c90..3f1fd2cb70 100644 --- a/libavformat/hls.c +++ b/libavformat/hls.c @@ -2199,10 +2199,15 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt) tb = get_timebase(pls); ts_diff = av_rescale_rnd(pls->pkt->dts, AV_TIME_BASE, - tb.den, AV_ROUND_DOWN) - - pls->seek_timestamp; - if (ts_diff >= 0 && (pls->seek_flags & AVSEEK_FLAG_ANY || - pls->pkt->flags & AV_PKT_FLAG_KEY)) { + tb.den, AV_ROUND_DOWN) - + pls->seek_timestamp; + /* If AVSEEK_FLAG_BACKWARD set and not AVSEEK_FLAG_ANY, + * we return the first keyframe encountered */ + if ((pls->seek_flags & AVSEEK_FLAG_BACKWARD && + !(pls->seek_flags & AVSEEK_FLAG_ANY) && + pls->pkt->flags & AV_PKT_FLAG_KEY) || + (ts_diff >= 0 && (pls->seek_flags & AVSEEK_FLAG_ANY || + pls->pkt->flags & AV_PKT_FLAG_KEY))) { pls->seek_timestamp = AV_NOPTS_VALUE; break; } -- 2.25.1

7 23

[PATCH v2] avfilter/adelay: Add command support
by David Lacko 08 Feb '22

08 Feb '22

Adds command 'delays' to the adelay filter. This command accepts same values as the option with one difference, to apply delay to all channels prefix 'all:' to the argument is accepted. Signed-off-by: David Lacko <deiwo101(a)gmail.com> --- libavfilter/af_adelay.c | 182 ++++++++++++++++++++++++++++++++++------ 1 file changed, 156 insertions(+), 26 deletions(-) diff --git a/libavfilter/af_adelay.c b/libavfilter/af_adelay.c index ed8a8ae739..382be3dca2 100644 --- a/libavfilter/af_adelay.c +++ b/libavfilter/af_adelay.c @@ -31,6 +31,7 @@ typedef struct ChanDelay { int64_t delay; size_t delay_index; size_t index; + unsigned int samples_size; uint8_t *samples; } ChanDelay; @@ -48,13 +49,14 @@ typedef struct AudioDelayContext { void (*delay_channel)(ChanDelay *d, int nb_samples, const uint8_t *src, uint8_t *dst); + int (*resize_channel_samples)(ChanDelay *d, int64_t new_delay); } AudioDelayContext; #define OFFSET(x) offsetof(AudioDelayContext, x) #define A AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM static const AVOption adelay_options[] = { - { "delays", "set list of delays for each channel", OFFSET(delays), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, A }, + { "delays", "set list of delays for each channel", OFFSET(delays), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, A | AV_OPT_FLAG_RUNTIME_PARAM }, { "all", "use last available delay for remained channels", OFFSET(all), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, A }, { NULL } }; @@ -96,11 +98,93 @@ DELAY(s32, int32_t, 0) DELAY(flt, float, 0) DELAY(dbl, double, 0) +#define CHANGE_DELAY(name, type, fill) \ +static int resize_samples_## name ##p(ChanDelay *d, int64_t new_delay) \ +{ \ + type *samples; \ + \ + if (new_delay == d->delay) { \ + return 0; \ + } \ + \ + if (new_delay == 0) { \ + av_freep(&d->samples); \ + d->samples_size = 0; \ + d->delay = 0; \ + d->index = 0; \ + d->delay_index = 0; \ + return 0; \ + } \ + \ + samples = (type *) av_fast_realloc(d->samples, &d->samples_size, new_delay * sizeof(type)); \ + if (!samples) { \ + return AVERROR(ENOMEM); \ + } \ + \ + if (new_delay < d->delay) { \ + if (d->index > new_delay) { \ + d->index -= new_delay; \ + memmove(samples, &samples[new_delay], d->index * sizeof(type)); \ + d->delay_index = new_delay; \ + } else if (d->delay_index > d->index) { \ + memmove(&samples[d->index], &samples[d->index+(d->delay-new_delay)], \ + (new_delay - d->index) * sizeof(type)); \ + d->delay_index -= d->delay - new_delay; \ + } \ + } else { \ + size_t block_size; \ + if (d->delay_index >= d->delay) { \ + block_size = (d->delay - d->index) * sizeof(type); \ + memmove(&samples[d->index+(new_delay - d->delay)], &samples[d->index], block_size); \ + d->delay_index = new_delay; \ + } else { \ + d->delay_index += new_delay - d->delay; \ + } \ + block_size = (new_delay - d->delay) * sizeof(type); \ + memset(&samples[d->index], fill, block_size); \ + } \ + d->delay = new_delay; \ + d->samples = (void *) samples; \ + return 0; \ +} + +CHANGE_DELAY(u8, uint8_t, 0x80) +CHANGE_DELAY(s16, int16_t, 0) +CHANGE_DELAY(s32, int32_t, 0) +CHANGE_DELAY(flt, float, 0) +CHANGE_DELAY(dbl, double, 0) + +static int parse_delays(char *p, char **saveptr, int64_t *result, AVFilterContext *ctx, int sample_rate) { + float delay, div; + int ret; + char *arg; + char type = 0; + + if (!(arg = av_strtok(p, "|", saveptr))) + return 1; + + ret = av_sscanf(arg, "%"SCNd64"%c", result, &type); + if (ret != 2 || type != 'S') { + div = type == 's' ? 1.0 : 1000.0; + if (av_sscanf(arg, "%f", &delay) != 1) { + av_log(ctx, AV_LOG_ERROR, "Invalid syntax for delay.\n"); + return AVERROR(EINVAL); + } + *result = delay * sample_rate / div; + } + + if (*result < 0) { + av_log(ctx, AV_LOG_ERROR, "Delay must be non negative number.\n"); + return AVERROR(EINVAL); + } + return 0; +} + static int config_input(AVFilterLink *inlink) { AVFilterContext *ctx = inlink->dst; AudioDelayContext *s = ctx->priv; - char *p, *arg, *saveptr = NULL; + char *p, *saveptr = NULL; int i; s->chandelay = av_calloc(inlink->channels, sizeof(*s->chandelay)); @@ -112,29 +196,14 @@ static int config_input(AVFilterLink *inlink) p = s->delays; for (i = 0; i < s->nb_delays; i++) { ChanDelay *d = &s->chandelay[i]; - float delay, div; - char type = 0; int ret; - if (!(arg = av_strtok(p, "|", &saveptr))) + ret = parse_delays(p, &saveptr, &d->delay, ctx, inlink->sample_rate); + if (ret == 1) break; - + else if (ret < 0) + return ret; p = NULL; - - ret = av_sscanf(arg, "%"SCNd64"%c", &d->delay, &type); - if (ret != 2 || type != 'S') { - div = type == 's' ? 1.0 : 1000.0; - if (av_sscanf(arg, "%f", &delay) != 1) { - av_log(ctx, AV_LOG_ERROR, "Invalid syntax for delay.\n"); - return AVERROR(EINVAL); - } - d->delay = delay * inlink->sample_rate / div; - } - - if (d->delay < 0) { - av_log(ctx, AV_LOG_ERROR, "Delay must be non negative number.\n"); - return AVERROR(EINVAL); - } } if (s->all && i) { @@ -171,21 +240,81 @@ static int config_input(AVFilterLink *inlink) d->samples = av_malloc_array(d->delay, s->block_align); if (!d->samples) return AVERROR(ENOMEM); + d->samples_size = d->delay * s->block_align; s->max_delay = FFMAX(s->max_delay, d->delay); } switch (inlink->format) { - case AV_SAMPLE_FMT_U8P : s->delay_channel = delay_channel_u8p ; break; - case AV_SAMPLE_FMT_S16P: s->delay_channel = delay_channel_s16p; break; - case AV_SAMPLE_FMT_S32P: s->delay_channel = delay_channel_s32p; break; - case AV_SAMPLE_FMT_FLTP: s->delay_channel = delay_channel_fltp; break; - case AV_SAMPLE_FMT_DBLP: s->delay_channel = delay_channel_dblp; break; + case AV_SAMPLE_FMT_U8P : s->delay_channel = delay_channel_u8p ; + s->resize_channel_samples = resize_samples_u8p; break; + case AV_SAMPLE_FMT_S16P: s->delay_channel = delay_channel_s16p; + s->resize_channel_samples = resize_samples_s16p; break; + case AV_SAMPLE_FMT_S32P: s->delay_channel = delay_channel_s32p; + s->resize_channel_samples = resize_samples_s32p; break; + case AV_SAMPLE_FMT_FLTP: s->delay_channel = delay_channel_fltp; + s->resize_channel_samples = resize_samples_fltp; break; + case AV_SAMPLE_FMT_DBLP: s->delay_channel = delay_channel_dblp; + s->resize_channel_samples = resize_samples_dblp; break; } return 0; } +static int process_command(AVFilterContext *ctx, const char *cmd, const char *args, + char *res, int res_len, int flags) +{ + int ret = AVERROR(ENOSYS); + AVFilterLink *inlink = ctx->inputs[0]; + AudioDelayContext *s = ctx->priv; + + if (!strcmp(cmd, "delays")) { + int64_t delay; + char *p, *saveptr = NULL; + int64_t all_delay = -1; + int64_t max_delay = 0; + char *args_cpy = av_strdup(args); + if (args_cpy == NULL) { + return AVERROR(ENOMEM); + } + + ret = 0; + p = args_cpy; + + if (!strncmp(args, "all:", 4)) { + p = &args_cpy[4]; + ret = parse_delays(p, &saveptr, &all_delay, ctx, inlink->sample_rate); + if (ret == 1) + ret = AVERROR(EINVAL); + else if (ret == 0) + delay = all_delay; + } + + if (!ret) { + for (int i = 0; i < s->nb_delays; i++) { + ChanDelay *d = &s->chandelay[i]; + + if (all_delay < 0) { + ret = parse_delays(p, &saveptr, &delay, ctx, inlink->sample_rate); + if (ret != 0) { + ret = 0; + break; + } + p = NULL; + } + + ret = s->resize_channel_samples(d, delay); + if (ret) + break; + max_delay = FFMAX(max_delay, d->delay); + } + s->max_delay = FFMAX(s->max_delay, max_delay); + } + av_freep(&args_cpy); + } + return ret; +} + static int filter_frame(AVFilterLink *inlink, AVFrame *frame) { AVFilterContext *ctx = inlink->dst; @@ -330,4 +459,5 @@ const AVFilter ff_af_adelay = { FILTER_SAMPLEFMTS(AV_SAMPLE_FMT_U8P, AV_SAMPLE_FMT_S16P, AV_SAMPLE_FMT_S32P, AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_DBLP), .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL, + .process_command = process_command, }; -- 2.34.1

2 3

[PATCH] libavutil: include assembly with full path from source root
by Alexander Kanavin 08 Feb '22

08 Feb '22

From: Alexander Kanavin <alex.kanavin(a)gmail.com> Otherwise nasm writes the full host-specific paths into .o output, which breaks binary reproducibility. Signed-off-by: Alexander Kanavin <alex.kanavin(a)gmail.com> --- libavutil/x86/cpuid.asm | 2 +- libavutil/x86/emms.asm | 2 +- libavutil/x86/fixed_dsp.asm | 2 +- libavutil/x86/float_dsp.asm | 2 +- libavutil/x86/lls.asm | 2 +- libavutil/x86/pixelutils.asm | 2 +- libavutil/x86/tx_float.asm | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libavutil/x86/cpuid.asm b/libavutil/x86/cpuid.asm index c3f7866ec7..766f77fcdf 100644 --- a/libavutil/x86/cpuid.asm +++ b/libavutil/x86/cpuid.asm @@ -21,7 +21,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION .text diff --git a/libavutil/x86/emms.asm b/libavutil/x86/emms.asm index 8611762d73..df84f2221b 100644 --- a/libavutil/x86/emms.asm +++ b/libavutil/x86/emms.asm @@ -18,7 +18,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION .text diff --git a/libavutil/x86/fixed_dsp.asm b/libavutil/x86/fixed_dsp.asm index 979dd5c334..2f411850f4 100644 --- a/libavutil/x86/fixed_dsp.asm +++ b/libavutil/x86/fixed_dsp.asm @@ -20,7 +20,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION .text diff --git a/libavutil/x86/float_dsp.asm b/libavutil/x86/float_dsp.asm index 517fd63638..b773e61a64 100644 --- a/libavutil/x86/float_dsp.asm +++ b/libavutil/x86/float_dsp.asm @@ -20,7 +20,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION_RODATA 32 pd_reverse: dd 7, 6, 5, 4, 3, 2, 1, 0 diff --git a/libavutil/x86/lls.asm b/libavutil/x86/lls.asm index 317fba6fca..d2526d1ff4 100644 --- a/libavutil/x86/lls.asm +++ b/libavutil/x86/lls.asm @@ -20,7 +20,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION .text diff --git a/libavutil/x86/pixelutils.asm b/libavutil/x86/pixelutils.asm index 36c57c5f7f..8b45ead78b 100644 --- a/libavutil/x86/pixelutils.asm +++ b/libavutil/x86/pixelutils.asm @@ -21,7 +21,7 @@ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" SECTION .text diff --git a/libavutil/x86/tx_float.asm b/libavutil/x86/tx_float.asm index 4d2283fae1..ea39f2172a 100644 --- a/libavutil/x86/tx_float.asm +++ b/libavutil/x86/tx_float.asm @@ -29,7 +29,7 @@ ; replace some shuffles with vblends? ; avx512 split-radix -%include "x86util.asm" +%include "libavutil/x86/x86util.asm" %if ARCH_X86_64 %define ptr resq -- 2.20.1

3 8

[PATCH 1/2] lavc/aarch64: add hevc qpel assembly
by J. Dekker 07 Feb '22

07 Feb '22

Based on patch by: Rafal Dabrowa <fatwildcat(a)gmail.com> --- libavcodec/aarch64/Makefile | 1 + libavcodec/aarch64/hevcdsp_init_aarch64.c | 69 + libavcodec/aarch64/hevcdsp_qpel_neon.S | 2799 +++++++++++++++++++++ 3 files changed, 2869 insertions(+) Some changes since last time it was submitted, namely: split, macro'd and some scheduling and other improvements. create mode 100644 libavcodec/aarch64/hevcdsp_qpel_neon.S diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile index 954461f81d..8592692479 100644 --- a/libavcodec/aarch64/Makefile +++ b/libavcodec/aarch64/Makefile @@ -63,4 +63,5 @@ NEON-OBJS-$(CONFIG_VP9_DECODER) += aarch64/vp9itxfm_16bpp_neon.o \ aarch64/vp9mc_neon.o NEON-OBJS-$(CONFIG_HEVC_DECODER) += aarch64/hevcdsp_idct_neon.o \ aarch64/hevcdsp_init_aarch64.o \ + aarch64/hevcdsp_qpel_neon.o \ aarch64/hevcdsp_sao_neon.o diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c b/libavcodec/aarch64/hevcdsp_init_aarch64.c index 1e40be740c..69f0d9bc6f 100644 --- a/libavcodec/aarch64/hevcdsp_init_aarch64.c +++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c @@ -58,7 +58,63 @@ void ff_hevc_sao_band_filter_8x8_8_neon(uint8_t *_dst, uint8_t *_src, int16_t *sao_offset_val, int sao_left_class, int width, int height); +#define NEON8_FNPROTO(fn, args) \ + void ff_hevc_put_hevc_##fn##4_8_neon args; \ + void ff_hevc_put_hevc_##fn##6_8_neon args; \ + void ff_hevc_put_hevc_##fn##8_8_neon args; \ + void ff_hevc_put_hevc_##fn##12_8_neon args; \ + void ff_hevc_put_hevc_##fn##16_8_neon args; \ + void ff_hevc_put_hevc_##fn##24_8_neon args; \ + void ff_hevc_put_hevc_##fn##32_8_neon args; \ + void ff_hevc_put_hevc_##fn##48_8_neon args; \ + void ff_hevc_put_hevc_##fn##64_8_neon args; \ +NEON8_FNPROTO(qpel_h, (int16_t *dst, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_v, (int16_t *dst, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_hv, (int16_t *dst, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_uni_h, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_uni_v, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_uni_hv, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_bi_h, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, int16_t *src2, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_bi_v, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, int16_t *src2, + int height, intptr_t mx, intptr_t my, int width)); + +NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride, + uint8_t *src, ptrdiff_t srcstride, int16_t *src2, + int height, intptr_t mx, intptr_t my, int width)); + +#define NEON8_FNASSIGN(member, v, h, fn) \ + member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon; \ + member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon; \ + member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon; \ + member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon; \ + member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon; \ + member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon; \ + member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon; \ + member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon; \ + member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon; av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) { @@ -80,6 +136,19 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) // for the current size, but if enabled for bigger sizes, the cases // of non-multiple of 8 seem to arise. // c->sao_band_filter[0] = ff_hevc_sao_band_filter_8x8_8_neon; + + NEON8_FNASSIGN(c->put_hevc_qpel, 0, 0, pel_pixels); + NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h); + NEON8_FNASSIGN(c->put_hevc_qpel, 1, 0, qpel_v); + NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv); + NEON8_FNASSIGN(c->put_hevc_qpel_uni, 0, 1, qpel_uni_h); + NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 0, qpel_uni_v); + NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 1, qpel_uni_hv); + NEON8_FNASSIGN(c->put_hevc_qpel_bi, 0, 0, pel_bi_pixels); + NEON8_FNASSIGN(c->put_hevc_qpel_bi, 0, 1, qpel_bi_h); + NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 0, qpel_bi_v); + NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 1, qpel_bi_hv); + } if (bit_depth == 10) { c->add_residual[0] = ff_hevc_add_residual_4x4_10_neon; diff --git a/libavcodec/aarch64/hevcdsp_qpel_neon.S b/libavcodec/aarch64/hevcdsp_qpel_neon.S new file mode 100644 index 0000000000..e8cc6f5f25 --- /dev/null +++ b/libavcodec/aarch64/hevcdsp_qpel_neon.S @@ -0,0 +1,2799 @@ +/* -*-arm64-*- + * vim: syntax=arm64asm + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/aarch64/asm.S" +#define MAX_PB_SIZE 64 + +.Lqpel_filters: + .byte 0, 0, 0, 0, 0, 0, 0, 0 + .byte -1, 4,-10, 58, 17, -5, 1, 0 + .byte -1, 4,-11, 40, 40,-11, 4, -1 + .byte 0, 1, -5, 17, 58,-10, 4, -1 + +.macro load_qpel_filterb freg, xreg + adr \xreg, .Lqpel_filters + add \xreg, \xreg, \freg, lsl #3 + ld4r {v0.16b, v1.16b, v2.16b, v3.16b}, [\xreg], #4 + ld4r {v4.16b, v5.16b, v6.16b, v7.16b}, [\xreg] + neg v0.16b, v0.16b + neg v2.16b, v2.16b + neg v5.16b, v5.16b + neg v7.16b, v7.16b +.endm + +.macro calc_qpelb dst, src0, src1, src2, src3, src4, src5, src6, src7 + umlsl \dst\().8h, \src0\().8b, v0.8b + umlal \dst\().8h, \src1\().8b, v1.8b + umlsl \dst\().8h, \src2\().8b, v2.8b + umlal \dst\().8h, \src3\().8b, v3.8b + umlal \dst\().8h, \src4\().8b, v4.8b + umlsl \dst\().8h, \src5\().8b, v5.8b + umlal \dst\().8h, \src6\().8b, v6.8b + umlsl \dst\().8h, \src7\().8b, v7.8b +.endm + +.macro calc_qpelb2 dst, src0, src1, src2, src3, src4, src5, src6, src7 + umlsl2 \dst\().8h, \src0\().16b, v0.16b + umlal2 \dst\().8h, \src1\().16b, v1.16b + umlsl2 \dst\().8h, \src2\().16b, v2.16b + umlal2 \dst\().8h, \src3\().16b, v3.16b + umlal2 \dst\().8h, \src4\().16b, v4.16b + umlsl2 \dst\().8h, \src5\().16b, v5.16b + umlal2 \dst\().8h, \src6\().16b, v6.16b + umlsl2 \dst\().8h, \src7\().16b, v7.16b +.endm + +.macro load_qpel_filterh freg, xreg + adr \xreg, .Lqpel_filters + add \xreg, \xreg, \freg, lsl #3 + ld1 {v0.8b}, [\xreg] + sxtl v0.8h, v0.8b +.endm + +.macro calc_qpelh dst, src0, src1, src2, src3, src4, src5, src6, src7, op, shift=6 + smull \dst\().4s, \src0\().4h, v0.h[0] + smlal \dst\().4s, \src1\().4h, v0.h[1] + smlal \dst\().4s, \src2\().4h, v0.h[2] + smlal \dst\().4s, \src3\().4h, v0.h[3] + smlal \dst\().4s, \src4\().4h, v0.h[4] + smlal \dst\().4s, \src5\().4h, v0.h[5] + smlal \dst\().4s, \src6\().4h, v0.h[6] + smlal \dst\().4s, \src7\().4h, v0.h[7] +.ifc \op, sshr + sshr \dst\().4s, \dst\().4s, \shift +.else + \op \dst\().4h, \dst\().4s, \shift +.endif +.endm + +.macro calc_qpelh2 dst, dstt, src0, src1, src2, src3, src4, src5, src6, src7, op, shift=6 + smull2 \dstt\().4s, \src0\().8h, v0.h[0] + smlal2 \dstt\().4s, \src1\().8h, v0.h[1] + smlal2 \dstt\().4s, \src2\().8h, v0.h[2] + smlal2 \dstt\().4s, \src3\().8h, v0.h[3] + smlal2 \dstt\().4s, \src4\().8h, v0.h[4] + smlal2 \dstt\().4s, \src5\().8h, v0.h[5] + smlal2 \dstt\().4s, \src6\().8h, v0.h[6] + smlal2 \dstt\().4s, \src7\().8h, v0.h[7] +.ifc \op, sshr + sshr \dst\().4s, \dstt\().4s, \shift +.else + \op \dst\().8h, \dstt\().4s, \shift +.endif +.endm + +function ff_hevc_put_hevc_qpel_h4_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #8 + mov x14, #(MAX_PB_SIZE * 2) +1: ld1 {v16.8b}, [x1], #8 + ld1 {v17.s}[0], [x1], x2 +.macro calc src0, src1, idx + ushr \src0\().2d, \src1\().2d, #8 + mov \src0\().b[7], v17.b[\idx] +.endm + calc v18, v16, 0 + calc v19, v18, 1 + calc v20, v19, 2 + ushr v21.2d, v20.2d, #8 + ushr v22.2d, v21.2d, #8 + ushr v23.2d, v22.2d, #8 + ushr v24.2d, v23.2d, #8 + movi v28.8h, #0 + calc_qpelb v28, v16, v18, v19, v20, v21, v22, v23, v24 + subs w3, w3, #1 + st1 {v28.4h}, [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h6_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + mov x14, #(MAX_PB_SIZE * 2 - 8) +1: ld1 {v16.8b, v17.8b}, [x1], x2 + // same macro + calc v18, v16, 0 + calc v19, v18, 1 + calc v20, v19, 2 + calc v21, v20, 3 + calc v22, v21, 4 + ushr v23.2d, v22.2d, #8 + ushr v24.2d, v23.2d, #8 + movi v28.8h, #0 + calc_qpelb v28, v16, v18, v19, v20, v21, v22, v23, v24 + st1 {v28.4h}, [x0], #8 + subs w3, w3, #1 + st1 {v28.s}[2], [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h8_8_neon, export=1 + sxtw x4, w4 + sxtw x7, w7 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + mov x14, #(MAX_PB_SIZE * 2) +1: ld1 {v16.8b, v17.8b}, [x1], x2 + // same macro + calc v18, v16, 0 + calc v19, v18, 1 + calc v20, v19, 2 + calc v21, v20, 3 + calc v22, v21, 4 + calc v23, v22, 5 + calc v24, v23, 6 +.purgem calc + movi v28.8h, #0 + calc_qpelb v28, v16, v18, v19, v20, v21, v22, v23, v24 + subs w3, w3, #1 + st1 {v28.8h}, [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h12_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #16 + mov x14, #(MAX_PB_SIZE * 2 - 16) +1: ld2 {v16.8b, v17.8b}, [x1], #16 + ld1 {v27.s}[0], [x1], x2 + ushr v18.2d, v16.2d, #8 + ushr v19.2d, v17.2d, #8 + mov v18.b[7], v27.b[0] + mov v19.b[7], v27.b[1] + ushr v20.2d, v18.2d, #8 + ushr v21.2d, v19.2d, #8 + mov v20.b[7], v27.b[2] + mov v21.b[7], v27.b[3] + ushr v22.2d, v20.2d, #8 + ushr v23.2d, v21.2d, #8 + ushr v24.2d, v22.2d, #8 + movi v28.8h, #0 + movi v29.8h, #0 + calc_qpelb v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb v29, v17, v18, v19, v20, v21, v22, v23, v24 + zip1 v16.8h, v28.8h, v29.8h + zip2 v17.8h, v28.8h, v29.8h + st1 {v16.8h}, [x0], #16 + subs w3, w3, #1 + st1 {v17.4h}, [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h16_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #16 + mov x14, #(MAX_PB_SIZE * 2) +1: ld2 {v16.8b, v17.8b}, [x1], #16 + ld1 {v27.8b}, [x1], x2 + ushr v18.2d, v16.2d, #8 + ushr v19.2d, v17.2d, #8 + mov v18.b[7], v27.b[0] + mov v19.b[7], v27.b[1] + ushr v20.2d, v18.2d, #8 + ushr v21.2d, v19.2d, #8 + mov v20.b[7], v27.b[2] + mov v21.b[7], v27.b[3] + ushr v22.2d, v20.2d, #8 + ushr v23.2d, v21.2d, #8 + mov v22.b[7], v27.b[4] + mov v23.b[7], v27.b[5] + ushr v24.2d, v22.2d, #8 + mov v24.b[7], v27.b[6] + movi v28.8h, #0 + movi v29.8h, #0 + calc_qpelb v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb v29, v17, v18, v19, v20, v21, v22, v23, v24 + subs w3, w3, #1 + st2 {v28.8h, v29.8h}, [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h24_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #24 + mov x14, #(MAX_PB_SIZE * 2) +1: ld3 {v16.8b, v17.8b, v18.8b}, [x1], #24 + ld1 {v27.8b}, [x1], x2 + ushr v19.2d, v16.2d, #8 + ushr v20.2d, v17.2d, #8 + ushr v21.2d, v18.2d, #8 + mov v19.b[7], v27.b[0] + mov v20.b[7], v27.b[1] + mov v21.b[7], v27.b[2] + ushr v22.2d, v19.2d, #8 + ushr v23.2d, v20.2d, #8 + ushr v24.2d, v21.2d, #8 + mov v22.b[7], v27.b[3] + mov v23.b[7], v27.b[4] + mov v24.b[7], v27.b[5] + ushr v25.2d, v22.2d, #8 + mov v25.b[7], v27.b[6] + movi v28.8h, #0 + movi v29.8h, #0 + movi v30.8h, #0 + calc_qpelb v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb v29, v17, v18, v19, v20, v21, v22, v23, v24 + calc_qpelb v30, v18, v19, v20, v21, v22, v23, v24, v25 + subs w3, w3, #1 + st3 {v28.8h, v29.8h, v30.8h}, [x0], x14 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h32_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #32 + mov x14, #(MAX_PB_SIZE * 2) +1: ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [x1], #32 + movi v28.8h, #0 + movi v29.8h, #0 + ld1 {v27.8b}, [x1], x2 + movi v30.8h, #0 + movi v31.8h, #0 + ushr v20.2d, v16.2d, #8 + ushr v21.2d, v17.2d, #8 + ushr v22.2d, v18.2d, #8 + ushr v23.2d, v19.2d, #8 + mov v20.b[7], v27.b[0] + mov v21.b[7], v27.b[1] + mov v22.b[7], v27.b[2] + mov v23.b[7], v27.b[3] + ushr v24.2d, v20.2d, #8 + ushr v25.2d, v21.2d, #8 + ushr v26.2d, v22.2d, #8 + mov v24.b[7], v27.b[4] + mov v25.b[7], v27.b[5] + mov v26.b[7], v27.b[6] + calc_qpelb v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb v29, v17, v18, v19, v20, v21, v22, v23, v24 + calc_qpelb v30, v18, v19, v20, v21, v22, v23, v24, v25 + calc_qpelb v31, v19, v20, v21, v22, v23, v24, v25, v26 + st4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x0], x14 + subs w3, w3, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h48_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #48 + mov x7, #24 + mov x14, #80 +1: ld3 {v16.16b, v17.16b, v18.16b}, [x1], x7 + movi v28.8h, #0 + ld1 {v26.8b}, [x1], x7 + movi v29.8h, #0 + ld1 {v27.8b}, [x1], x2 + movi v30.8h, #0 + ushr v19.2d, v16.2d, #8 + ushr v20.2d, v17.2d, #8 + ushr v21.2d, v18.2d, #8 + mov v19.b[7], v26.b[0] + mov v19.b[15], v27.b[0] + mov v20.b[7], v26.b[1] + mov v20.b[15], v27.b[1] + mov v21.b[7], v26.b[2] + mov v21.b[15], v27.b[2] + ushr v22.2d, v19.2d, #8 + ushr v23.2d, v20.2d, #8 + ushr v24.2d, v21.2d, #8 + mov v22.b[7], v26.b[3] + mov v22.b[15], v27.b[3] + mov v23.b[7], v26.b[4] + mov v23.b[15], v27.b[4] + mov v24.b[7], v26.b[5] + mov v24.b[15], v27.b[5] + ushr v25.2d, v22.2d, #8 + mov v25.b[7], v26.b[6] + mov v25.b[15], v27.b[6] + calc_qpelb v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb v29, v17, v18, v19, v20, v21, v22, v23, v24 + calc_qpelb v30, v18, v19, v20, v21, v22, v23, v24, v25 + st3 {v28.8h, v29.8h, v30.8h}, [x0], #48 + movi v28.8h, #0 + movi v29.8h, #0 + movi v30.8h, #0 + calc_qpelb2 v28, v16, v17, v18, v19, v20, v21, v22, v23 + calc_qpelb2 v29, v17, v18, v19, v20, v21, v22, v23, v24 + calc_qpelb2 v30, v18, v19, v20, v21, v22, v23, v24, v25 + st3 {v28.8h, v29.8h, v30.8h}, [x0], x14 + subs w3, w3, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_h64_8_neon, export=1 + load_qpel_filterb x4, x5 + sub x1, x1, #3 + sub x2, x2, #64 + mov x7, #32 +1: ld4 {v16.16b, v17.16b, v18.16b, v19.16b}, [x1], x7 + ld1 {v27.8b}, [x1], x7 + ld1 {v28.8b}, [x1], x2 + ushr v20.2d, v16.2d, #8 + ushr v21.2d, v17.2d, #8 + ushr v22.2d, v18.2d, #8 + ushr v23.2d, v19.2d, #8 + mov v20.b[7], v27.b[0] + mov v21.b[7], v27.b[1] + mov v22.b[7], v27.b[2] + mov v23.b[7], v27.b[3] + mov v20.b[15], v28.b[0] + mov v21.b[15], v28.b[1] + mov v22.b[15], v28.b[2] + mov v23.b[15], v28.b[3] + ushr v24.2d, v20.2d, #8 + ushr v25.2d, v21.2d, #8 + ushr v26.2d, v22.2d, #8 + mov v24.b[7], v27.b[4] + mov v25.b[7], v27.b[5] + mov v26.b[7], v27.b[6] + mov v24.b[15], v28.b[4] + mov v25.b[15], v28.b[5] + mov v26.b[15], v28.b[6] +.macro calc fn + movi v28.8h, #0 + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 + \fn v28, v16, v17, v18, v19, v20, v21, v22, v23 + \fn v29, v17, v18, v19, v20, v21, v22, v23, v24 + \fn v30, v18, v19, v20, v21, v22, v23, v24, v25 + \fn v31, v19, v20, v21, v22, v23, v24, v25, v26 + st4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x0], #64 +.endm + calc calc_qpelb + calc calc_qpelb2 +.purgem calc + subs w3, w3, #1 + b.ne 1b + ret +endfunc + +.macro calc_all + calc v23, v16, v17, v18, v19, v20, v21, v22, v23 + b.eq 2f + calc v16, v17, v18, v19, v20, v21, v22, v23, v16 + b.eq 2f + calc v17, v18, v19, v20, v21, v22, v23, v16, v17 + b.eq 2f + calc v18, v19, v20, v21, v22, v23, v16, v17, v18 + b.eq 2f + calc v19, v20, v21, v22, v23, v16, v17, v18, v19 + b.eq 2f + calc v20, v21, v22, v23, v16, v17, v18, v19, v20 + b.eq 2f + calc v21, v22, v23, v16, v17, v18, v19, v20, v21 + b.eq 2f + calc v22, v23, v16, v17, v18, v19, v20, v21, v22 + b.hi 1b +.endm + +.macro calc_all2 + calc v30, v31, v16, v18, v20, v22, v24, v26, v28, v30, v17, v19, v21, v23, v25, v27, v29, v31 + b.eq 2f + calc v16, v17, v18, v20, v22, v24, v26, v28, v30, v16, v19, v21, v23, v25, v27, v29, v31, v17 + b.eq 2f + calc v18, v19, v20, v22, v24, v26, v28, v30, v16, v18, v21, v23, v25, v27, v29, v31, v17, v19 + b.eq 2f + calc v20, v21, v22, v24, v26, v28, v30, v16, v18, v20, v23, v25, v27, v29, v31, v17, v19, v21 + b.eq 2f + calc v22, v23, v24, v26, v28, v30, v16, v18, v20, v22, v25, v27, v29, v31, v17, v19, v21, v23 + b.eq 2f + calc v24, v25, v26, v28, v30, v16, v18, v20, v22, v24, v27, v29, v31, v17, v19, v21, v23, v25 + b.eq 2f + calc v26, v27, v28, v30, v16, v18, v20, v22, v24, v26, v29, v31, v17, v19, v21, v23, v25, v27 + b.eq 2f + calc v28, v29, v30, v16, v18, v20, v22, v24, v26, v28, v31, v17, v19, v21, v23, v25, v27, v29 + b.hi 1b +.endm + +function ff_hevc_put_hevc_qpel_v4_8_neon, export=1 + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2) + sub x1, x1, x2 + ld1 {v16.s}[0], [x1], x2 + ld1 {v17.s}[0], [x1], x2 + ld1 {v18.s}[0], [x1], x2 + ld1 {v19.s}[0], [x1], x2 + ld1 {v20.s}[0], [x1], x2 + ld1 {v21.s}[0], [x1], x2 + ld1 {v22.s}[0], [x1], x2 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().s}[0], [x1], x2 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + st1 {v24.4h}, [x0], x9 + subs w3, w3, #1 + b.eq 2f +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_v6_8_neon, export=1 + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2 - 8) + sub x1, x1, x2 + ld1 {v16.8b}, [x1], x2 + ld1 {v17.8b}, [x1], x2 + ld1 {v18.8b}, [x1], x2 + ld1 {v19.8b}, [x1], x2 + ld1 {v20.8b}, [x1], x2 + ld1 {v21.8b}, [x1], x2 + ld1 {v22.8b}, [x1], x2 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x1], x2 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + st1 {v24.4h}, [x0], #8 + st1 {v24.s}[2], [x0], x9 + subs w3, w3, #1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_v8_8_neon, export=1 + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2) + sub x1, x1, x2 + ld1 {v16.8b}, [x1], x2 + ld1 {v17.8b}, [x1], x2 + ld1 {v18.8b}, [x1], x2 + ld1 {v19.8b}, [x1], x2 + ld1 {v20.8b}, [x1], x2 + ld1 {v21.8b}, [x1], x2 + ld1 {v22.8b}, [x1], x2 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x1], x2 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + st1 {v24.8h}, [x0], x9 + subs w3, w3, #1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_v12_8_neon, export=1 + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2 - 16) + sub x1, x1, x2 + ld1 {v16.16b}, [x1], x2 + ld1 {v17.16b}, [x1], x2 + ld1 {v18.16b}, [x1], x2 + ld1 {v19.16b}, [x1], x2 + ld1 {v20.16b}, [x1], x2 + ld1 {v21.16b}, [x1], x2 + ld1 {v22.16b}, [x1], x2 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x1], x2 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + st1 {v24.8h}, [x0], #16 + subs w3, w3, #1 + st1 {v25.4h}, [x0], x9 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_v16_8_neon, export=1 + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2) + sub x1, x1, x2 + ld1 {v16.16b}, [x1], x2 + ld1 {v17.16b}, [x1], x2 + ld1 {v18.16b}, [x1], x2 + ld1 {v19.16b}, [x1], x2 + ld1 {v20.16b}, [x1], x2 + ld1 {v21.16b}, [x1], x2 + ld1 {v22.16b}, [x1], x2 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x1], x2 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + subs w3, w3, #1 + st1 {v24.8h, v25.8h}, [x0], x9 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +// todo: reads #32 bytes +function ff_hevc_put_hevc_qpel_v24_8_neon, export=1 + sub sp, sp, #48 + st1 {v8.16b, v9.16b, v10.16b}, [sp] + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + sub x1, x1, x2 + mov x9, #(MAX_PB_SIZE * 2) + ld1 {v16.16b, v17.16b}, [x1], x2 + ld1 {v18.16b, v19.16b}, [x1], x2 + ld1 {v20.16b, v21.16b}, [x1], x2 + ld1 {v22.16b, v23.16b}, [x1], x2 + ld1 {v24.16b, v25.16b}, [x1], x2 + ld1 {v26.16b, v27.16b}, [x1], x2 + ld1 {v28.16b, v29.16b}, [x1], x2 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().16b, \tmp1\().16b}, [x1], x2 + movi v8.8h, #0 + movi v9.8h, #0 + movi v10.8h, #0 + calc_qpelb v8, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v9, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb v10, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + subs w3, w3, #1 + st1 {v8.8h, v9.8h, v10.8h}, [x0], x9 +.endm +1: calc_all2 +.purgem calc +2: ld1 {v8.16b, v9.16b, v10.16b}, [sp], #48 + ret +endfunc + +function ff_hevc_put_hevc_qpel_v32_8_neon, export=1 + sub sp, sp, #64 + st1 {v8.16b-v11.16b}, [sp] + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + mov x9, #(MAX_PB_SIZE * 2) + sub x1, x1, x2 + ld1 {v16.16b, v17.16b}, [x1], x2 + ld1 {v18.16b, v19.16b}, [x1], x2 + ld1 {v20.16b, v21.16b}, [x1], x2 + ld1 {v22.16b, v23.16b}, [x1], x2 + ld1 {v24.16b, v25.16b}, [x1], x2 + ld1 {v26.16b, v27.16b}, [x1], x2 + ld1 {v28.16b, v29.16b}, [x1], x2 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().16b, \tmp1\().16b}, [x1], x2 + movi v8.8h, #0 + movi v9.8h, #0 + movi v10.8h, #0 + movi v11.8h, #0 + calc_qpelb v8, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v9, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb v10, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + calc_qpelb2 v11, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + subs w3, w3, #1 + st1 {v8.8h-v11.8h}, [x0], x9 +.endm +1: calc_all2 +.purgem calc +2: ld1 {v8.16b-v11.16b}, [sp], #64 + ret +endfunc + +function ff_hevc_put_hevc_qpel_v48_8_neon, export=1 + stp x5, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_v24_8_neon) + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + ldr x5, [sp] + add x0, x0, #48 + add x1, x1, #24 + bl X(ff_hevc_put_hevc_qpel_v24_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_v64_8_neon, export=1 + sub sp, sp, #64 + st1 {v8.16b-v11.16b}, [sp] + load_qpel_filterb x5, x4 + sub x1, x1, x2, lsl #1 + sub x1, x1, x2 + mov x9, #(MAX_PB_SIZE * 2) +0: mov x8, x1 // src + ld1 {v16.16b, v17.16b}, [x8], x2 + mov w11, w3 // height + ld1 {v18.16b, v19.16b}, [x8], x2 + mov x10, x0 // dst + ld1 {v20.16b, v21.16b}, [x8], x2 + ld1 {v22.16b, v23.16b}, [x8], x2 + ld1 {v24.16b, v25.16b}, [x8], x2 + ld1 {v26.16b, v27.16b}, [x8], x2 + ld1 {v28.16b, v29.16b}, [x8], x2 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().16b, \tmp1\().16b}, [x8], x2 + movi v8.8h, #0 + movi v9.8h, #0 + movi v10.8h, #0 + movi v11.8h, #0 + calc_qpelb v8, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v9, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb v10, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + calc_qpelb2 v11, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + subs x11, x11, #1 + st1 {v8.8h-v11.8h}, [x10], x9 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #64 + add x1, x1, #32 + subs w6, w6, #32 + b.hi 0b + ld1 {v8.16b-v11.16b}, [sp], #64 + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv4_8_neon, export=1 + add w10, w3, #7 + mov x7, #128 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + sub x1, x1, x2, lsl #1 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h4_8_neon) + ldp x5, x30, [sp], #16 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + ld1 {v16.4h}, [sp], x7 + ld1 {v17.4h}, [sp], x7 + ld1 {v18.4h}, [sp], x7 + ld1 {v19.4h}, [sp], x7 + ld1 {v20.4h}, [sp], x7 + ld1 {v21.4h}, [sp], x7 + ld1 {v22.4h}, [sp], x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().4h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + subs w3, w3, #1 + st1 {v1.4h}, [x0], x7 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv6_8_neon, export=1 + add w10, w3, #7 + mov x7, #128 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + sub x1, x1, x2, lsl #1 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h6_8_neon) + ldp x5, x30, [sp], #16 + mov x8, #120 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + ld1 {v16.8h}, [sp], x7 + ld1 {v17.8h}, [sp], x7 + ld1 {v18.8h}, [sp], x7 + ld1 {v19.8h}, [sp], x7 + ld1 {v20.8h}, [sp], x7 + ld1 {v21.8h}, [sp], x7 + ld1 {v22.8h}, [sp], x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + st1 {v1.4h}, [x0], #8 + subs w3, w3, #1 + st1 {v1.s}[2], [x0], x8 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv8_8_neon, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h8_8_neon) + ldp x5, x30, [sp], #16 + mov x7, #128 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + ld1 {v16.8h}, [sp], x7 + ld1 {v17.8h}, [sp], x7 + ld1 {v18.8h}, [sp], x7 + ld1 {v19.8h}, [sp], x7 + ld1 {v20.8h}, [sp], x7 + ld1 {v21.8h}, [sp], x7 + ld1 {v22.8h}, [sp], x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h}, [x0], x7 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv12_8_neon, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h12_8_neon) + ldp x5, x30, [sp], #16 + mov x7, #128 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + mov x8, #112 + ld1 {v16.8h, v17.8h}, [sp], x7 + ld1 {v18.8h, v19.8h}, [sp], x7 + ld1 {v20.8h, v21.8h}, [sp], x7 + ld1 {v22.8h, v23.8h}, [sp], x7 + ld1 {v24.8h, v25.8h}, [sp], x7 + ld1 {v26.8h, v27.8h}, [sp], x7 + ld1 {v28.8h, v29.8h}, [sp], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + st1 {v1.8h}, [x0], #16 + subs w3, w3, #1 + st1 {v2.4h}, [x0], x8 +.endm +1: calc_all2 +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv16_8_neon, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x3, x3, #7 + add x0, sp, #32 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h16_8_neon) + ldp x5, x30, [sp], #16 + mov x7, #128 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + ld1 {v16.8h, v17.8h}, [sp], x7 + ld1 {v18.8h, v19.8h}, [sp], x7 + ld1 {v20.8h, v21.8h}, [sp], x7 + ld1 {v22.8h, v23.8h}, [sp], x7 + ld1 {v24.8h, v25.8h}, [sp], x7 + ld1 {v26.8h, v27.8h}, [sp], x7 + ld1 {v28.8h, v29.8h}, [sp], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + calc_qpelh2 v2, v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h, v2.8h}, [x0], x7 +.endm +1: calc_all2 +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv24_8_neon, export=1 + sub sp, sp, #64 + st1 {v8.16b-v11.16b}, [sp] + sub x1, x1, x2, lsl #1 + sub sp, sp, #64 + add w10, w3, #7 + st1 {v12.16b-v15.16b}, [sp] + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h24_8_neon) + ldp x5, x30, [sp], #16 + mov x7, #128 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 + ld1 {v8.8h-v10.8h}, [sp], x7 + ld1 {v11.8h-v13.8h}, [sp], x7 + ld1 {v14.8h-v16.8h}, [sp], x7 + ld1 {v17.8h-v19.8h}, [sp], x7 + ld1 {v20.8h-v22.8h}, [sp], x7 + ld1 {v23.8h-v25.8h}, [sp], x7 + ld1 {v26.8h-v28.8h}, [sp], x7 +1: ld1 {v29.8h-v31.8h}, [sp], x7 + calc_qpelh v1, v8, v11, v14, v17, v20, v23, v26, v29, sqshrn + calc_qpelh2 v1, v2, v8, v11, v14, v17, v20, v23, v26, v29, sqshrn2 + calc_qpelh v2, v9, v12, v15, v18, v21, v24, v27, v30, sqshrn + calc_qpelh2 v2, v3, v9, v12, v15, v18, v21, v24, v27, v30, sqshrn2 + calc_qpelh v3, v10, v13, v16, v19, v22, v25, v28, v31, sqshrn + calc_qpelh2 v3, v4, v10, v13, v16, v19, v22, v25, v28, v31, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v8.8h-v10.8h}, [sp], x7 + calc_qpelh v1, v11, v14, v17, v20, v23, v26, v29, v8, sqshrn + calc_qpelh2 v1, v2, v11, v14, v17, v20, v23, v26, v29, v8, sqshrn2 + calc_qpelh v2, v12, v15, v18, v21, v24, v27, v30, v9, sqshrn + calc_qpelh2 v2, v3, v12, v15, v18, v21, v24, v27, v30, v9, sqshrn2 + calc_qpelh v3, v13, v16, v19, v22, v25, v28, v31, v10, sqshrn + calc_qpelh2 v3, v4, v13, v16, v19, v22, v25, v28, v31, v10, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v11.8h-v13.8h}, [sp], x7 + calc_qpelh v1, v14, v17, v20, v23, v26, v29, v8, v11, sqshrn + calc_qpelh2 v1, v2, v14, v17, v20, v23, v26, v29, v8, v11, sqshrn2 + calc_qpelh v2, v15, v18, v21, v24, v27, v30, v9, v12, sqshrn + calc_qpelh2 v2, v3, v15, v18, v21, v24, v27, v30, v9, v12, sqshrn2 + calc_qpelh v3, v16, v19, v22, v25, v28, v31, v10, v13, sqshrn + calc_qpelh2 v3, v4, v16, v19, v22, v25, v28, v31, v10, v13, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v14.8h-v16.8h}, [sp], x7 + calc_qpelh v1, v17, v20, v23, v26, v29, v8, v11, v14, sqshrn + calc_qpelh2 v1, v2, v17, v20, v23, v26, v29, v8, v11, v14, sqshrn2 + calc_qpelh v2, v18, v21, v24, v27, v30, v9, v12, v15, sqshrn + calc_qpelh2 v2, v3, v18, v21, v24, v27, v30, v9, v12, v15, sqshrn2 + calc_qpelh v3, v19, v22, v25, v28, v31, v10, v13, v16, sqshrn + calc_qpelh2 v3, v4, v19, v22, v25, v28, v31, v10, v13, v16, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v17.8h-v19.8h}, [sp], x7 + calc_qpelh v1, v20, v23, v26, v29, v8, v11, v14, v17, sqshrn + calc_qpelh2 v1, v2, v20, v23, v26, v29, v8, v11, v14, v17, sqshrn2 + calc_qpelh v2, v21, v24, v27, v30, v9, v12, v15, v18, sqshrn + calc_qpelh2 v2, v3, v21, v24, v27, v30, v9, v12, v15, v18, sqshrn2 + calc_qpelh v3, v22, v25, v28, v31, v10, v13, v16, v19, sqshrn + calc_qpelh2 v3, v4, v22, v25, v28, v31, v10, v13, v16, v19, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v20.8h-v22.8h}, [sp], x7 + calc_qpelh v1, v23, v26, v29, v8, v11, v14, v17, v20, sqshrn + calc_qpelh2 v1, v2, v23, v26, v29, v8, v11, v14, v17, v20, sqshrn2 + calc_qpelh v2, v24, v27, v30, v9, v12, v15, v18, v21, sqshrn + calc_qpelh2 v2, v3, v24, v27, v30, v9, v12, v15, v18, v21, sqshrn2 + calc_qpelh v3, v25, v28, v31, v10, v13, v16, v19, v22, sqshrn + calc_qpelh2 v3, v4, v25, v28, v31, v10, v13, v16, v19, v22, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v23.8h-v25.8h}, [sp], x7 + calc_qpelh v1, v26, v29, v8, v11, v14, v17, v20, v23, sqshrn + calc_qpelh2 v1, v2, v26, v29, v8, v11, v14, v17, v20, v23, sqshrn2 + calc_qpelh v2, v27, v30, v9, v12, v15, v18, v21, v24, sqshrn + calc_qpelh2 v2, v3, v27, v30, v9, v12, v15, v18, v21, v24, sqshrn2 + calc_qpelh v3, v28, v31, v10, v13, v16, v19, v22, v25, sqshrn + calc_qpelh2 v3, v4, v28, v31, v10, v13, v16, v19, v22, v25, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v26.8h-v28.8h}, [sp], x7 + calc_qpelh v1, v29, v8, v11, v14, v17, v20, v23, v26, sqshrn + calc_qpelh2 v1, v2, v29, v8, v11, v14, v17, v20, v23, v26, sqshrn2 + calc_qpelh v2, v30, v9, v12, v15, v18, v21, v24, v27, sqshrn + calc_qpelh2 v2, v3, v30, v9, v12, v15, v18, v21, v24, v27, sqshrn2 + calc_qpelh v3, v31, v10, v13, v16, v19, v22, v25, v28, sqshrn + calc_qpelh2 v3, v4, v31, v10, v13, v16, v19, v22, v25, v28, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.hi 1b +2: ld1 {v12.16b-v15.16b}, [sp], #64 + ld1 {v8.16b-v11.16b}, [sp], #64 + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv32_8_neon, export=1 + add w10, w3, #7 + sub x1, x1, x2, lsl #1 + lsl x10, x10, #7 + sub x1, x1, x2 + sub sp, sp, x10 // tmp_array + stp x0, x3, [sp, #-16]! + add x3, x3, #7 + stp x5, x30, [sp, #-16]! + add x0, sp, #32 + bl X(ff_hevc_put_hevc_qpel_h32_8_neon) + ldp x5, x30, [sp], #16 + mov x7, #128 + ldp x0, x3, [sp], #16 + load_qpel_filterh x5, x4 +0: mov x8, sp // src + ld1 {v16.8h, v17.8h}, [x8], x7 + mov w9, w3 // height + ld1 {v18.8h, v19.8h}, [x8], x7 + mov x5, x0 // dst + ld1 {v20.8h, v21.8h}, [x8], x7 + ld1 {v22.8h, v23.8h}, [x8], x7 + ld1 {v24.8h, v25.8h}, [x8], x7 + ld1 {v26.8h, v27.8h}, [x8], x7 + ld1 {v28.8h, v29.8h}, [x8], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [x8], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + calc_qpelh2 v2, v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn2 + subs x9, x9, #1 + st1 {v1.8h, v2.8h}, [x5], x7 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #32 + add sp, sp, #32 + subs w6, w6, #16 + b.hi 0b + add w10, w3, #6 + add sp, sp, #64 // discard rest of first line + lsl x10, x10, #7 + add sp, sp, x10 // tmp_array without first line + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv48_8_neon, export=1 + stp xzr, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_hv24_8_neon) + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + add x1, x1, #24 + add x0, x0, #48 + bl X(ff_hevc_put_hevc_qpel_hv24_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv64_8_neon, export=1 + stp xzr, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + mov x6, #32 + bl X(ff_hevc_put_hevc_qpel_hv32_8_neon) + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + add x1, x1, #32 + add x0, x0, #64 + mov x6, #32 + bl X(ff_hevc_put_hevc_qpel_hv32_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h4_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 +.macro calc op, src + \op v20.8h, v16.8b, v\src\().8b + ushr v16.2d, v16.2d, #8 + mov v16.b[7], v17.b[\src] +.endm + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 +// no purgem + umlsl v20.8h, v16.8b, v7.8b + sqrshrun v20.8b, v20.8h, #6 + subs w4, w4, #1 + st1 {v20.s}[0], [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h6_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 + sub x1, x1, #4 +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 +// same macro as above + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 + umlsl v20.8h, v16.8b, v7.8b + sqrshrun v20.8b, v20.8h, #6 + st1 {v20.s}[0], [x0], #4 + subs w4, w4, #1 + st1 {v20.h}[2], [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h8_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 +// same macro as above + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 +.purgem calc + umlsl v20.8h, v16.8b, v7.8b + sqrshrun v20.8b, v20.8h, #6 + subs w4, w4, #1 + st1 {v20.8b}, [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h12_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 + sub x1, x1, #8 +1: ld2 {v16.8b, v17.8b}, [x2] + movi v20.8h, #0 + ldr w12, [x2, #16] + movi v21.8h, #0 +.macro calc op1, op2, r0, r1, r2, src0, src1, tail=0 + \op1 \r0\().8h, \r1\().8b, \src0\().8b + \op2 \r0\().8h, \r2\().8b, \src1\().8b +.if \tail-1 + ushr \r1\().2d, \r1\().2d, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v0, v1 + mov v16.b[7], w12 + lsr x12, x12, #8 + calc umlsl, umlal, v21, v17, v16, v0, v1 + mov v17.b[7], w12 + lsr x12, x12, #8 + calc umlsl, umlal, v20, v16, v17, v2, v3 + mov v16.b[7], w12 + calc umlsl, umlal, v21, v17, v16, v2, v3 + calc umlal, umlsl, v20, v16, v17, v4, v5 + calc umlal, umlsl, v21, v17, v16, v4, v5 + calc umlal, umlsl, v20, v16, v17, v6, v7 + calc umlal, umlsl, v21, v17, v16, v6, v7, 1 +.purgem calc + zip1 v16.8h, v20.8h, v21.8h + zip2 v17.8h, v20.8h, v21.8h + sqrshrun v20.8b, v16.8h, #6 + sqrshrun2 v20.16b, v17.8h, #6 + st1 {v20.8b}, [x0], #8 + add x2, x2, x3 + st1 {v20.s}[2], [x0], x1 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h16_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld2 {v16.8b, v17.8b}, [x2] + ldr x12, [x2, #16] + movi v20.8h, #0 + movi v21.8h, #0 +.macro calc op1, op2, dst, r0, r1, src0, src1, tail=0 + \op1 \dst\().8h, \r0\().8b, \src0\().8b + \op2 \dst\().8h, \r1\().8b, \src1\().8b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 +.if \tail-1 + lsr x12, x12, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v0, v1 + calc umlsl, umlal, v21, v17, v16, v0, v1 + calc umlsl, umlal, v20, v16, v17, v2, v3 + calc umlsl, umlal, v21, v17, v16, v2, v3 + calc umlal, umlsl, v20, v16, v17, v4, v5 + calc umlal, umlsl, v21, v17, v16, v4, v5 + calc umlal, umlsl, v20, v16, v17, v6, v7, 1 +.purgem calc + umlal v21.8h, v17.8b, v6.8b + umlsl v21.8h, v16.8b, v7.8b + sqrshrun v20.8b, v20.8h, #6 + sqrshrun v21.8b, v21.8h, #6 + st2 {v20.8b, v21.8b}, [x0], x1 + add x2, x2, x3 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h24_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld3 {v16.8b-v18.8b}, [x2] + ldr x12, [x2, #24] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 +.macro calc op1, op2, dst, r0, r1, r2, src0, src1, src2 + \op1 \dst\().8h, \r0\().8b, \src0\().8b + \op2 \dst\().8h, \r1\().8b, \src1\().8b + umlsl \dst\().8h, \r2\().8b, \src2\().8b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 + lsr x12, x12, #8 +.endm + calc umlsl, umlal, v20, v16, v17, v18, v0, v1, v2 + calc umlsl, umlal, v21, v17, v18, v16, v0, v1, v2 + calc umlsl, umlal, v22, v18, v16, v17, v0, v1, v2 + calc umlal, umlal, v20, v16, v17, v18, v3, v4, v5 + calc umlal, umlal, v21, v17, v18, v16, v3, v4, v5 + calc umlal, umlal, v22, v18, v16, v17, v3, v4, v5 +.purgem calc + umlal v20.8h, v16.8b, v6.8b + umlsl v20.8h, v17.8b, v7.8b + ushr v16.2d, v16.2d, #8 + mov v16.b[7], w12 + umlal v21.8h, v17.8b, v6.8b + umlsl v21.8h, v18.8b, v7.8b + umlal v22.8h, v18.8b, v6.8b + umlsl v22.8h, v16.8b, v7.8b + sqrshrun v20.8b, v20.8h, #6 + sqrshrun v22.8b, v22.8h, #6 + sqrshrun v21.8b, v21.8h, #6 + st3 {v20.8b-v22.8b}, [x0], x1 + add x2, x2, x3 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h32_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld4 {v16.8b-v19.8b}, [x2] + ldr x12, [x2, #32] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + movi v23.8h, #0 +.macro calc op1, op2, dst, r0, r1, r2, r3, src0, src1, src2, src3, tail=0 + \op1 \dst\().8h, \r0\().8b, \src0\().8b + \op2 \dst\().8h, \r1\().8b, \src1\().8b + \op1 \dst\().8h, \r2\().8b, \src2\().8b + \op2 \dst\().8h, \r3\().8b, \src3\().8b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 +.if \tail-1 + lsr x12, x12, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v18, v19, v0, v1, v2, v3 + calc umlsl, umlal, v21, v17, v18, v19, v16, v0, v1, v2, v3 + calc umlsl, umlal, v22, v18, v19, v16, v17, v0, v1, v2, v3 + calc umlsl, umlal, v23, v19, v16, v17, v18, v0, v1, v2, v3 + calc umlal, umlsl, v20, v16, v17, v18, v19, v4, v5, v6, v7 + calc umlal, umlsl, v21, v17, v18, v19, v16, v4, v5, v6, v7 + calc umlal, umlsl, v22, v18, v19, v16, v17, v4, v5, v6, v7, 1 +.purgem calc + umlal v23.8h, v19.8b, v4.8b + sqrshrun v20.8b, v20.8h, #6 + umlsl v23.8h, v16.8b, v5.8b + sqrshrun v21.8b, v21.8h, #6 + umlal v23.8h, v17.8b, v6.8b + sqrshrun v22.8b, v22.8h, #6 + umlsl v23.8h, v18.8b, v7.8b + sqrshrun v23.8b, v23.8h, #6 + st4 {v20.8b-v23.8b}, [x0], x1 + add x2, x2, x3 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h48_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld3 {v16.16b-v18.16b}, [x2] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + ldr x12, [x2, #24] + movi v23.8h, #0 + movi v24.8h, #0 + movi v25.8h, #0 + ldr x13, [x2, #48] +.macro calc op1, op2, dst0, dst1, r0, r1, r2, src0, src1, src2 + \op1 \dst0\().8h, \r0\().8b, \src0\().8b + \op2 \dst0\().8h, \r1\().8b, \src1\().8b + umlsl \dst0\().8h, \r2\().8b, \src2\().8b + \op1\()2 \dst1\().8h, \r0\().16b, \src0\().16b + \op2\()2 \dst1\().8h, \r1\().16b, \src1\().16b + umlsl2 \dst1\().8h, \r2\().16b, \src2\().16b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 + mov \r0\().b[15], w13 + lsr x12, x12, #8 + lsr x13, x13, #8 +.endm + calc umlsl, umlal, v20, v23, v16, v17, v18, v0, v1, v2 + calc umlsl, umlal, v21, v24, v17, v18, v16, v0, v1, v2 + calc umlsl, umlal, v22, v25, v18, v16, v17, v0, v1, v2 + calc umlal, umlal, v20, v23, v16, v17, v18, v3, v4, v5 + calc umlal, umlal, v21, v24, v17, v18, v16, v3, v4, v5 + calc umlal, umlal, v22, v25, v18, v16, v17, v3, v4, v5 +.purgem calc +.macro calc r0, r1, r2, r3 + umlal \r0\().8h, \r2\().8b, v6.8b + umlsl \r0\().8h, \r3\().8b, v7.8b + umlal2 \r1\().8h, \r2\().16b, v6.16b + umlsl2 \r1\().8h, \r3\().16b, v7.16b +.endm + calc v20, v23, v16, v17 + ushr v16.2d, v16.2d, #8 + mov v16.b[7], w12 + mov v16.b[15], w13 + calc v21, v24, v17, v18 + calc v22, v25, v18, v16 +.purgem calc + sqrshrun v20.8b, v20.8h, #6 + sqrshrun v21.8b, v21.8h, #6 + sqrshrun v22.8b, v22.8h, #6 + sqrshrun2 v20.16b, v23.8h, #6 + sqrshrun2 v21.16b, v24.8h, #6 + sqrshrun2 v22.16b, v25.8h, #6 + st3 {v20.16b-v22.16b}, [x0], x1 + add x2, x2, x3 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_h64_8_neon, export=1 + load_qpel_filterb x5, x6 + sub x2, x2, #3 +1: ld4 {v16.16b-v19.16b}, [x2] + ldr x12, [x2, #32] + ldr x13, [x2, #64] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + movi v23.8h, #0 + movi v24.8h, #0 + movi v25.8h, #0 + movi v26.8h, #0 + movi v27.8h, #0 +.macro calc op1, op2, dst0, dst1, r0, r1, r2, r3, src0, src1, src2, src3, tail=0 + \op1 \dst0\().8h, \r0\().8b, \src0\().8b + \op2 \dst0\().8h, \r1\().8b, \src1\().8b + \op1 \dst0\().8h, \r2\().8b, \src2\().8b + \op2 \dst0\().8h, \r3\().8b, \src3\().8b + \op1\()2 \dst1\().8h, \r0\().16b, \src0\().16b + \op2\()2 \dst1\().8h, \r1\().16b, \src1\().16b + \op1\()2 \dst1\().8h, \r2\().16b, \src2\().16b + \op2\()2 \dst1\().8h, \r3\().16b, \src3\().16b +.if \tail-1 + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 + mov \r0\().b[15], w13 + lsr x12, x12, #8 + lsr x13, x13, #8 +.endif +.endm + calc umlsl, umlal, v20, v24, v16, v17, v18, v19, v0, v1, v2, v3 + calc umlsl, umlal, v21, v25, v17, v18, v19, v16, v0, v1, v2, v3 + calc umlsl, umlal, v22, v26, v18, v19, v16, v17, v0, v1, v2, v3 + calc umlsl, umlal, v23, v27, v19, v16, v17, v18, v0, v1, v2, v3 + calc umlal, umlsl, v20, v24, v16, v17, v18, v19, v4, v5, v6, v7 + calc umlal, umlsl, v21, v25, v17, v18, v19, v16, v4, v5, v6, v7 + calc umlal, umlsl, v22, v26, v18, v19, v16, v17, v4, v5, v6, v7 + calc umlal, umlsl, v23, v27, v19, v16, v17, v18, v4, v5, v6, v7, 1 +.purgem calc + sqrshrun v20.8b, v20.8h, #6 + sqrshrun v21.8b, v21.8h, #6 + sqrshrun v22.8b, v22.8h, #6 + sqrshrun v23.8b, v23.8h, #6 + sqrshrun2 v20.16b, v24.8h, #6 + sqrshrun2 v21.16b, v25.8h, #6 + sqrshrun2 v22.16b, v26.8h, #6 + sqrshrun2 v23.16b, v27.8h, #6 + st4 {v20.16b-v23.16b}, [x0], x1 + add x2, x2, x3 + subs w4, w4, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v4_8_neon, export=1 + load_qpel_filterb x6, x5 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + ld1 {v16.s}[0], [x2], x3 + ld1 {v17.s}[0], [x2], x3 + ld1 {v18.s}[0], [x2], x3 + ld1 {v19.s}[0], [x2], x3 + ld1 {v20.s}[0], [x2], x3 + ld1 {v21.s}[0], [x2], x3 + ld1 {v22.s}[0], [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().s}[0], [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + sqrshrun v24.8b, v24.8h, #6 + subs w4, w4, #1 + st1 {v24.s}[0], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v6_8_neon, export=1 + load_qpel_filterb x6, x5 + sub x2, x2, x3, lsl #1 + sub x1, x1, #4 + sub x2, x2, x3 + ld1 {v16.8b}, [x2], x3 + ld1 {v17.8b}, [x2], x3 + ld1 {v18.8b}, [x2], x3 + ld1 {v19.8b}, [x2], x3 + ld1 {v20.8b}, [x2], x3 + ld1 {v21.8b}, [x2], x3 + ld1 {v22.8b}, [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + sqrshrun v24.8b, v24.8h, #6 + st1 {v24.s}[0], [x0], #4 + subs w4, w4, #1 + st1 {v24.h}[2], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v8_8_neon, export=1 + load_qpel_filterb x6, x5 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + ld1 {v16.8b}, [x2], x3 + ld1 {v17.8b}, [x2], x3 + ld1 {v18.8b}, [x2], x3 + ld1 {v19.8b}, [x2], x3 + ld1 {v20.8b}, [x2], x3 + ld1 {v21.8b}, [x2], x3 + ld1 {v22.8b}, [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + sqrshrun v24.8b, v24.8h, #6 + subs w4, w4, #1 + st1 {v24.8b}, [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v12_8_neon, export=1 + load_qpel_filterb x6, x5 + sub x2, x2, x3, lsl #1 + sub x1, x1, #8 + sub x2, x2, x3 +0: mov x8, x2 // src + ld1 {v16.16b}, [x8], x3 + mov w11, w4 // height + ld1 {v17.16b}, [x8], x3 + mov x10, x0 // dst + ld1 {v18.16b}, [x8], x3 + ld1 {v19.16b}, [x8], x3 + ld1 {v20.16b}, [x8], x3 + ld1 {v21.16b}, [x8], x3 + ld1 {v22.16b}, [x8], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x8], x3 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + sqrshrun v24.8b, v24.8h, #6 + sqrshrun2 v24.16b, v25.8h, #6 + st1 {v24.8b}, [x10], #8 + subs x11, x11, #1 + st1 {v24.s}[2], [x10], x1 +.endm +1: calc_all +.purgem calc +2: add x0, x0, #12 + add x2, x2, #12 + subs w7, w7, #12 + b.ne 0b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v16_8_neon, export=1 + load_qpel_filterb x6, x5 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 +0: mov x8, x2 // src + ld1 {v16.16b}, [x8], x3 + mov w11, w4 // height + ld1 {v17.16b}, [x8], x3 + mov x10, x0 // dst + ld1 {v18.16b}, [x8], x3 + ld1 {v19.16b}, [x8], x3 + ld1 {v20.16b}, [x8], x3 + ld1 {v21.16b}, [x8], x3 + ld1 {v22.16b}, [x8], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x8], x3 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + sqrshrun v24.8b, v24.8h, #6 + sqrshrun2 v24.16b, v25.8h, #6 + subs x11, x11, #1 + st1 {v24.16b}, [x10], x1 +.endm +1: calc_all +.purgem calc +2: add x0, x0, #16 + add x2, x2, #16 + subs w7, w7, #16 + b.ne 0b + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_v24_8_neon, export=1 + b X(ff_hevc_put_hevc_qpel_uni_v12_8_neon) +endfunc + +function ff_hevc_put_hevc_qpel_uni_v32_8_neon, export=1 + b X(ff_hevc_put_hevc_qpel_uni_v16_8_neon) +endfunc + +function ff_hevc_put_hevc_qpel_uni_v48_8_neon, export=1 + b X(ff_hevc_put_hevc_qpel_uni_v16_8_neon) +endfunc + +function ff_hevc_put_hevc_qpel_uni_v64_8_neon, export=1 + b X(ff_hevc_put_hevc_qpel_uni_v16_8_neon) +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv4_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x30, xzr, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add x3, x4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h4_8_neon) + ldp x30, xzr, [sp], #16 + ldp x4, x6, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x6, x5 + ld1 {v16.4h}, [sp], x9 + ld1 {v17.4h}, [sp], x9 + ld1 {v18.4h}, [sp], x9 + ld1 {v19.4h}, [sp], x9 + ld1 {v20.4h}, [sp], x9 + ld1 {v21.4h}, [sp], x9 + ld1 {v22.4h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().4h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn, #12 + sqxtun v1.8b, v1.8h + subs w4, w4, #1 + st1 {v1.s}[0], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv6_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x30, xzr, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h6_8_neon) + ldp x30, xzr, [sp], #16 + ldp x4, x6, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x6, x5 + sub x1, x1, #4 + ld1 {v16.8h}, [sp], x9 + ld1 {v17.8h}, [sp], x9 + ld1 {v18.8h}, [sp], x9 + ld1 {v19.8h}, [sp], x9 + ld1 {v20.8h}, [sp], x9 + ld1 {v21.8h}, [sp], x9 + ld1 {v22.8h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn, #12 + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn2, #12 + sqxtun v1.8b, v1.8h + st1 {v1.s}[0], [x0], #4 + subs w4, w4, #1 + st1 {v1.h}[2], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv8_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x30, xzr, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h8_8_neon) + ldp x30, xzr, [sp], #16 + ldp x4, x6, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x6, x5 + ld1 {v16.8h}, [sp], x9 + ld1 {v17.8h}, [sp], x9 + ld1 {v18.8h}, [sp], x9 + ld1 {v19.8h}, [sp], x9 + ld1 {v20.8h}, [sp], x9 + ld1 {v21.8h}, [sp], x9 + ld1 {v22.8h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn, #12 + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn2, #12 + sqxtun v1.8b, v1.8h + subs w4, w4, #1 + st1 {v1.8b}, [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv12_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x7, x30, [sp, #-16]! + sub x1, x1, x3 + mov x2, x3 + add x0, sp, #48 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h12_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x6, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x6, x5 + sub x1, x1, #8 + ld1 {v16.8h, v17.8h}, [sp], x9 + ld1 {v18.8h, v19.8h}, [sp], x9 + ld1 {v20.8h, v21.8h}, [sp], x9 + ld1 {v22.8h, v23.8h}, [sp], x9 + ld1 {v24.8h, v25.8h}, [sp], x9 + ld1 {v26.8h, v27.8h}, [sp], x9 + ld1 {v28.8h, v29.8h}, [sp], x9 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn, #12 + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn2, #12 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqrshrn, #12 + sqxtun v1.8b, v1.8h + sqxtun2 v1.16b, v2.8h + st1 {v1.8b}, [x0], #8 + subs w4, w4, #1 + st1 {v1.s}[2], [x0], x1 +.endm +1: calc_all2 +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv16_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + sub x1, x1, x3 + mov x2, x3 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h16_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x6, [sp], #16 + ldp x0, x1, [sp], #16 +.Lqpel_uni_hv16_loop: + mov x9, #(MAX_PB_SIZE * 2) + load_qpel_filterh x6, x5 + sub w12, w9, w7, lsl #1 +0: mov x8, sp // src + ld1 {v16.8h, v17.8h}, [x8], x9 + mov w11, w4 // height + ld1 {v18.8h, v19.8h}, [x8], x9 + mov x10, x0 // dst + ld1 {v20.8h, v21.8h}, [x8], x9 + ld1 {v22.8h, v23.8h}, [x8], x9 + ld1 {v24.8h, v25.8h}, [x8], x9 + ld1 {v26.8h, v27.8h}, [x8], x9 + ld1 {v28.8h, v29.8h}, [x8], x9 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [x8], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn, #12 + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqrshrn2, #12 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqrshrn, #12 + calc_qpelh2 v2, v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqrshrn2, #12 + sqxtun v1.8b, v1.8h + subs x11, x11, #1 + sqxtun2 v1.16b, v2.8h + st1 {v1.16b}, [x10], x1 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #16 + add sp, sp, #32 + subs w7, w7, #16 + b.ne 0b + add w10, w4, #6 + add sp, sp, x12 // discard rest of first line + lsl x10, x10, #7 + add sp, sp, x10 // tmp_array without first line + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv24_8_neon, export=1 + stp x6, x30, [sp, #-16]! + mov x7, #16 + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_uni_hv16_8_neon) + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + add x2, x2, #16 + ldp x0, x1, [sp], #16 + mov x7, #8 + add x0, x0, #16 + ldr x6, [sp] + bl X(ff_hevc_put_hevc_qpel_uni_hv8_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv32_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + stp x7, x30, [sp, #-16]! + sub x1, x2, x3, lsl #1 + add x0, sp, #48 + sub x1, x1, x3 + mov x2, x3 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h32_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x6, [sp], #16 + ldp x0, x1, [sp], #16 + b .Lqpel_uni_hv16_loop +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv48_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x7, x30, [sp, #-16]! + sub x1, x1, x3 + mov x2, x3 + add x0, sp, #48 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h48_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x6, [sp], #16 + ldp x0, x1, [sp], #16 + b .Lqpel_uni_hv16_loop +endfunc + +function ff_hevc_put_hevc_qpel_uni_hv64_8_neon, export=1 + add w10, w4, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x6, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + mov x2, x3 + sub x1, x1, x3 + add w3, w4, #7 + mov x4, x5 + bl X(ff_hevc_put_hevc_qpel_h64_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x6, [sp], #16 + ldp x0, x1, [sp], #16 + b .Lqpel_uni_hv16_loop +endfunc + +function ff_hevc_put_hevc_qpel_bi_h4_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #(MAX_PB_SIZE * 2) +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 +.macro calc op, idx + \op v20.8h, v16.8b, v\idx\().8b + ushr v16.2d, v16.2d, #8 + mov v16.b[7], v17.b[\idx] +.endm + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 + umlsl v20.8h, v16.8b, v7.8b + ld1 {v24.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqrshrun v16.8b, v16.8h, #7 + subs w5, w5, #1 + st1 {v16.s}[0], [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h6_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + sub x1, x1, #4 + mov x10, #(MAX_PB_SIZE * 2) +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 + // same macro + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 + umlsl v20.8h, v16.8b, v7.8b + ld1 {v24.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqrshrun v16.8b, v16.8h, #7 + st1 {v16.s}[0], [x0], #4 + subs w5, w5, #1 + st1 {v16.h}[2], [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h8_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #(MAX_PB_SIZE * 2) +1: ld1 {v16.8b, v17.8b}, [x2], x3 + movi v20.8h, #0 + // same macro + calc umlsl, 0 + calc umlal, 1 + calc umlsl, 2 + calc umlal, 3 + calc umlal, 4 + calc umlsl, 5 + calc umlal, 6 + umlsl v20.8h, v16.8b, v7.8b +.purgem calc + ld1 {v24.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqrshrun v16.8b, v16.8h, #7 + subs w5, w5, #1 + st1 {v16.8b}, [x0], x1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h12_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + sub x1, x1, #8 + mov x10, #(MAX_PB_SIZE * 2) +1: ld2 {v16.8b, v17.8b}, [x2] + movi v20.8h, #0 + ldr w12, [x2, #16] + movi v21.8h, #0 +.macro calc op1, op2, r0, r1, r2, src0, src1, tail=0 + \op1 \r0\().8h, \r1\().8b, \src0\().8b + \op2 \r0\().8h, \r2\().8b, \src1\().8b +.if \tail-1 + ushr \r1\().2d, \r1\().2d, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v0, v1 + mov v16.b[7], w12 + lsr x12, x12, #8 + calc umlsl, umlal, v21, v17, v16, v0, v1 + mov v17.b[7], w12 + lsr x12, x12, #8 + calc umlsl, umlal, v20, v16, v17, v2, v3 + mov v16.b[7], w12 + calc umlsl, umlal, v21, v17, v16, v2, v3 + calc umlal, umlsl, v20, v16, v17, v4, v5 + calc umlal, umlsl, v21, v17, v16, v4, v5 + calc umlal, umlsl, v20, v16, v17, v6, v7 + calc umlal, umlsl, v21, v17, v16, v6, v7, 1 +.purgem calc + ld2 {v24.8h, v25.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqadd v17.8h, v21.8h, v25.8h + sqrshrun v16.8b, v16.8h, #7 + sqrshrun v17.8b, v17.8h, #7 + zip1 v16.16b, v16.16b, v17.16b + st1 {v16.8b}, [x0], #8 + subs w5, w5, #1 + st1 {v16.s}[2], [x0], x1 + add x2, x2, x3 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h16_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #(MAX_PB_SIZE * 2) +1: ld2 {v16.8b, v17.8b}, [x2] + movi v20.8h, #0 + ldr x12, [x2, #16] + movi v21.8h, #0 +.macro calc op1, op2, r0, r1, r2, src0, src1, tail=0 + \op1 \r0\().8h, \r1\().8b, \src0\().8b + \op2 \r0\().8h, \r2\().8b, \src1\().8b + ushr \r1\().2d, \r1\().2d, #8 + mov \r1\().b[7], w12 +.if \tail-1 + lsr x12, x12, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v0, v1 + calc umlsl, umlal, v21, v17, v16, v0, v1 + calc umlsl, umlal, v20, v16, v17, v2, v3 + calc umlsl, umlal, v21, v17, v16, v2, v3 + calc umlal, umlsl, v20, v16, v17, v4, v5 + calc umlal, umlsl, v21, v17, v16, v4, v5 + calc umlal, umlsl, v20, v16, v17, v6, v7, 1 + umlal v21.8h, v17.8b, v6.8b + umlsl v21.8h, v16.8b, v7.8b +.purgem calc + ld2 {v24.8h, v25.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqadd v17.8h, v21.8h, v25.8h + sqrshrun v16.8b, v16.8h, #7 + sqrshrun v17.8b, v17.8h, #7 + subs w5, w5, #1 + st2 {v16.8b, v17.8b}, [x0], x1 + add x2, x2, x3 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h24_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #(MAX_PB_SIZE * 2) + mov x11, x7 // height +1: ld3 {v16.8b-v18.8b}, [x2] + movi v20.8h, #0 + movi v21.8h, #0 + ldr x12, [x2, #24] + movi v22.8h, #0 +.macro calc op1, op2, r0, r1, r2, r3, src0, src1, src2, tail=0 + \op1 \r0\().8h, \r1\().8b, \src0\().8b + \op2 \r0\().8h, \r2\().8b, \src1\().8b + umlsl \r0\().8h, \r3\().8b, \src2\().8b + ushr \r1\().2d, \r1\().2d, #8 + mov \r1\().b[7], w12 + lsr x12, x12, #8 +.endm + calc umlsl, umlal, v20, v16, v17, v18, v0, v1, v2 + calc umlsl, umlal, v21, v17, v18, v16, v0, v1, v2 + calc umlsl, umlal, v22, v18, v16, v17, v0, v1, v2 + calc umlal, umlal, v20, v16, v17, v18, v3, v4, v5 + calc umlal, umlal, v21, v17, v18, v16, v3, v4, v5 + calc umlal, umlal, v22, v18, v16, v17, v3, v4, v5 +.purgem calc + umlal v20.8h, v16.8b, v6.8b + umlsl v20.8h, v17.8b, v7.8b + ushr v16.2d, v16.2d, #8 + mov v16.b[7], w12 + umlal v21.8h, v17.8b, v6.8b + umlsl v21.8h, v18.8b, v7.8b + umlal v22.8h, v18.8b, v6.8b + umlsl v22.8h, v16.8b, v7.8b + ld3 {v23.8h, v24.8h, v25.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v23.8h + sqadd v17.8h, v21.8h, v24.8h + sqadd v18.8h, v22.8h, v25.8h + sqrshrun v16.8b, v16.8h, #7 + sqrshrun v17.8b, v17.8h, #7 + sqrshrun v18.8b, v18.8h, #7 + subs w5, w5, #1 + st3 {v16.8b, v17.8b, v18.8b}, [x0], x1 + add x2, x2, x3 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h32_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #(MAX_PB_SIZE * 2) + mov x11, x7 // height +1: ld4 {v16.8b-v19.8b}, [x2] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + ldr x12, [x2, #32] + movi v23.8h, #0 +.macro calc op1, op2, dst, r0, r1, r2, r3, src0, src1, src2, src3, tail=0 + \op1 \dst\().8h, \r0\().8b, \src0\().8b + \op2 \dst\().8h, \r1\().8b, \src1\().8b + \op1 \dst\().8h, \r2\().8b, \src2\().8b + \op2 \dst\().8h, \r3\().8b, \src3\().8b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 +.if \tail-1 + lsr x12, x12, #8 +.endif +.endm + calc umlsl, umlal, v20, v16, v17, v18, v19, v0, v1, v2, v3 + calc umlsl, umlal, v21, v17, v18, v19, v16, v0, v1, v2, v3 + calc umlsl, umlal, v22, v18, v19, v16, v17, v0, v1, v2, v3 + calc umlsl, umlal, v23, v19, v16, v17, v18, v0, v1, v2, v3 + calc umlal, umlsl, v20, v16, v17, v18, v19, v4, v5, v6, v7 + calc umlal, umlsl, v21, v17, v18, v19, v16, v4, v5, v6, v7 + calc umlal, umlsl, v22, v18, v19, v16, v17, v4, v5, v6, v7, 1 +.purgem calc + umlal v23.8h, v19.8b, v4.8b + umlsl v23.8h, v16.8b, v5.8b + umlal v23.8h, v17.8b, v6.8b + umlsl v23.8h, v18.8b, v7.8b + ld4 {v24.8h, v25.8h, v26.8h, v27.8h}, [x4], x10 + sqadd v16.8h, v20.8h, v24.8h + sqadd v17.8h, v21.8h, v25.8h + sqadd v18.8h, v22.8h, v26.8h + sqadd v19.8h, v23.8h, v27.8h + sqrshrun v16.8b, v16.8h, #7 + sqrshrun v17.8b, v17.8h, #7 + sqrshrun v18.8b, v18.8h, #7 + sqrshrun v19.8b, v19.8h, #7 + st4 {v16.8b-v19.8b}, [x0], x1 + add x2, x2, x3 + subs w5, w5, #1 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h48_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 + mov x10, #80 + mov x11, x7 // height +1: ld3 {v16.16b-v18.16b}, [x2] + ldr x12, [x2, #24] + ldr x13, [x2, #48] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + movi v23.8h, #0 + movi v24.8h, #0 + movi v25.8h, #0 +.macro calc op1, op2, dst0, dst1, r0, r1, r2, src0, src1, src2, tail=0 + \op1 \dst0\().8h, \r0\().8b, \src0\().8b + \op2 \dst0\().8h, \r1\().8b, \src1\().8b + umlsl \dst0\().8h, \r2\().8b, \src2\().8b + \op1\()2 \dst1\().8h, \r0\().16b, \src0\().16b + \op2\()2 \dst1\().8h, \r1\().16b, \src1\().16b + umlsl2 \dst1\().8h, \r2\().16b, \src2\().16b + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 + mov \r0\().b[15], w13 + + lsr x12, x12, #8 + lsr x13, x13, #8 + +.endm + calc umlsl, umlal, v20, v23, v16, v17, v18, v0, v1, v2 + calc umlsl, umlal, v21, v24, v17, v18, v16, v0, v1, v2 + calc umlsl, umlal, v22, v25, v18, v16, v17, v0, v1, v2 + calc umlal, umlal, v20, v23, v16, v17, v18, v3, v4, v5 + calc umlal, umlal, v21, v24, v17, v18, v16, v3, v4, v5 + calc umlal, umlal, v22, v25, v18, v16, v17, v3, v4, v5 +.purgem calc + umlal v20.8h, v16.8b, v6.8b + umlsl v20.8h, v17.8b, v7.8b + umlal2 v23.8h, v16.16b, v6.16b + umlsl2 v23.8h, v17.16b, v7.16b + ushr v16.2d, v16.2d, #8 + mov v16.b[7], w12 + mov v16.b[15], w13 + umlal v21.8h, v17.8b, v6.8b + umlsl v21.8h, v18.8b, v7.8b + umlal2 v24.8h, v17.16b, v6.16b + umlsl2 v24.8h, v18.16b, v7.16b + umlal v22.8h, v18.8b, v6.8b + umlsl v22.8h, v16.8b, v7.8b + umlal2 v25.8h, v18.16b, v6.16b + umlsl2 v25.8h, v16.16b, v7.16b + ld3 {v26.8h, v27.8h, v28.8h}, [x4], #48 + sqadd v16.8h, v20.8h, v26.8h + sqadd v17.8h, v21.8h, v27.8h + sqadd v18.8h, v22.8h, v28.8h + ld3 {v26.8h, v27.8h, v28.8h}, [x4], x10 + sqadd v19.8h, v23.8h, v26.8h + sqadd v20.8h, v24.8h, v27.8h + sqadd v21.8h, v25.8h, v28.8h + sqrshrun v16.8b, v16.8h, #7 + sqrshrun v17.8b, v17.8h, #7 + sqrshrun v18.8b, v18.8h, #7 + sqrshrun2 v16.16b, v19.8h, #7 + sqrshrun2 v17.16b, v20.8h, #7 + sqrshrun2 v18.16b, v21.8h, #7 + subs w5, w5, #1 + st3 {v16.16b-v18.16b}, [x0], x1 + add x2, x2, x3 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_h64_8_neon, export=1 + load_qpel_filterb x6, x7 + sub x2, x2, #3 +1: ld4 {v16.16b-v19.16b}, [x2] + movi v20.8h, #0 + movi v21.8h, #0 + movi v22.8h, #0 + movi v23.8h, #0 + ldr x12, [x2, #32] + movi v24.8h, #0 + movi v25.8h, #0 + ldr x13, [x2, #64] + movi v26.8h, #0 + movi v27.8h, #0 +.macro calc op1, op2, dst0, dst1, r0, r1, r2, r3, src0, src1, src2, src3, tail=0 + \op1 \dst0\().8h, \r0\().8b, \src0\().8b + \op2 \dst0\().8h, \r1\().8b, \src1\().8b + \op1 \dst0\().8h, \r2\().8b, \src2\().8b + \op2 \dst0\().8h, \r3\().8b, \src3\().8b + \op1\()2 \dst1\().8h, \r0\().16b, \src0\().16b + \op2\()2 \dst1\().8h, \r1\().16b, \src1\().16b + \op1\()2 \dst1\().8h, \r2\().16b, \src2\().16b + \op2\()2 \dst1\().8h, \r3\().16b, \src3\().16b +.if \tail-1 + ushr \r0\().2d, \r0\().2d, #8 + mov \r0\().b[7], w12 + lsr x12, x12, #8 + mov \r0\().b[15], w13 + lsr x13, x13, #8 +.endif +.endm + calc umlsl, umlal, v20, v24, v16, v17, v18, v19, v0, v1, v2, v3 + calc umlsl, umlal, v21, v25, v17, v18, v19, v16, v0, v1, v2, v3 + calc umlsl, umlal, v22, v26, v18, v19, v16, v17, v0, v1, v2, v3 + calc umlsl, umlal, v23, v27, v19, v16, v17, v18, v0, v1, v2, v3 + calc umlal, umlsl, v20, v24, v16, v17, v18, v19, v4, v5, v6, v7 + calc umlal, umlsl, v21, v25, v17, v18, v19, v16, v4, v5, v6, v7 + calc umlal, umlsl, v22, v26, v18, v19, v16, v17, v4, v5, v6, v7 + calc umlal, umlsl, v23, v27, v19, v16, v17, v18, v4, v5, v6, v7, 1 +.purgem calc + ld4 {v28.8h-v31.8h}, [x4], #64 + sqadd v20.8h, v20.8h, v28.8h + sqadd v21.8h, v21.8h, v29.8h + sqadd v22.8h, v22.8h, v30.8h + sqadd v23.8h, v23.8h, v31.8h + ld4 {v28.8h-v31.8h}, [x4], #64 + sqadd v24.8h, v24.8h, v28.8h + sqadd v25.8h, v25.8h, v29.8h + sqadd v26.8h, v26.8h, v30.8h + sqadd v27.8h, v27.8h, v31.8h + sqrshrun v16.8b, v20.8h, #7 + sqrshrun v17.8b, v21.8h, #7 + sqrshrun v18.8b, v22.8h, #7 + sqrshrun v19.8b, v23.8h, #7 + sqrshrun2 v16.16b, v24.8h, #7 + sqrshrun2 v17.16b, v25.8h, #7 + sqrshrun2 v18.16b, v26.8h, #7 + sqrshrun2 v19.16b, v27.8h, #7 + subs w5, w5, #1 + st4 {v16.16b-v19.16b}, [x0], x1 + add x2, x2, x3 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v4_8_neon, export=1 + load_qpel_filterb x7, x6 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + mov x12, #(MAX_PB_SIZE * 2) + ld1 {v16.s}[0], [x2], x3 + ld1 {v17.s}[0], [x2], x3 + ld1 {v18.s}[0], [x2], x3 + ld1 {v19.s}[0], [x2], x3 + ld1 {v20.s}[0], [x2], x3 + ld1 {v21.s}[0], [x2], x3 + ld1 {v22.s}[0], [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().s}[0], [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + ld1 {v25.4h}, [x4], x12 // src2 + sqadd v24.8h, v24.8h, v25.8h + sqrshrun v25.8b, v24.8h, #7 + subs w5, w5, #1 + st1 {v25.s}[0], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v6_8_neon, export=1 + load_qpel_filterb x7, x6 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + ld1 {v16.8b}, [x2], x3 + sub x1, x1, #4 + ld1 {v17.8b}, [x2], x3 + mov x12, #(MAX_PB_SIZE * 2) + ld1 {v18.8b}, [x2], x3 + ld1 {v19.8b}, [x2], x3 + ld1 {v20.8b}, [x2], x3 + ld1 {v21.8b}, [x2], x3 + ld1 {v22.8b}, [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + ld1 {v25.8h}, [x4], x12 // src2 + sqadd v24.8h, v24.8h, v25.8h + sqrshrun v25.8b, v24.8h, #7 + st1 {v25.s}[0], [x0], #4 + subs w5, w5, #1 + st1 {v25.h}[2], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v8_8_neon, export=1 + load_qpel_filterb x7, x6 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + mov x12, #(MAX_PB_SIZE * 2) + ld1 {v16.8b}, [x2], x3 + ld1 {v17.8b}, [x2], x3 + ld1 {v18.8b}, [x2], x3 + ld1 {v19.8b}, [x2], x3 + ld1 {v20.8b}, [x2], x3 + ld1 {v21.8b}, [x2], x3 + ld1 {v22.8b}, [x2], x3 + .macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8b}, [x2], x3 + movi v24.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + ld1 {v25.8h}, [x4], x12 // src2 + sqadd v24.8h, v24.8h, v25.8h + sqrshrun v25.8b, v24.8h, #7 + subs w5, w5, #1 + st1 {v25.8b}, [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v12_8_neon, export=1 + load_qpel_filterb x7, x6 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + sub x1, x1, #8 + ld1 {v16.16b}, [x2], x3 + mov x12, #(MAX_PB_SIZE * 2) + ld1 {v17.16b}, [x2], x3 + ld1 {v18.16b}, [x2], x3 + ld1 {v19.16b}, [x2], x3 + ld1 {v20.16b}, [x2], x3 + ld1 {v21.16b}, [x2], x3 + ld1 {v22.16b}, [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x2], x3 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + ld1 {v26.8h, v27.8h}, [x4], x12 // src2 + sqadd v24.8h, v24.8h, v26.8h + sqadd v25.8h, v25.8h, v27.8h + sqrshrun v26.8b, v24.8h, #7 + sqrshrun2 v26.16b, v25.8h, #7 + st1 {v26.8b}, [x0], #8 + subs w5, w5, #1 + st1 {v26.s}[2], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v16_8_neon, export=1 + load_qpel_filterb x7, x6 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + mov x12, #(MAX_PB_SIZE * 2) + ld1 {v16.16b}, [x2], x3 + ld1 {v17.16b}, [x2], x3 + ld1 {v18.16b}, [x2], x3 + ld1 {v19.16b}, [x2], x3 + ld1 {v20.16b}, [x2], x3 + ld1 {v21.16b}, [x2], x3 + ld1 {v22.16b}, [x2], x3 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().16b}, [x2], x3 + movi v24.8h, #0 + movi v25.8h, #0 + calc_qpelb v24, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v25, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + ld1 {v26.8h, v27.8h}, [x4], x12 // src2 + sqadd v24.8h, v24.8h, v26.8h + sqadd v25.8h, v25.8h, v27.8h + sqrshrun v26.8b, v24.8h, #7 + subs w5, w5, #1 + sqrshrun2 v26.16b, v25.8h, #7 + st1 {v26.16b}, [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v24_8_neon, export=1 + stp x7, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_bi_v16_8_neon) + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + ldr x7, [sp] + add x0, x0, #16 + add x2, x2, #16 + add x4, x4, #32 + bl X(ff_hevc_put_hevc_qpel_bi_v8_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v32_8_neon, export=1 + sub sp, sp, #64 + st1 {v12.16b-v15.16b}, [sp] + sub x2, x2, x3, lsl #1 + sub sp, sp, #64 + st1 {v8.16b-v11.16b}, [sp] + sub x2, x2, x3 + load_qpel_filterb x7, x6 + ldr w6, [sp, #128] + mov x12, #(MAX_PB_SIZE * 2) +0: mov x8, x2 // src + ld1 {v16.16b, v17.16b}, [x8], x3 + mov w11, w5 // height + ld1 {v18.16b, v19.16b}, [x8], x3 + mov x10, x0 // dst + ld1 {v20.16b, v21.16b}, [x8], x3 + mov x9, x4 // src2 + ld1 {v22.16b, v23.16b}, [x8], x3 + ld1 {v24.16b, v25.16b}, [x8], x3 + ld1 {v26.16b, v27.16b}, [x8], x3 + ld1 {v28.16b, v29.16b}, [x8], x3 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [x8], x3 + movi v8.8h, #0 + movi v9.8h, #0 + movi v10.8h, #0 + movi v11.8h, #0 + calc_qpelb v8, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb2 v9, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7 + calc_qpelb v10, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + calc_qpelb2 v11, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15 + ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [x9], x12 // src2 + sqadd v8.8h, v8.8h, v12.8h + sqadd v9.8h, v9.8h, v13.8h + sqadd v10.8h, v10.8h, v14.8h + sqadd v11.8h, v11.8h, v15.8h + sqrshrun v12.8b, v8.8h, #7 + sqrshrun2 v12.16b, v9.8h, #7 + sqrshrun v13.8b, v10.8h, #7 + sqrshrun2 v13.16b, v11.8h, #7 + subs x11, x11, #1 + st1 {v12.16b, v13.16b}, [x10], x1 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #32 // dst + add x2, x2, #32 // src + add x4, x4, #64 // src2 + subs w6, w6, #32 + b.ne 0b + ld1 {v8.16b-v11.16b}, [sp], #64 + ld1 {v12.16b-v15.16b}, [sp], #64 + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v48_8_neon, export=1 + stp x7, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + mov x8, #32 + stp x8, x8, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_bi_v32_8_neon) + ldp x8, xzr, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + ldr x7, [sp] + add x0, x0, #32 + add x2, x2, #32 + add x4, x4, #64 + bl X(ff_hevc_put_hevc_qpel_bi_v16_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_v64_8_neon, export=1 + b X(ff_hevc_put_hevc_qpel_bi_v32_8_neon) +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv4_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x7, x30, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add w3, w5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h4_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x7, x6 + ld1 {v16.4h}, [sp], x9 + ld1 {v17.4h}, [sp], x9 + ld1 {v18.4h}, [sp], x9 + ld1 {v19.4h}, [sp], x9 + ld1 {v20.4h}, [sp], x9 + ld1 {v21.4h}, [sp], x9 + ld1 {v22.4h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().4h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + ld1 {v5.4h}, [x4], x9 // src2 + saddw v1.4s, v1.4s, v5.4h + rshrn v1.4h, v1.4s, #7 + sqxtun v1.8b, v1.8h + subs w5, w5, #1 + st1 {v1.s}[0], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv6_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x7, x30, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add x3, x5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h6_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x7, x6 + sub x1, x1, #4 + ld1 {v16.8h}, [sp], x9 + ld1 {v17.8h}, [sp], x9 + ld1 {v18.8h}, [sp], x9 + ld1 {v19.8h}, [sp], x9 + ld1 {v20.8h}, [sp], x9 + ld1 {v21.8h}, [sp], x9 + ld1 {v22.8h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + calc_qpelh2 v2, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + ld1 {v5.8h}, [x4], x9 // src2 + saddw v1.4s, v1.4s, v5.4h + saddw2 v2.4s, v2.4s, v5.8h + rshrn v1.4h, v1.4s, #7 + rshrn2 v1.8h, v2.4s, #7 + sqxtun v1.8b, v1.8h + st1 {v1.s}[0], [x0], #4 + subs w5, w5, #1 + st1 {v1.h}[2], [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv8_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + sub x1, x2, x3, lsl #1 + stp x7, x30, [sp, #-16]! + sub x1, x1, x3 + add x0, sp, #48 + mov x2, x3 + add x3, x5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h8_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + mov x9, #(MAX_PB_SIZE * 2) + ldp x0, x1, [sp], #16 + load_qpel_filterh x7, x6 + ld1 {v16.8h}, [sp], x9 + ld1 {v17.8h}, [sp], x9 + ld1 {v18.8h}, [sp], x9 + ld1 {v19.8h}, [sp], x9 + ld1 {v20.8h}, [sp], x9 + ld1 {v21.8h}, [sp], x9 + ld1 {v22.8h}, [sp], x9 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + calc_qpelh2 v2, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + ld1 {v5.8h}, [x4], x9 // src2 + saddw v1.4s, v1.4s, v5.4h + saddw2 v2.4s, v2.4s, v5.8h + rshrn v1.4h, v1.4s, #7 + rshrn2 v1.8h, v2.4s, #7 + sqxtun v1.8b, v1.8h + subs w5, w5, #1 + st1 {v1.8b}, [x0], x1 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv12_8_neon, export=1 + stp xzr, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_bi_hv8_8_neon) + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + add x4, x4, #16 + ldp x0, x1, [sp], #16 + add x2, x2, #8 + add x0, x0, #8 + bl X(ff_hevc_put_hevc_qpel_bi_hv4_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv16_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + sub x1, x1, x3 + mov x2, x3 + add w3, w5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h16_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x0, x1, [sp], #16 + mov x6, #16 // width +.Lqpel_bi_hv16_loop: + load_qpel_filterh x7, x8 + mov x9, #(MAX_PB_SIZE * 2) + mov x10, x6 + +0: mov x8, sp // src + ld1 {v16.8h, v17.8h}, [x8], x9 + mov w11, w5 // height + ld1 {v18.8h, v19.8h}, [x8], x9 + mov x12, x4 // src2 + ld1 {v20.8h, v21.8h}, [x8], x9 + mov x7, x0 // dst + ld1 {v22.8h, v23.8h}, [x8], x9 + ld1 {v24.8h, v25.8h}, [x8], x9 + ld1 {v26.8h, v27.8h}, [x8], x9 + ld1 {v28.8h, v29.8h}, [x8], x9 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [x8], x9 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + calc_qpelh2 v2, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sshr + calc_qpelh v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sshr + calc_qpelh2 v4, v4, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sshr + ld1 {v5.8h, v6.8h}, [x12], x9 // src2 + saddw v1.4s, v1.4s, v5.4h + saddw2 v2.4s, v2.4s, v5.8h + saddw v3.4s, v3.4s, v6.4h + saddw2 v4.4s, v4.4s, v6.8h + rshrn v1.4h, v1.4s, #7 + rshrn2 v1.8h, v2.4s, #7 + rshrn v2.4h, v3.4s, #7 + rshrn2 v2.8h, v4.4s, #7 + sqxtun v1.8b, v1.8h + sqxtun2 v1.16b, v2.8h + subs x11, x11, #1 + st1 {v1.16b}, [x7], x1 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #16 + add sp, sp, #32 + subs x10, x10, #16 + add x4, x4, #32 + b.ne 0b + add w10, w5, #7 + lsl x10, x10, #7 + sub x10, x10, x6, lsl #1 // part of first line + add sp, sp, x10 // tmp_array without first line + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv24_8_neon, export=1 + stp xzr, x30, [sp, #-16]! + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + bl X(ff_hevc_put_hevc_qpel_bi_hv16_8_neon) + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + add x4, x4, #32 + add x2, x2, #16 + add x0, x0, #16 + bl X(ff_hevc_put_hevc_qpel_bi_hv8_8_neon) + ldp xzr, x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv32_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + mov x2, x3 + sub x1, x1, x3 + add w3, w5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h32_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x0, x1, [sp], #16 + mov x6, #32 // width + b .Lqpel_bi_hv16_loop +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv48_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + mov x2, x3 + sub x1, x1, x3 + add w3, w5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h48_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x0, x1, [sp], #16 + mov x6, #48 // width + b .Lqpel_bi_hv16_loop +endfunc + +function ff_hevc_put_hevc_qpel_bi_hv64_8_neon, export=1 + add w10, w5, #7 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x0, x1, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x7, x30, [sp, #-16]! + add x0, sp, #48 + sub x1, x2, x3, lsl #1 + mov x2, x3 + sub x1, x1, x3 + add w3, w5, #7 + mov x4, x6 + bl X(ff_hevc_put_hevc_qpel_h64_8_neon) + ldp x7, x30, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x0, x1, [sp], #16 + mov x6, #64 // width + b .Lqpel_bi_hv16_loop +endfunc -- 2.32.0 (Apple Git-132)

2 5

[PATCH] configure: Fix Microsoft tools detection
by Kacper Michajłow 07 Feb '22

07 Feb '22

LLVM tools print installation path upon execution. If one uses LLVM tools bundled with Microsoft Visual Studio installation, they would be incorrectly detected as Microsoft's ones. Signed-off-by: Kacper Michajłow <kasper93(a)gmail.com> --- configure | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 94f513288a..f27fd067eb 100755 --- a/configure +++ b/configure @@ -4820,9 +4820,9 @@ probe_cc(){ _flags_filter=msvc_flags _ld_lib='lib%.a' _ld_path='-libpath:' - elif $_cc -nologo- 2>&1 | grep -q Microsoft || { $_cc -v 2>&1 | grep -q clang && $_cc -? > /dev/null 2>&1; }; then + elif $_cc -nologo- 2>&1 | grep -q 'Microsoft.*Optimizing.*Compiler' || { $_cc -v 2>&1 | grep -q clang && $_cc -? > /dev/null 2>&1; }; then _type=msvc - if $_cc -nologo- 2>&1 | grep -q Microsoft; then + if $_cc -nologo- 2>&1 | grep -q 'Microsoft.*Optimizing.*Compiler'; then _ident=$($_cc 2>&1 | head -n1 | tr -d '\r') else _ident=$($_cc --version 2>/dev/null | head -n1 | tr -d '\r') @@ -4927,7 +4927,7 @@ if [ -z "$CC_DEPFLAGS" ] && [ "$dep_cc" != "$cc" ]; then DEPCCFLAGS=$_flags fi -if $ar 2>&1 | grep -q Microsoft; then +if $ar 2>&1 | grep -q 'Microsoft.*Library.*Manager'; then arflags="-nologo" ar_o='-out:$@' elif $ar 2>&1 | grep -q "\[D\] "; then -- 2.34.1

5 11

[PATCH V2 1/3] libavcodec/vaapi_encode: Add new API adaption to vaapi_encode
by Wenbin Chen 07 Feb '22

07 Feb '22

Add vaSyncBuffer to VAAPI encoder. Old version API vaSyncSurface wait surface to complete. When surface is used for multiple operation, it waits all operations to finish. vaSyncBuffer only wait one channel to finish. Add wait param to vaapi_encode_wait() to prepare for the async_depth option. "wait=1" means wait until operation ready. "wait=0" means query operation's status. If it is ready return 0, if it is still in progress return EAGAIN. Signed-off-by: Wenbin Chen <wenbin.chen(a)intel.com> --- libavcodec/vaapi_encode.c | 47 +++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c index 3bf379b1a0..b87b58a42b 100644 --- a/libavcodec/vaapi_encode.c +++ b/libavcodec/vaapi_encode.c @@ -134,7 +134,8 @@ static int vaapi_encode_make_misc_param_buffer(AVCodecContext *avctx, } static int vaapi_encode_wait(AVCodecContext *avctx, - VAAPIEncodePicture *pic) + VAAPIEncodePicture *pic, + uint8_t wait) { VAAPIEncodeContext *ctx = avctx->priv_data; VAStatus vas; @@ -150,11 +151,43 @@ static int vaapi_encode_wait(AVCodecContext *avctx, "(input surface %#x).\n", pic->display_order, pic->encode_order, pic->input_surface); - vas = vaSyncSurface(ctx->hwctx->display, pic->input_surface); - if (vas != VA_STATUS_SUCCESS) { - av_log(avctx, AV_LOG_ERROR, "Failed to sync to picture completion: " - "%d (%s).\n", vas, vaErrorStr(vas)); +#if VA_CHECK_VERSION(1, 9, 0) + // Try vaSyncBuffer. + vas = vaSyncBuffer(ctx->hwctx->display, + pic->output_buffer, + wait ? VA_TIMEOUT_INFINITE : 0); + if (vas == VA_STATUS_ERROR_TIMEDOUT) { + return AVERROR(EAGAIN); + } else if (vas != VA_STATUS_SUCCESS && vas != VA_STATUS_ERROR_UNIMPLEMENTED) { + av_log(avctx, AV_LOG_ERROR, "Failed to sync to output buffer completion: " + "%d (%s).\n", vas, vaErrorStr(vas)); return AVERROR(EIO); + } else if (vas == VA_STATUS_ERROR_UNIMPLEMENTED) + // If vaSyncBuffer is not implemented, try old version API. +#endif + { + if (!wait) { + VASurfaceStatus surface_status; + vas = vaQuerySurfaceStatus(ctx->hwctx->display, + pic->input_surface, + &surface_status); + if (vas == VA_STATUS_SUCCESS && + surface_status != VASurfaceReady && + surface_status != VASurfaceSkipped) { + return AVERROR(EAGAIN); + } else if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to query surface status: " + "%d (%s).\n", vas, vaErrorStr(vas)); + return AVERROR(EIO); + } + } else { + vas = vaSyncSurface(ctx->hwctx->display, pic->input_surface); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to sync to picture completion: " + "%d (%s).\n", vas, vaErrorStr(vas)); + return AVERROR(EIO); + } + } } // Input is definitely finished with now. @@ -633,7 +666,7 @@ static int vaapi_encode_output(AVCodecContext *avctx, uint8_t *ptr; int err; - err = vaapi_encode_wait(avctx, pic); + err = vaapi_encode_wait(avctx, pic, 1); if (err < 0) return err; @@ -695,7 +728,7 @@ fail: static int vaapi_encode_discard(AVCodecContext *avctx, VAAPIEncodePicture *pic) { - vaapi_encode_wait(avctx, pic); + vaapi_encode_wait(avctx, pic, 1); if (pic->output_buffer_ref) { av_log(avctx, AV_LOG_DEBUG, "Discard output for pic " -- 2.25.1

4 7

[PATCH 1/2] lavf/udp: set ttl upper bound to 255
by Zhao Zhili 06 Feb '22

06 Feb '22

--- libavformat/udp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/udp.c b/libavformat/udp.c index 83c042d079..b441d2ea0d 100644 --- a/libavformat/udp.c +++ b/libavformat/udp.c @@ -134,7 +134,7 @@ static const AVOption options[] = { { "reuse", "explicitly allow reusing UDP sockets", OFFSET(reuse_socket), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, D|E }, { "reuse_socket", "explicitly allow reusing UDP sockets", OFFSET(reuse_socket), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, .flags = D|E }, { "broadcast", "explicitly allow or disallow broadcast destination", OFFSET(is_broadcast), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, E }, - { "ttl", "Time to live (multicast only)", OFFSET(ttl), AV_OPT_TYPE_INT, { .i64 = 16 }, 0, INT_MAX, E }, + { "ttl", "Time to live (multicast only)", OFFSET(ttl), AV_OPT_TYPE_INT, { .i64 = 16 }, 0, 255, E }, { "connect", "set if connect() should be called on socket", OFFSET(is_connected), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, .flags = D|E }, { "fifo_size", "set the UDP receiving circular buffer size, expressed as a number of packets with size of 188 bytes", OFFSET(circular_buffer_size), AV_OPT_TYPE_INT, {.i64 = 7*4096}, 0, INT_MAX, D }, { "overrun_nonfatal", "survive in case of UDP receiving circular buffer overrun", OFFSET(overrun_nonfatal), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, D }, -- 2.29.2

4 3