[FFmpeg-devel] [PATCH] libavfilter: added atempo filter (revised patch)
Clément Bœsch
ubitux at gmail.com
Thu Jun 7 08:00:31 CEST 2012
On Wed, Jun 06, 2012 at 10:17:28AM -0600, pkoshevoy at gmail.com wrote:
> From: Pavel Koshevoy <pkoshevoy at gmail.com>
>
> Added atempo audio filter for adjusting audio tempo without affecting
> pitch. This filter implements WSOLA algorithm with fast cross
> correlation calculation in frequency domain.
>
> Signed-off-by: Pavel Koshevoy <pavel at homestead.aragog.com>
> ---
> Changelog | 2 +-
> configure | 1 +
> doc/filters.texi | 17 +
> libavfilter/Makefile | 2 +
> libavfilter/af_atempo.c | 1245 ++++++++++++++++++++++++++++++++++++++++++++++
> libavfilter/allfilters.c | 1 +
> libavfilter/version.h | 2 +-
> 7 files changed, 1268 insertions(+), 2 deletions(-)
> create mode 100644 libavfilter/af_atempo.c
>
> diff --git a/Changelog b/Changelog
> index 41b0bdc..cc25c9b 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -5,7 +5,7 @@ version next:
> - INI and flat output in ffprobe
> - Scene detection in libavfilter
> - Indeo Audio decoder
> -
Please keep that empty line :)
> +- atempo filter
>
> version 0.11:
>
> diff --git a/configure b/configure
> index 33bd439..7b82b64 100755
> --- a/configure
> +++ b/configure
> @@ -1687,6 +1687,7 @@ amovie_filter_deps="avcodec avformat"
> aresample_filter_deps="swresample"
> ass_filter_deps="libass"
> asyncts_filter_deps="avresample"
> +atempo_filter_deps="avcodec"
> blackframe_filter_deps="gpl"
> boxblur_filter_deps="gpl"
> colormatrix_filter_deps="gpl"
> diff --git a/doc/filters.texi b/doc/filters.texi
> index d9d503f..0b7dc8e 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -271,6 +271,23 @@ For example, to resample the input audio to 44100Hz:
> aresample=44100
> @end example
>
> + at section atempo
> +
> +Adjust audio tempo.
> +
> +The filter accepts exactly one parameter, the audio tempo. If not
> +specified then the filter will assume nominal tempo.
add "(a value of 1.0)"?
> +
> +For example, to slow down audio to 80% tempo:
> + at example
> +atempo=0.8
> + at end example
> +
> +For example, to speed up audio to 125% tempo:
> + at example
> +atempo=1.25
> + at end example
> +
> @section ashowinfo
>
> Show a line containing various information for each input audio frame.
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 29345fc..a1ced51 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -8,6 +8,7 @@ FFLIBS-$(CONFIG_RESAMPLE_FILTER) += avresample
> FFLIBS-$(CONFIG_ACONVERT_FILTER) += swresample
> FFLIBS-$(CONFIG_AMOVIE_FILTER) += avformat avcodec
> FFLIBS-$(CONFIG_ARESAMPLE_FILTER) += swresample
> +FFLIBS-$(CONFIG_ATEMPO_FILTER) += avcodec
> FFLIBS-$(CONFIG_MOVIE_FILTER) += avformat avcodec
> FFLIBS-$(CONFIG_PAN_FILTER) += swresample
> FFLIBS-$(CONFIG_REMOVELOGO_FILTER) += avformat avcodec
> @@ -54,6 +55,7 @@ OBJS-$(CONFIG_ASHOWINFO_FILTER) += af_ashowinfo.o
> OBJS-$(CONFIG_ASPLIT_FILTER) += split.o
> OBJS-$(CONFIG_ASTREAMSYNC_FILTER) += af_astreamsync.o
> OBJS-$(CONFIG_ASYNCTS_FILTER) += af_asyncts.o
> +OBJS-$(CONFIG_ATEMPO_FILTER) += af_atempo.o
> OBJS-$(CONFIG_EARWAX_FILTER) += af_earwax.o
> OBJS-$(CONFIG_PAN_FILTER) += af_pan.o
> OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o
> diff --git a/libavfilter/af_atempo.c b/libavfilter/af_atempo.c
> new file mode 100644
> index 0000000..1866d0a
> --- /dev/null
> +++ b/libavfilter/af_atempo.c
> @@ -0,0 +1,1245 @@
> +/*
> + * Copyright (c) 2012 Pavel Koshevoy
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * tempo scaling audio filter -- an implementation of WSOLA algorithm
> + */
> +
> +#include <float.h>
> +#include "libavcodec/avfft.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/eval.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/samplefmt.h"
> +#include "avfilter.h"
> +#include "audio.h"
> +#include "internal.h"
> +
> +/**
> + * A fragment of audio waveform
> + */
> +typedef struct {
> + // index of the first sample of this fragment in the overall waveform;
> + // 0: input sample position
> + // 1: output sample position
> + int64_t position[2];
> +
> + // original packed multi-channel samples:
> + unsigned char *data;
> +
> + // number of samples in this fragment:
> + int nsamples;
> +
> + // FFT transform of the downmixed mono fragment, used for
> + // fast waveform alignment via correlation in frequency domain:
> + FFTComplex *xdat;
> +
> +} TAudioFragment;
> +
> +/**
> + * Filter state machine states
> + */
> +typedef enum {
> + kLoadFragment = 0,
> + kAdjustPosition = 1,
> + kReloadFragment = 2,
> + kOutputOverlapAdd = 3,
> + kFlushOutput = 4
> +
nit: you can add a comma at the end of this line so adding new entries
won't require a modification of this line.
> +} TState;
> +
> +/**
> + * Filter state machine
> + */
> +typedef struct {
> + // ring-buffer of input samples, necessary because some times
> + // input fragment position may be adjusted backwards:
> + unsigned char *buffer;
> +
Please s/unsigned char/uint8_t/ all over the file
> + // ring-buffer maximum capacity,
> + // expressed as number of multi-channel sample units;
> + //
> + // for example, given stereo data 1 multi-channel sample unit
> + // refers to 2 samples for left/right channels:
> + int ring;
> +
> + // ring-buffer house keeping:
> + int size;
> + int head;
> + int tail;
> +
> + // 0: input sample position corresponding to the ring buffer tail
> + // 1: output sample position
> + int64_t position[2];
> +
> + // sample format:
> + enum AVSampleFormat format;
> +
> + // number of channels:
> + int channels;
> +
> + // row of bytes to skip from one sample to next, across multple channels;
> + // stride = (number-of-channels * bits-per-sample-per-channel) / 8
> + int stride;
> +
> + // fragment window size, power-of-two integer:
> + int window;
> +
> + // Hann window coefficients, for feathering
> + // (blending) the overlapping fragment region:
> + float *hann;
> +
> + // tempo scaling factor:
> + double tempo;
> +
> + // cumulative alignment drift:
> + int drift;
> +
> + // current/previous fragment ring-buffer:
> + TAudioFragment frag[2];
> +
> + // current fragment index:
> + uint64_t nfrag;
> +
> + // current state:
> + TState state;
> +
> + // for fast correlation calculation in frequency domain:
> + FFTContext *fft_forward;
> + FFTContext *fft_inverse;
> + FFTComplex *correlation;
> +
> + // for managing AVFilterPad::request_frame and AVFilterPad::filter_samples
nit: AVFilterPad.request_frame and AVFilterPad.filter_samples
> + int request_fulfilled;
> + AVFilterBufferRef *dst_buffer;
> + unsigned char *dst;
> + unsigned char *dst_end;
> + uint64_t nsamples_in;
> + uint64_t nsamples_out;
> +
> +} ATempoContext;
> +
> +/**
> + * Initialize filter state.
> + */
> +static void yae_constructor(ATempoContext *atempo)
> +{
> + atempo->ring = 0;
> + atempo->size = 0;
> + atempo->head = 0;
> + atempo->tail = 0;
> +
> + atempo->format = AV_SAMPLE_FMT_NONE;
> + atempo->channels = 0;
> +
> + atempo->window = 0;
> + atempo->tempo = 1.0;
> + atempo->drift = 0;
> +
> + memset(&atempo->frag[0], 0, sizeof(atempo->frag));
> +
> + atempo->nfrag = 0;
> + atempo->state = kLoadFragment;
> +
> + atempo->position[0] = 0;
> + atempo->position[1] = 0;
> +
> + atempo->fft_forward = NULL;
> + atempo->fft_inverse = NULL;
> + atempo->correlation = NULL;
> +
> + atempo->request_fulfilled = 0;
> + atempo->dst_buffer = NULL;
> + atempo->dst = NULL;
> + atempo->dst_end = NULL;
> + atempo->nsamples_in = 0;
> + atempo->nsamples_out = 0;
> +}
> +
This context should already be zero-ed, so it's likely this function can
be simplified to:
atempo->format = AV_SAMPLE_FMT_NONE;
atempo->tempo = 1.0;
atempo->state = kLoadFragment;
> +/**
> + * Deallocate filter buffers.
> + */
> +static void yae_destructor(ATempoContext *atempo)
> +{
> + av_freep(&atempo->frag[0].data);
> + av_freep(&atempo->frag[1].data);
> + av_freep(&atempo->frag[0].xdat);
> + av_freep(&atempo->frag[1].xdat);
> +
> + av_freep(&atempo->buffer);
> + av_freep(&atempo->hann);
> + av_freep(&atempo->correlation);
> +
> + if (atempo->fft_forward) {
> + av_fft_end(atempo->fft_forward);
> + atempo->fft_forward = NULL;
> + }
> +
> + if (atempo->fft_inverse) {
> + av_fft_end(atempo->fft_inverse);
> + atempo->fft_inverse = NULL;
> + }
> +
You don't need to check for fft_forward and fft_inverse, av_fft_end() will
do it.
> +}
> +
> +/**
> + * Reset given fragment to initial state
> + */
> +static void yae_clear_frag(TAudioFragment *frag)
> +{
> + frag->position[0] = 0;
> + frag->position[1] = 0;
> + frag->nsamples = 0;
> +}
> +
> +/**
> + * Reset filter to initial state
> + */
> +static void yae_clear(ATempoContext *atempo)
> +{
> + atempo->size = 0;
> + atempo->head = 0;
> + atempo->tail = 0;
> +
> + atempo->drift = 0;
> + atempo->nfrag = 0;
> + atempo->state = kLoadFragment;
> +
> + atempo->position[0] = 0;
> + atempo->position[1] = 0;
> +
> + yae_clear_frag(&atempo->frag[0]);
> + yae_clear_frag(&atempo->frag[1]);
> +
> + // shift left position of 1st fragment by half a window
> + // so that no re-normalization would be required for
> + // the left half of the 1st fragment:
> + atempo->frag[0].position[0] = -(int64_t)(atempo->window / 2);
> + atempo->frag[0].position[1] = -(int64_t)(atempo->window / 2);
> +
> + if (atempo->dst_buffer) {
> + avfilter_unref_buffer(atempo->dst_buffer);
> + atempo->dst_buffer = NULL;
avfilter_unref_bufferp()
> + atempo->dst = NULL;
> + atempo->dst_end = NULL;
> + }
> +
> + atempo->request_fulfilled = 0;
> + atempo->nsamples_in = 0;
> + atempo->nsamples_out = 0;
> +}
> +
> +/**
> + * Prepare filter for processing audio data of given format,
> + * sample rate and number of channels.
> + */
> +static void yae_reset(ATempoContext *atempo,
> + enum AVSampleFormat format,
> + int sample_rate,
> + int channels)
> +{
> + const int sample_size = av_get_bytes_per_sample(format);
> + unsigned int nlevels = 0;
> + unsigned int pot;
> +
> + atempo->format = format;
> + atempo->channels = channels;
> + atempo->stride = sample_size * channels;
> +
> + // pick a segment window size:
> + atempo->window = sample_rate / 24;
> +
> + // adjust window size to be a power-of-two integer:
> + nlevels = av_log2_c(atempo->window);
av_log2()
> + pot = 1 << nlevels;
> + av_assert0(pot <= atempo->window);
> +
> + if (pot < atempo->window) {
> + atempo->window = pot * 2;
> + nlevels++;
> + }
> +
> + atempo->frag[0].data = av_realloc(atempo->frag[0].data,
> + atempo->window * atempo->stride);
> +
> + atempo->frag[1].data = av_realloc(atempo->frag[1].data,
> + atempo->window * atempo->stride);
> +
> + atempo->frag[0].xdat = av_realloc(atempo->frag[0].xdat,
> + atempo->window * 2 *
> + sizeof(FFTComplex));
> +
> + atempo->frag[1].xdat = av_realloc(atempo->frag[1].xdat,
> + atempo->window * 2 *
> + sizeof(FFTComplex));
> +
Here and below, it would be nice to check for the realloc and raise AVERROR(ENOMEM) in
case of error.
> + // initialize FFT contexts:
> + if (atempo->fft_forward) {
> + av_fft_end(atempo->fft_forward);
> + }
> +
> + if (atempo->fft_inverse) {
> + av_fft_end(atempo->fft_inverse);
> + }
> +
> + atempo->fft_forward = av_fft_init(nlevels + 1, 0);
> + atempo->fft_inverse = av_fft_init(nlevels + 1, 1);
> + atempo->correlation = (FFTComplex *)av_realloc(atempo->correlation,
> + atempo->window * 2 *
> + sizeof(FFTComplex));
You don't need that cast in C.
> +
> + atempo->ring = atempo->window * 3;
> + atempo->buffer = av_realloc(atempo->buffer, atempo->ring * atempo->stride);
> +
> + // sample the Hann window function:
> + atempo->hann = av_realloc(atempo->hann, atempo->window * sizeof(float));
> + for (int i = 0; i < atempo->window; i++) {
I think we still try to avoid declaring the "int" in the loop for some
compatibility issues.
> + double t = (double)i / (double)(atempo->window - 1);
> + double h = 0.5 * (1.0 - cos(2.0 * M_PI * t));
> + atempo->hann[i] = (float)h;
> + }
> +
> + yae_clear(atempo);
> +}
> +
> +static int yae_set_tempo(ATempoContext *atempo,
> + double tempo,
> + AVFilterContext *ctx)
> +{
> + if (tempo < 0.5 || tempo > 2.0) {
> + av_log(ctx, AV_LOG_ERROR, "tempo value %f exceeds [0.5, 2.0] range\n",
> + tempo);
> + return AVERROR(EINVAL);
> + }
> +
> + atempo->tempo = tempo;
> + return 0;
> +}
> +
> +inline static TAudioFragment * yae_curr_frag(ATempoContext *atempo)
> +{
> + return &atempo->frag[atempo->nfrag % 2];
> +}
> +
> +inline static TAudioFragment * yae_prev_frag(ATempoContext *atempo)
> +{
> + return &atempo->frag[(atempo->nfrag + 1) % 2];
> +}
> +
> +/**
> + * Find the minimum of two scalars
> + */
> +#define yae_min(TScalar, a, b) \
> + ((TScalar)a < (TScalar)b ? \
> + (TScalar)a : \
> + (TScalar)b)
> +
> +/**
> + * Find the maximum of two scalars
> + */
> +#define yae_max(TScalar, a, b) \
> + ((TScalar)a < (TScalar)b ? \
> + (TScalar)b : \
> + (TScalar)a)
> +
Is the cast really needed?
Also, you can use FFMIN() and FFMAX()
> +
> +/**
> + * A helper macro for initializing complex data buffer with scalar data
> + * of a given type.
> + */
> +#define yae_init_xdat(TScalar, scalar_max) \
> + do { \
> + const unsigned char *src_end = \
> + src + frag->nsamples * atempo->channels * sizeof(TScalar); \
> + \
> + FFTComplex *xdat = frag->xdat; \
> + TScalar tmp; \
> + \
> + if (atempo->channels == 1) { \
> + float s; \
> + \
> + for (; src < src_end; blend++) { \
> + memcpy(&tmp, src, sizeof(TScalar)); \
> + src += sizeof(TScalar); \
> + \
tmp = *src++ is not possible?
> + s = (float)tmp; \
> + \
> + xdat->re = s; \
> + xdat->im = 0; \
> + xdat++; \
> + } \
> + } else { \
> + float s; \
> + float t0; \
> + float max; \
> + float ti; \
> + float s0; \
> + \
> + for (; src < src_end; blend++) { \
> + memcpy(&tmp, src, sizeof(TScalar)); \
> + src += sizeof(TScalar); \
> + \
> + t0 = (float)tmp; \
> + s = yae_min(float, scalar_max, fabsf(t0)); \
> + max = (float)t0; \
> + \
> + for (int i = 1; i < atempo->channels; i++) { \
ditto int
> + memcpy(&tmp, src, sizeof(TScalar)); \
> + src += sizeof(TScalar); \
> + \
> + ti = (float)tmp; \
> + s0 = yae_min(float, scalar_max, fabsf(ti)); \
> + \
> + if (s < s0) { \
> + s = s0; \
> + max = ti; \
> + } \
> + } \
> + \
> + xdat->re = max; \
> + xdat->im = 0; \
> + xdat++; \
Wouldn't it be simpler to request for planar formats? (U16P, S16P, etc)
[...]
> +
> +/**
> + * Frame request callback. A call to this should result in at least
> + * one frame being output over the given link. This should return
> + * zero on success.
> + */
> +static int request_frame(AVFilterLink *outlink)
> +{
> + AVFilterContext *ctx = outlink->src;
> + ATempoContext *atempo = ctx->priv;
> + int ret;
> +
> + atempo->request_fulfilled = 0;
> + do {
> + ret = avfilter_request_frame(ctx->inputs[0]);
> + }
> + while (!atempo->request_fulfilled && ret >= 0);
> +
> + if (ret == AVERROR_EOF) {
> + // flush the filter:
> + int n_max = atempo->ring;
> + int n_out;
> + int err = AVERROR(EAGAIN);
> +
> + while (err == AVERROR(EAGAIN)) {
> + if (!atempo->dst_buffer) {
> + atempo->dst_buffer = ff_get_audio_buffer(outlink,
> + AV_PERM_WRITE,
> + n_max);
> +
> + atempo->dst = atempo->dst_buffer->data[0];
> + atempo->dst_end = atempo->dst + n_max * atempo->stride;
> + }
> +
> + err = yae_flush(atempo, &atempo->dst, atempo->dst_end);
> +
> + n_out = ((atempo->dst - atempo->dst_buffer->data[0]) /
> + atempo->stride);
> +
> + if (n_out) {
> + atempo->dst_buffer->audio->sample_rate = outlink->sample_rate;
> + atempo->dst_buffer->audio->nb_samples = n_out;
> +
> + // adjust the PTS:
> + atempo->dst_buffer->pts =
> + av_rescale(outlink->time_base.den,
> + atempo->nsamples_out,
> + outlink->time_base.num * outlink->sample_rate);
> +
> + ff_filter_samples(outlink, atempo->dst_buffer);
> + atempo->dst_buffer = NULL;
> + atempo->dst = NULL;
> + atempo->dst_end = NULL;
> +
> + atempo->nsamples_out += n_out;
> + }
> + }
> +
> + if (atempo->dst_buffer) {
> + avfilter_unref_buffer(atempo->dst_buffer);
> + atempo->dst_buffer = NULL;
avfilter_unref_bufferp()
> + atempo->dst = NULL;
> + atempo->dst_end = NULL;
> + }
> +
> + return AVERROR_EOF;
> + }
> +
> + return ret;
> +}
> +
[...]
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index b9d44f2..e8c8406 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -44,6 +44,7 @@ void avfilter_register_all(void)
> REGISTER_FILTER (ASPLIT, asplit, af);
> REGISTER_FILTER (ASTREAMSYNC, astreamsync, af);
> REGISTER_FILTER (ASYNCTS, asyncts, af);
> + REGISTER_FILTER (ATEMPO, atempo, af);
> REGISTER_FILTER (EARWAX, earwax, af);
> REGISTER_FILTER (PAN, pan, af);
> REGISTER_FILTER (SILENCEDETECT, silencedetect, af);
> diff --git a/libavfilter/version.h b/libavfilter/version.h
> index 76f649e..c90b4ad 100644
> --- a/libavfilter/version.h
> +++ b/libavfilter/version.h
> @@ -30,7 +30,7 @@
>
> #define LIBAVFILTER_VERSION_MAJOR 2
> #define LIBAVFILTER_VERSION_MINOR 78
> -#define LIBAVFILTER_VERSION_MICRO 100
> +#define LIBAVFILTER_VERSION_MICRO 101
>
I think you need to bump MINOR instead.
[...]
No more comment from me, but a more serious review from someone else would
be welcome.
Thank you for your submission.
--
Clément B.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 490 bytes
Desc: not available
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20120607/b2894c86/attachment.asc>
More information about the ffmpeg-devel
mailing list