[FFmpeg-devel] [PATCH] avfilter/vf_freezedetect: add filter to detect frozen input

Paul B Mahol onemda at gmail.com
Sun Nov 11 22:20:57 EET 2018


On 11/11/18, Marton Balint <cus at passwd.hu> wrote:
> Signed-off-by: Marton Balint <cus at passwd.hu>
> ---
>  Changelog                     |   1 +
>  configure                     |   1 +
>  doc/filters.texi              |  29 +++++
>  libavfilter/Makefile          |   1 +
>  libavfilter/allfilters.c      |   1 +
>  libavfilter/version.h         |   2 +-
>  libavfilter/vf_freezedetect.c | 282
> ++++++++++++++++++++++++++++++++++++++++++
>  7 files changed, 316 insertions(+), 1 deletion(-)
>  create mode 100644 libavfilter/vf_freezedetect.c
>
> diff --git a/Changelog b/Changelog
> index e38a607025..0eba82b477 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -4,6 +4,7 @@ releases are sorted from youngest to oldest.
>  version <next>:
>  - tpad filter
>  - AV1 decoding support through libdav1d
> +- freezedetect filter
>
>
>  version 4.1:
> diff --git a/configure b/configure
> index b02b4ccb2e..e42957ba9d 100755
> --- a/configure
> +++ b/configure
> @@ -3402,6 +3402,7 @@ firequalizer_filter_deps="avcodec"
>  firequalizer_filter_select="rdft"
>  flite_filter_deps="libflite"
>  framerate_filter_select="scene_sad"
> +freezedetect_filter_select="scene_sad"
>  frei0r_filter_deps="frei0r libdl"
>  frei0r_src_filter_deps="frei0r libdl"
>  fspp_filter_deps="gpl"
> diff --git a/doc/filters.texi b/doc/filters.texi
> index fb1dd8f353..bdc9aca2dd 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -10016,6 +10016,35 @@ Select frame after every @code{step} frames.
>  Allowed values are positive integers higher than 0. Default value is
> @code{1}.
>  @end table
>
> + at section freezedetect
> +
> +Detect frozen video.
> +
> +This filter logs a message and sets frame metadata when it detects that the
> +input video has no significant change in content during a specified
> duration.
> +Video freeze detection calculates the mean average absolute difference of
> all
> +the components of video frames and compares it to a noise floor.
> +
> +The printed times and duration are expressed in seconds. The
> + at code{lavfi.freezedetect.freeze_start} metadata key is set on the first
> frame
> +whose timestamp equals or exceeds the detection duration and it contains
> the
> +timstamp of the first frame of the freeze. The
> + at code{lavfi.freezedetect.freeze_duration} and
> + at code{lavfi.freezedetect.freeze_end} metadata keys are set on the first
> frame
> +after the freeze.
> +
> +The filter accepts the following options:
> +
> + at table @option
> + at item noise, n
> +Set noise tolerance. Can be specified in dB (in case "dB" is appended to
> the
> +specified value) or as a difference ratio between 0 and 1. Default is
> -60dB, or
> +0.001.
> +
> + at item duration, d
> +Set freeze duration until notification (default is 2 seconds).
> + at end table
> +
>  @anchor{frei0r}
>  @section frei0r
>
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 7c6fc836e5..30a8b8f921 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -236,6 +236,7 @@ OBJS-$(CONFIG_FPS_FILTER)                    += vf_fps.o
>  OBJS-$(CONFIG_FRAMEPACK_FILTER)              += vf_framepack.o
>  OBJS-$(CONFIG_FRAMERATE_FILTER)              += vf_framerate.o
>  OBJS-$(CONFIG_FRAMESTEP_FILTER)              += vf_framestep.o
> +OBJS-$(CONFIG_FREEZEDETECT_FILTER)           += vf_freezedetect.o
>  OBJS-$(CONFIG_FREI0R_FILTER)                 += vf_frei0r.o
>  OBJS-$(CONFIG_FSPP_FILTER)                   += vf_fspp.o
>  OBJS-$(CONFIG_GBLUR_FILTER)                  += vf_gblur.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 484b080dea..f0f0521dee 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -222,6 +222,7 @@ extern AVFilter ff_vf_fps;
>  extern AVFilter ff_vf_framepack;
>  extern AVFilter ff_vf_framerate;
>  extern AVFilter ff_vf_framestep;
> +extern AVFilter ff_vf_freezedetect;
>  extern AVFilter ff_vf_frei0r;
>  extern AVFilter ff_vf_fspp;
>  extern AVFilter ff_vf_gblur;
> diff --git a/libavfilter/version.h b/libavfilter/version.h
> index 83b18008ce..b4bb8f7bab 100644
> --- a/libavfilter/version.h
> +++ b/libavfilter/version.h
> @@ -30,7 +30,7 @@
>  #include "libavutil/version.h"
>
>  #define LIBAVFILTER_VERSION_MAJOR   7
> -#define LIBAVFILTER_VERSION_MINOR  43
> +#define LIBAVFILTER_VERSION_MINOR  44
>  #define LIBAVFILTER_VERSION_MICRO 100
>
>  #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
> diff --git a/libavfilter/vf_freezedetect.c b/libavfilter/vf_freezedetect.c
> new file mode 100644
> index 0000000000..df59eb2134
> --- /dev/null
> +++ b/libavfilter/vf_freezedetect.c
> @@ -0,0 +1,282 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +
> +/**
> + * @file
> + * video freeze detection filter
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/timestamp.h"
> +
> +#include "avfilter.h"
> +#include "filters.h"
> +#include "scene_sad.h"
> +
> +typedef struct FreezeDetectContext {
> +    const AVClass *class;
> +
> +    ptrdiff_t width[4];
> +    ptrdiff_t height[4];
> +    ff_scene_sad_fn sad;
> +    int bitdepth;
> +    AVFrame *reference_frame;
> +    int64_t n;
> +    int64_t reference_n;
> +    int frozen;
> +
> +    double noise;
> +    int64_t duration;            ///< minimum duration of frozen frame
> until notification
> +} FreezeDetectContext;
> +
> +#define OFFSET(x) offsetof(FreezeDetectContext, x)
> +#define V AV_OPT_FLAG_VIDEO_PARAM
> +#define F AV_OPT_FLAG_FILTERING_PARAM
> +
> +static const AVOption freezedetect_options[] = {
> +    { "n",                   "set noise tolerance",
> OFFSET(noise),  AV_OPT_TYPE_DOUBLE,   {.dbl=0.001},     0,       1.0, V|F },
> +    { "noise",               "set noise tolerance",
> OFFSET(noise),  AV_OPT_TYPE_DOUBLE,   {.dbl=0.001},     0,       1.0, V|F },
> +    { "d",                   "set minimum duration in seconds",
> OFFSET(duration),  AV_OPT_TYPE_DURATION, {.i64=2000000},   0, INT64_MAX, V|F
> },
> +    { "duration",            "set minimum duration in seconds",
> OFFSET(duration),  AV_OPT_TYPE_DURATION, {.i64=2000000},   0, INT64_MAX, V|F
> },
> +
> +    {NULL}
> +};
> +
> +AVFILTER_DEFINE_CLASS(freezedetect);
> +
> +static int query_formats(AVFilterContext *ctx)
> +{
> +    static const enum AVPixelFormat pix_fmts[] = {
> +        AV_PIX_FMT_YUV420P,
> +        AV_PIX_FMT_YUYV422,
> +        AV_PIX_FMT_RGB24,
> +        AV_PIX_FMT_BGR24,
> +        AV_PIX_FMT_YUV422P,
> +        AV_PIX_FMT_YUV444P,
> +        AV_PIX_FMT_YUV410P,
> +        AV_PIX_FMT_YUV411P,
> +        AV_PIX_FMT_GRAY8,
> +        AV_PIX_FMT_YUVJ420P,
> +        AV_PIX_FMT_YUVJ422P,
> +        AV_PIX_FMT_YUVJ444P,
> +        AV_PIX_FMT_UYVY422,
> +        AV_PIX_FMT_NV12,
> +        AV_PIX_FMT_NV21,
> +        AV_PIX_FMT_ARGB,
> +        AV_PIX_FMT_RGBA,
> +        AV_PIX_FMT_ABGR,
> +        AV_PIX_FMT_BGRA,
> +        AV_PIX_FMT_GRAY16,
> +        AV_PIX_FMT_YUV440P,
> +        AV_PIX_FMT_YUVJ440P,
> +        AV_PIX_FMT_YUVA420P,
> +        AV_PIX_FMT_YUV420P16,
> +        AV_PIX_FMT_YUV422P16,
> +        AV_PIX_FMT_YUV444P16,
> +        AV_PIX_FMT_YA8,
> +        AV_PIX_FMT_YUV420P9,
> +        AV_PIX_FMT_YUV420P10,
> +        AV_PIX_FMT_YUV422P10,
> +        AV_PIX_FMT_YUV444P9,
> +        AV_PIX_FMT_YUV444P10,
> +        AV_PIX_FMT_YUV422P9,
> +        AV_PIX_FMT_GBRP,
> +        AV_PIX_FMT_GBRP9,
> +        AV_PIX_FMT_GBRP10,
> +        AV_PIX_FMT_GBRP16,
> +        AV_PIX_FMT_YUVA422P,
> +        AV_PIX_FMT_YUVA444P,
> +        AV_PIX_FMT_YUVA420P9,
> +        AV_PIX_FMT_YUVA422P9,
> +        AV_PIX_FMT_YUVA444P9,
> +        AV_PIX_FMT_YUVA420P10,
> +        AV_PIX_FMT_YUVA422P10,
> +        AV_PIX_FMT_YUVA444P10,
> +        AV_PIX_FMT_YUVA420P16,
> +        AV_PIX_FMT_YUVA422P16,
> +        AV_PIX_FMT_YUVA444P16,
> +        AV_PIX_FMT_NV16,
> +        AV_PIX_FMT_YVYU422,
> +        AV_PIX_FMT_GBRAP,
> +        AV_PIX_FMT_GBRAP16,
> +        AV_PIX_FMT_YUV420P12,
> +        AV_PIX_FMT_YUV420P14,
> +        AV_PIX_FMT_YUV422P12,
> +        AV_PIX_FMT_YUV422P14,
> +        AV_PIX_FMT_YUV444P12,
> +        AV_PIX_FMT_YUV444P14,
> +        AV_PIX_FMT_GBRP12,
> +        AV_PIX_FMT_GBRP14,
> +        AV_PIX_FMT_YUVJ411P,
> +        AV_PIX_FMT_YUV440P10,
> +        AV_PIX_FMT_YUV440P12,
> +        AV_PIX_FMT_GBRAP12,
> +        AV_PIX_FMT_GBRAP10,
> +        AV_PIX_FMT_GRAY12,
> +        AV_PIX_FMT_GRAY10,
> +        AV_PIX_FMT_GRAY9,
> +        AV_PIX_FMT_GRAY14,
> +        AV_PIX_FMT_NONE

Please make this list more compact, make use of several items per line.

> +    };
> +
> +    AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
> +    if (!fmts_list)
> +        return AVERROR(ENOMEM);
> +    return ff_set_common_formats(ctx, fmts_list);
> +}
> +
> +static int config_input(AVFilterLink *inlink)
> +{
> +    AVFilterContext *ctx = inlink->dst;
> +    FreezeDetectContext *s = ctx->priv;
> +    const AVPixFmtDescriptor *pix_desc =
> av_pix_fmt_desc_get(inlink->format);
> +
> +    s->bitdepth = pix_desc->comp[0].depth;
> +
> +    for (int plane = 0; plane < 4; plane++) {
> +        ptrdiff_t line_size = av_image_get_linesize(inlink->format,
> inlink->w, plane);
> +        s->width[plane] = line_size >> (s->bitdepth > 8);
> +        s->height[plane] = inlink->h >> ((plane == 1 || plane == 2) ?
> pix_desc->log2_chroma_h : 0);
> +    }
> +
> +    s->sad = ff_scene_sad_get_fn(s->bitdepth == 8 ? 8 : 16);
> +    if (!s->sad)
> +        return AVERROR(EINVAL);
> +
> +    return 0;
> +}
> +
> +static av_cold void uninit(AVFilterContext *ctx)
> +{
> +    FreezeDetectContext *s = ctx->priv;
> +    av_frame_free(&s->reference_frame);
> +}
> +
> +static int is_frozen(FreezeDetectContext *s, AVFrame *reference, AVFrame
> *frame)
> +{
> +    uint64_t sad = 0;
> +    uint64_t count = 0;
> +    double mafd;
> +    for (int plane = 0; plane < 4; plane++) {
> +        if (s->width[plane]) {
> +            uint64_t plane_sad;
> +            s->sad(frame->data[plane], frame->linesize[plane],
> +                   reference->data[plane], reference->linesize[plane],
> +                   s->width[plane], s->height[plane], &plane_sad);
> +            sad += plane_sad;
> +            count += s->width[plane] * s->height[plane];
> +        }
> +    }
> +    emms_c();
> +    mafd = (double)sad / count / (1ULL << s->bitdepth);
> +    return (mafd <= s->noise);
> +}
> +
> +static int set_meta(FreezeDetectContext *s, AVFrame *frame, const char
> *key, const char *value)
> +{
> +    av_log(s, AV_LOG_INFO, "%s: %s\n", key, value);
> +    return av_dict_set(&frame->metadata, key, value, 0);
> +}
> +
> +static int activate(AVFilterContext *ctx)
> +{
> +    int ret;
> +    AVFilterLink *inlink = ctx->inputs[0];
> +    AVFilterLink *outlink = ctx->outputs[0];
> +    FreezeDetectContext *s = ctx->priv;
> +    AVFrame *frame;
> +
> +    FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
> +
> +    ret = ff_inlink_consume_frame(inlink, &frame);
> +    if (ret < 0)
> +        return ret;
> +
> +    if (frame) {
> +        int frozen = 0;
> +        s->n++;
> +
> +        if (s->reference_frame) {
> +            int64_t duration;
> +            if (s->reference_frame->pts == AV_NOPTS_VALUE || frame->pts ==
> AV_NOPTS_VALUE || frame->pts < s->reference_frame->pts)     //
> Discontinuity?
> +                duration = inlink->frame_rate.num > 0 ? av_rescale_q(s->n -
> s->reference_n, av_inv_q(inlink->frame_rate), AV_TIME_BASE_Q) : 0;
> +            else
> +                duration = av_rescale_q(frame->pts -
> s->reference_frame->pts, inlink->time_base, AV_TIME_BASE_Q);
> +
> +            frozen = is_frozen(s, s->reference_frame, frame);
> +            if (duration >= s->duration) {
> +                if (frozen) {
> +                    if (!s->frozen)
> +                        set_meta(s, frame,
> "lavfi.freezedetect.freeze_start", av_ts2timestr(s->reference_frame->pts,
> &inlink->time_base));
> +                } else {
> +                    set_meta(s, frame,
> "lavfi.freezedetect.freeze_duration", av_ts2timestr(duration,
> &AV_TIME_BASE_Q));
> +                    set_meta(s, frame, "lavfi.freezedetect.freeze_end",
> av_ts2timestr(frame->pts, &inlink->time_base));
> +                }
> +                s->frozen = frozen;
> +            }
> +        }
> +
> +        if (!frozen) {
> +            av_frame_free(&s->reference_frame);
> +            s->reference_frame = av_frame_clone(frame);
> +            s->reference_n = s->n;
> +            if (!s->reference_frame) {
> +                av_frame_free(&frame);
> +                return AVERROR(ENOMEM);
> +            }
> +        }
> +        return ff_filter_frame(outlink, frame);
> +    }
> +
> +    FF_FILTER_FORWARD_STATUS(inlink, outlink);
> +    FF_FILTER_FORWARD_WANTED(outlink, inlink);
> +
> +    return FFERROR_NOT_READY;
> +}
> +
> +static const AVFilterPad freezedetect_inputs[] = {
> +    {
> +        .name         = "default",
> +        .type         = AVMEDIA_TYPE_VIDEO,
> +        .config_props = config_input,
> +    },
> +    { NULL }
> +};
> +
> +static const AVFilterPad freezedetect_outputs[] = {
> +    {
> +        .name          = "default",
> +        .type          = AVMEDIA_TYPE_VIDEO,
> +    },
> +    { NULL }
> +};
> +
> +AVFilter ff_vf_freezedetect = {
> +    .name          = "freezedetect",
> +    .description   = NULL_IF_CONFIG_SMALL("Detects frozen video input."),
> +    .priv_size     = sizeof(FreezeDetectContext),
> +    .priv_class    = &freezedetect_class,
> +    .uninit        = uninit,
> +    .query_formats = query_formats,
> +    .inputs        = freezedetect_inputs,
> +    .outputs       = freezedetect_outputs,
> +    .activate      = activate,
> +};
> --
> 2.16.4
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>


More information about the ffmpeg-devel mailing list