[WIP] [PATCH 1/2] lavfi: add an API to synchronize multiple video inputs.
Compared to dualinput, this API can handle more than two inputs and can generate frames synchronized to any or all input streams. Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/framesync.c | 222 +++++++++++++++++++++++++++++++++++++++++++ libavfilter/framesync.h | 242 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 464 insertions(+) create mode 100644 libavfilter/framesync.c create mode 100644 libavfilter/framesync.h Note: this is work in progress, but I do not know when I will be able to work on it again. It came as a reaction to mail on the user mailing list: how to overlay a video on top of a constant background. The problem is that the overlay filter, and the dualinput API that was extracted from it to serve for other filters, can only synchronize on the background video: since the background is a constant image, it does not move. If the video on top has constant frame rate, the problem can be solved by setting vf_fps on the background. Otherwise, the suggested solution was to make a sandwich: overlay the completely opaque background on top of the video to get the background with the video timestamps, and then overlay the video. That works (except if the background itself has alpha), but that is ugly and inefficient. The obvious quick-and-dirty solution would be to add sync=second option to the dualinput API, but that is just that: quick and dirty, it does not cover other uses: combining R, G, B input planes (three inputs), computing the PSNR of videos with the same frame rate but different skipped frames (must generate an event for frames on any input). This API aims to handle all the cases. diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c new file mode 100644 index 0000000..2e0b1db --- /dev/null +++ b/libavfilter/framesync.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2013 Nicolas George + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/avassert.h" +#include "avfilter.h" +#include "bufferqueue.h" +#include "framesync.h" +#include "internal.h" + +/* + * TODO + * Actually compute time base, den = GCM(dens) and num = LCD(nums) + * Some more testing (especially wrt write status when cloning frame). + * Uninit structure and check for leaks. + */ + +#define OFFSET(member) offsetof(FFFrameSync, member) + +static const char *framesync_name(void *ptr) +{ + return "framesync"; +} + +static const AVClass framesync_class = { + .version = LIBAVUTIL_VERSION_INT, + .class_name = "framesync", + .item_name = framesync_name, + .category = AV_CLASS_CATEGORY_FILTER, + .option = NULL, + .parent_log_context_offset = OFFSET(parent), +}; + +enum { + STATE_BOF, + STATE_RUN, + STATE_EOF, +}; + +void ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in) +{ + fs->class = &framesync_class; + fs->parent = parent; + fs->nb_in = nb_in; +} + +int ff_framesync_configure(FFFrameSync *fs) +{ + unsigned i; + + if (!fs->time_base.num) { + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].sync) { + if (fs->time_base.num) { + //av_assert0(!"todo"); + } else { + fs->time_base = fs->in[i].time_base; + } + } + } + if (!fs->time_base.num) { + av_log(fs, AV_LOG_ERROR, "Impossible to set time base\n"); + return AVERROR(EINVAL); + } + } + + for (i = 0; i < fs->nb_in; i++) + fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE; + + return 0; +} + +static void framesync_advance(FFFrameSync *fs) +{ + int latest; + unsigned i; + int64_t pts; + + if (fs->eof) + return; + while (!fs->frame_ready) { + latest = -1; + for (i = 0; i < fs->nb_in; i++) { + if (!fs->in[i].have_next) { + if (latest < 0 || fs->in[i].pts < fs->in[latest].pts) + latest = i; + } + } + if (latest >= 0) { + fs->in_request = latest; + break; + } + + pts = fs->in[0].pts_next; + for (i = 1; i < fs->nb_in; i++) + if (fs->in[i].pts_next < pts) + pts = fs->in[i].pts_next; + if (pts == INT64_MAX) { + fs->eof = 1; + break; + } + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].pts_next == pts || + (fs->in[i].before == EXT_INFINITY && + fs->in[i].state == STATE_BOF)) { + av_frame_free(&fs->in[i].frame); + fs->in[i].frame = fs->in[i].frame_next; + fs->in[i].pts = fs->in[i].pts_next; + fs->in[i].frame_next = NULL; + fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->in[i].have_next = 0; + fs->in[i].state = fs->in[i].frame ? STATE_RUN : STATE_EOF; + if (fs->in[i].sync && fs->in[i].frame) + fs->frame_ready = 1; + if (fs->in[i].state == STATE_EOF && + fs->in[i].after == EXT_NULL) { + fs->eof = 1; + } + } + } + if (fs->eof) + fs->frame_ready = 0; + if (fs->frame_ready) + for (i = 0; i < fs->nb_in; i++) + if ((fs->in[i].state == STATE_BOF && + fs->in[i].before == EXT_NULL)) + fs->frame_ready = 0; + fs->pts = pts; + } +} + +static void framesync_inject_frame(FFFrameSync *fs, unsigned in, AVFrame *frame) +{ + int64_t pts; + + av_assert0(!fs->in[in].have_next); + if (frame) { + pts = av_rescale_q(frame->pts, fs->in[in].time_base, fs->time_base); + frame->pts = pts; + } else { + pts = fs->in[in].state != STATE_RUN || fs->in[in].after == EXT_INFINITY ? INT64_MAX : + fs->in[in].pts + 1; /* TODO extrapolate */ + } + fs->in[in].frame_next = frame; + fs->in[in].pts_next = pts; + fs->in[in].have_next = 1; +} + +int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame) +{ + av_assert1(in < fs->nb_in); + if (!fs->in[in].have_next) + framesync_inject_frame(fs, in, frame); + else + ff_bufqueue_add(fs, &fs->in[in].queue, frame); + return 0; +} + +void ff_framesync_next(FFFrameSync *fs) +{ + unsigned i; + + av_assert0(!fs->frame_ready); + for (i = 0; i < fs->nb_in; i++) + if (!fs->in[i].have_next && fs->in[i].queue.available) + framesync_inject_frame(fs, i, ff_bufqueue_get(&fs->in[i].queue)); + fs->frame_ready = 0; + framesync_advance(fs); +} + +void ff_framesync_drop(FFFrameSync *fs) +{ + fs->frame_ready = 0; +} + +int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, + unsigned get) +{ + AVFrame *frame; + unsigned need_copy = 0, i; + int64_t pts_next; + + if (!fs->in[in].frame) { + *rframe = NULL; + return 0; + } + frame = fs->in[in].frame; + if (get) { + /* Find out if we need to copy the frame: is there another sync + stream, and do we know if its current frame will outlast this one? */ + pts_next = fs->in[in].have_next ? fs->in[in].pts_next : INT64_MAX; + for (i = 0; i < fs->nb_in && !need_copy; i++) + if (i != in && fs->in[i].sync && + (!fs->in[i].have_next || fs->in[i].pts_next < pts_next)) + need_copy = 1; + if (need_copy) { + if (!(frame = av_frame_clone(frame))) + return AVERROR(ENOMEM); + } else { + fs->in[in].frame = NULL; + } + fs->frame_ready = 0; + } + *rframe = frame; + return 0; +} diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h new file mode 100644 index 0000000..ecbb88c --- /dev/null +++ b/libavfilter/framesync.h @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2013 Nicolas George + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVFILTER_FRAMESYNC_H +#define AVFILTER_FRAMESYNC_H + +#include "bufferqueue.h" + +/* + * TODO + * Callback-based API similar to dualinput. + * Rename ExtModes. + * Export convenient options. + */ + +/** + * This API is intended as a helper for filters that have several video + * input and need to combine them somehow. If the inputs have different or + * variable frame rate, getting the input frames to match requires a rather + * complex logic and a few user-tunable options. + * + * In this API, when a set of synchronized input frames is ready to be + * procesed is called a frame event. Frame event can be generated in + * response to input frames on any or all inputs and the handling of + * situations where some stream extend beyond the beginning or the end of + * others can be configured. + * + * The basic working of this API is the following: + * + * - When a frame is available on any input, add it using + * ff_framesync_add_frame(). + * + * - When a frame event is ready to be processed (i.e. after adding a frame + * or when requested on input): + * - call ff_framesync_next(); + * - if fs->frame_ready is true, process the frames; + * - call ff_framesync_drop(). + */ + +/** + * Stream extrapolation mode + * + * Describe how the frames of a stream are extrapolated before the first one + * and after EOF to keep sync with possibly longer other streams. + */ +enum FFFrameSyncExtMode { + /** + * Completely stop all streams with this one. + */ + EXT_STOP, + + /** + * Ignore this stream and continue processing the other ones. + */ + EXT_NULL, + + /** + * Extend the frame to infinity. + */ + EXT_INFINITY, +}; + +/** + * Input stream structure + */ +typedef struct FFFrameSyncIn { + + /** + * Queue of incoming AVFrame, and NULL to mark EOF + */ + struct FFBufQueue queue; + + /** + * Extrapolation mode for timestamps before the first frame + */ + enum FFFrameSyncExtMode before; + + /** + * Extrapolation mode for timestamps after the last frame + */ + enum FFFrameSyncExtMode after; + + /** + * Time base for the incoming frames + */ + AVRational time_base; + + /** + * Current frame, may be NULL before the first one or after EOF + */ + AVFrame *frame; + + /** + * Next frame, for internal use + */ + AVFrame *frame_next; + + /** + * PTS of the current frame + */ + int64_t pts; + + /** + * PTS of the next frame, for internal use + */ + int64_t pts_next; + + /** + * Boolean flagging the next frame, for internal use + */ + uint8_t have_next; + + /** + * State: before first, in stream or after EOF, for internal use + */ + uint8_t state; + + /** + * Synchronization flag: if not zero, then frames on this stream cause + * frames on the output. + */ + unsigned sync; + +} FFFrameSyncIn; + +/** + * Frame sync structure. + */ +typedef struct FFFrameSync { + const AVClass *class; + void *parent; + + /** + * Number of input streams + */ + unsigned nb_in; + + /** + * Time base for the output events + */ + AVRational time_base; + + /** + * Timestamp of the current event + */ + int64_t pts; + + /** + * Index of the input that requires a request + */ + unsigned in_request; + + /** + * Flag indicating that a frame event is ready + */ + uint8_t frame_ready; + + /** + * Flag indicating that output has reached EOF. + */ + uint8_t eof; + + /** + * Array of inputs; all inputs must be in consecutive memory + */ + FFFrameSyncIn in[1]; /* must be last field */ +} FFFrameSync; + +/** + * Initialize a frame sync structure. + * + * The entire structure is expected to be already set to 0. + * + * @param fs frame sync structure to initialize + * @param parent parent object, used for logging + * @param nb_in number of inputs + */ +void ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in); + +/** + * Configure a frame sync structure. + * + * Must be called after all options are set but before all use. + * + * @return >= 0 for success or a negative error code + */ +int ff_framesync_configure(FFFrameSync *fs); + +/** + * Add a frame to an input + * + * Typically called from the filter_frame() method. + * + * @param fs frame sync structure + * @param in index of the input + * @param frame input frame, or NULL for EOF + */ +int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame); + +/** + * Prepare the next frame event. + * + * The status of the operation can be found in fs->frame_ready and fs->eof. + */ +void ff_framesync_next(FFFrameSync *fs); + +/** + * Drop the current frame event. + */ +void ff_framesync_drop(FFFrameSync *fs); + +/** + * Get the current frame in an input. + * + * @param fs frame sync structure + * @param in index of the input + * @param rframe used to return the current frame (or NULL) + * @param get if not zero, the calling code needs to get ownership of + * the returned frame; the current frame will either be + * duplicated or removed from the framesync structure + */ +int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, + unsigned get); + +#endif /* AVFILTER_FRAMESYNC_H */ -- 1.7.10.4
Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/Makefile | 8 +-- libavfilter/dualinput.c | 127 +++++++++++++++++++--------------------------- libavfilter/dualinput.h | 5 ++ libavfilter/vf_overlay.c | 5 ++ 4 files changed, 67 insertions(+), 78 deletions(-) Also TODO: add the init function call to other filers than overlay, and rework the filters to use framesync directly, so they can benefit from the new features. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 3751d54..f2d1c1b 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -111,7 +111,7 @@ OBJS-$(CONFIG_ALPHAMERGE_FILTER) += vf_alphamerge.o OBJS-$(CONFIG_BBOX_FILTER) += bbox.o vf_bbox.o OBJS-$(CONFIG_BLACKDETECT_FILTER) += vf_blackdetect.o OBJS-$(CONFIG_BLACKFRAME_FILTER) += vf_blackframe.o -OBJS-$(CONFIG_BLEND_FILTER) += vf_blend.o dualinput.o +OBJS-$(CONFIG_BLEND_FILTER) += vf_blend.o dualinput.o framesync.o OBJS-$(CONFIG_BOXBLUR_FILTER) += vf_boxblur.o OBJS-$(CONFIG_COLORBALANCE_FILTER) += vf_colorbalance.o OBJS-$(CONFIG_COLORCHANNELMIXER_FILTER) += vf_colorchannelmixer.o @@ -139,7 +139,7 @@ OBJS-$(CONFIG_FPS_FILTER) += vf_fps.o OBJS-$(CONFIG_FREI0R_FILTER) += vf_frei0r.o OBJS-$(CONFIG_GEQ_FILTER) += vf_geq.o OBJS-$(CONFIG_GRADFUN_FILTER) += vf_gradfun.o -OBJS-$(CONFIG_HALDCLUT_FILTER) += vf_lut3d.o dualinput.o +OBJS-$(CONFIG_HALDCLUT_FILTER) += vf_lut3d.o dualinput.o framesync.o OBJS-$(CONFIG_HFLIP_FILTER) += vf_hflip.o OBJS-$(CONFIG_HISTEQ_FILTER) += vf_histeq.o OBJS-$(CONFIG_HISTOGRAM_FILTER) += vf_histogram.o @@ -163,14 +163,14 @@ OBJS-$(CONFIG_NOISE_FILTER) += vf_noise.o OBJS-$(CONFIG_NULL_FILTER) += vf_null.o OBJS-$(CONFIG_OCV_FILTER) += vf_libopencv.o OBJS-$(CONFIG_OPENCL) += deshake_opencl.o unsharp_opencl.o -OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o dualinput.o +OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o dualinput.o framesync.o OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o OBJS-$(CONFIG_PP_FILTER) += vf_pp.o -OBJS-$(CONFIG_PSNR_FILTER) += vf_psnr.o dualinput.o +OBJS-$(CONFIG_PSNR_FILTER) += vf_psnr.o dualinput.o framesync.o OBJS-$(CONFIG_REMOVELOGO_FILTER) += bbox.o lswsutils.o lavfutils.o vf_removelogo.o OBJS-$(CONFIG_ROTATE_FILTER) += vf_rotate.o OBJS-$(CONFIG_SEPARATEFIELDS_FILTER) += vf_separatefields.o diff --git a/libavfilter/dualinput.c b/libavfilter/dualinput.c index 10e3652..25e4ae0 100644 --- a/libavfilter/dualinput.c +++ b/libavfilter/dualinput.c @@ -16,67 +16,59 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +/* + * TODO + * Clean up obsolete code + * Map shortest to the corresponding flags + * Implement repeatlast + */ + #define MAIN 0 #define SECOND 1 #include "dualinput.h" #include "libavutil/timestamp.h" -static int try_filter_frame(FFDualInputContext *s, - AVFilterContext *ctx, AVFrame *mainpic) +int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s) { - int ret; + FFFrameSyncIn *in = s->fs.in; - /* Discard obsolete second frames: if there is a next second frame with pts - * before the main frame, we can drop the current second. */ - while (1) { - AVFrame *next_overpic = ff_bufqueue_peek(&s->queue[SECOND], 0); - if (!next_overpic && s->second_eof && !s->repeatlast) { - av_frame_free(&s->second_frame); - break; - } - if (!next_overpic || av_compare_ts(next_overpic->pts, ctx->inputs[SECOND]->time_base, - mainpic->pts, ctx->inputs[MAIN]->time_base) > 0) - break; - ff_bufqueue_get(&s->queue[SECOND]); - av_frame_free(&s->second_frame); - s->second_frame = next_overpic; - } + ff_framesync_init(&s->fs, ctx, 2); + in[0].time_base = ctx->inputs[0]->time_base; + in[1].time_base = ctx->inputs[1]->time_base; + in[0].sync = 1; - /* If there is no next frame and no EOF and the second frame is before - * the main frame, we can not know yet if it will be superseded. */ - if (!s->queue[SECOND].available && !s->second_eof && - (!s->second_frame || av_compare_ts(s->second_frame->pts, ctx->inputs[SECOND]->time_base, - mainpic->pts, ctx->inputs[MAIN]->time_base) < 0)) - return AVERROR(EAGAIN); + in[1].after = EXT_NULL; - /* At this point, we know that the current second frame extends to the - * time of the main frame. */ - av_dlog(ctx, "main_pts:%s main_pts_time:%s", - av_ts2str(mainpic->pts), av_ts2timestr(mainpic->pts, &ctx->inputs[MAIN]->time_base)); - if (s->second_frame) - av_dlog(ctx, " second_pts:%s second_pts_time:%s", - av_ts2str(s->second_frame->pts), av_ts2timestr(s->second_frame->pts, &ctx->inputs[SECOND]->time_base)); - av_dlog(ctx, "\n"); - - if (s->second_frame && !ctx->is_disabled) - mainpic = s->process(ctx, mainpic, s->second_frame); - ret = ff_filter_frame(ctx->outputs[0], mainpic); - av_assert1(ret != AVERROR(EAGAIN)); - s->frame_requested = 0; - return ret; + return ff_framesync_configure(&s->fs); } static int try_filter_next_frame(FFDualInputContext *s, AVFilterContext *ctx) { - AVFrame *next_mainpic = ff_bufqueue_peek(&s->queue[MAIN], 0); - int ret; + AVFrame *mainpic = NULL, *secondpic = NULL; + int ret = 0; - if (!next_mainpic) + ff_framesync_next(&s->fs); + if (s->fs.eof) + return AVERROR_EOF; + if (!s->fs.frame_ready) return AVERROR(EAGAIN); - if ((ret = try_filter_frame(s, ctx, next_mainpic)) == AVERROR(EAGAIN)) + if ((ret = ff_framesync_get_frame(&s->fs, 0, &mainpic, 1)) < 0 || + (ret = ff_framesync_get_frame(&s->fs, 1, &secondpic, 0)) < 0) { + av_frame_free(&mainpic); return ret; - ff_bufqueue_get(&s->queue[MAIN]); + } + if (!mainpic) { /* FIXME temporary */ + av_log(0, 16, "null filtering with secondpic %p %ld\n", secondpic, secondpic ? secondpic->pts : 666666); + ff_framesync_drop(&s->fs); + return 0; + } + if (secondpic && !ctx->is_disabled) + mainpic = s->process(ctx, mainpic, secondpic); + ret = ff_filter_frame(ctx->outputs[0], mainpic); + av_assert1(ret != AVERROR(EAGAIN)); + s->frame_requested = 0; + ff_framesync_drop(&s->fs); return ret; } @@ -88,38 +80,29 @@ static int flush_frames(FFDualInputContext *s, AVFilterContext *ctx) return ret == AVERROR(EAGAIN) ? 0 : ret; } -int ff_dualinput_filter_frame_main(FFDualInputContext *s, - AVFilterLink *inlink, AVFrame *in) +static int dualinput_filter_frame_common(FFDualInputContext *s, unsigned in_no, + AVFilterLink *inlink, AVFrame *in) { AVFilterContext *ctx = inlink->dst; int ret; if ((ret = flush_frames(s, ctx)) < 0) return ret; - if ((ret = try_filter_frame(s, ctx, in)) < 0) { - if (ret != AVERROR(EAGAIN)) - return ret; - ff_bufqueue_add(ctx, &s->queue[MAIN], in); - } - - if (!s->second_frame) - return 0; - flush_frames(s, ctx); + if ((ret = ff_framesync_add_frame(&s->fs, in_no, in)) < 0) + return ret; + return flush_frames(s, ctx); +} - return 0; +int ff_dualinput_filter_frame_main(FFDualInputContext *s, + AVFilterLink *inlink, AVFrame *in) +{ + return dualinput_filter_frame_common(s, 0, inlink, in); } int ff_dualinput_filter_frame_second(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in) { - AVFilterContext *ctx = inlink->dst; - int ret; - - if ((ret = flush_frames(s, ctx)) < 0) - return ret; - ff_bufqueue_add(ctx, &s->queue[SECOND], in); - ret = try_filter_next_frame(s, ctx); - return ret == AVERROR(EAGAIN) ? 0 : ret; + return dualinput_filter_frame_common(s, 1, inlink, in); } int ff_dualinput_request_frame(FFDualInputContext *s, AVFilterLink *outlink) @@ -131,19 +114,15 @@ int ff_dualinput_request_frame(FFDualInputContext *s, AVFilterLink *outlink) return 0; s->frame_requested = 1; while (s->frame_requested) { - /* TODO if we had a frame duration, we could guess more accurately */ - input = !s->second_eof && (s->queue[MAIN].available || - s->queue[SECOND].available < 2) ? - SECOND : MAIN; + input = s->fs.in_request; ret = ff_request_frame(ctx->inputs[input]); - /* EOF on main is reported immediately */ - if (ret == AVERROR_EOF && input == SECOND) { - s->second_eof = 1; - if (s->shortest) + if (ret == AVERROR_EOF) { + if ((ret = ff_framesync_add_frame(&s->fs, input, NULL)) < 0) return ret; - if ((ret = try_filter_next_frame(s, ctx)) != AVERROR(EAGAIN)) + if ((ret = try_filter_next_frame(s, ctx)) < 0 && + ret != AVERROR(EAGAIN)) return ret; - ret = 0; /* continue requesting frames on main */ + ret = 0; } if (ret < 0) return ret; diff --git a/libavfilter/dualinput.h b/libavfilter/dualinput.h index 98d0544..5893f13 100644 --- a/libavfilter/dualinput.h +++ b/libavfilter/dualinput.h @@ -26,9 +26,13 @@ #include <stdint.h> #include "bufferqueue.h" +#include "framesync.h" #include "internal.h" typedef struct { + FFFrameSync fs; + FFFrameSyncIn second_input; /* must be immediately after fs */ + uint8_t frame_requested; uint8_t second_eof; AVFrame *second_frame; @@ -38,6 +42,7 @@ typedef struct { int repeatlast; ///< repeat last second frame } FFDualInputContext; +int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s); int ff_dualinput_filter_frame_main(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in); int ff_dualinput_filter_frame_second(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in); int ff_dualinput_request_frame(FFDualInputContext *s, AVFilterLink *outlink); diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c index 7d93c9e..d838594 100644 --- a/libavfilter/vf_overlay.c +++ b/libavfilter/vf_overlay.c @@ -310,6 +310,11 @@ static int config_input_overlay(AVFilterLink *inlink) static int config_output(AVFilterLink *outlink) { AVFilterContext *ctx = outlink->src; + OverlayContext *s = ctx->priv; + int ret; + + if ((ret = ff_dualinput_init(ctx, &s->dinput)) < 0) + return ret; outlink->w = ctx->inputs[MAIN]->w; outlink->h = ctx->inputs[MAIN]->h; -- 1.7.10.4
On 8/27/13, Nicolas George <nicolas.george@normalesup.org> wrote:
Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/Makefile | 8 +-- libavfilter/dualinput.c | 127 +++++++++++++++++++--------------------------- libavfilter/dualinput.h | 5 ++ libavfilter/vf_overlay.c | 5 ++ 4 files changed, 67 insertions(+), 78 deletions(-)
Also TODO: add the init function call to other filers than overlay, and rework the filters to use framesync directly, so they can benefit from the new features.
I would like to apply this 2 because they fix at least 2 bugs.
Le primidi 1er vendémiaire, an CCXXII, Paul B Mahol a écrit :
Also TODO: add the init function call to other filers than overlay, and rework the filters to use framesync directly, so they can benefit from the new features. I would like to apply this 2 because they fix at least 2 bugs.
I have pushed the following patches to my tree: 53319d5 lavfi: add an API to synchronize multiple video inputs. 231e501 lavfi/dualinput: reimplement on top of framesync. I ran FATE with the rebased version; I believe Michael can apply it. Note that the "add the init function call to other filers than overlay" TODO is done. The rest is independent. Regards, -- Nicolas George
On Mon, Sep 23, 2013 at 09:55:54AM +0200, Nicolas George wrote:
Le primidi 1er vendémiaire, an CCXXII, Paul B Mahol a écrit :
Also TODO: add the init function call to other filers than overlay, and rework the filters to use framesync directly, so they can benefit from the new features. I would like to apply this 2 because they fix at least 2 bugs.
I have pushed the following patches to my tree:
53319d5 lavfi: add an API to synchronize multiple video inputs. 231e501 lavfi/dualinput: reimplement on top of framesync.
I ran FATE with the rebased version; I believe Michael can apply it.
merged thanks [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB Why not whip the teacher when the pupil misbehaves? -- Diogenes of Sinope
On 8/27/13, Nicolas George <nicolas.george@normalesup.org> wrote:
Compared to dualinput, this API can handle more than two inputs and can generate frames synchronized to any or all input streams.
Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/framesync.c | 222 +++++++++++++++++++++++++++++++++++++++++++ libavfilter/framesync.h | 242 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 464 insertions(+) create mode 100644 libavfilter/framesync.c create mode 100644 libavfilter/framesync.h
Note: this is work in progress, but I do not know when I will be able to work on it again.
It came as a reaction to mail on the user mailing list: how to overlay a video on top of a constant background. The problem is that the overlay filter, and the dualinput API that was extracted from it to serve for other filters, can only synchronize on the background video: since the background is a constant image, it does not move. If the video on top has constant frame rate, the problem can be solved by setting vf_fps on the background. Otherwise, the suggested solution was to make a sandwich: overlay the completely opaque background on top of the video to get the background with the video timestamps, and then overlay the video. That works (except if the background itself has alpha), but that is ugly and inefficient.
Why adding filter that just repeats frame is (still) ignored? Btw, I really like multiple input support feature and I'm interested in this feature being applied.
The obvious quick-and-dirty solution would be to add sync=second option to the dualinput API, but that is just that: quick and dirty, it does not cover other uses: combining R, G, B input planes (three inputs), computing the PSNR of videos with the same frame rate but different skipped frames (must generate an event for frames on any input).
This API aims to handle all the cases.
[...]
Le primidi 11 fructidor, an CCXXI, Paul B Mahol a écrit :
Why adding filter that just repeats frame is (still) ignored?
I do not understand what you mean exactly here. There is vf_fps, of course, but it will not synchronize on another stream.
Btw, I really like multiple input support feature and I'm interested in this feature being applied.
The patch series has progressed to the point it is almost a drop-in replacement for dualinput (it needs an additional _init() call) and the corresponding filters pass FATE (assuming they all are covered). The next step would be to make the filters use the new API directly so they can benefit from the new features. Is anyone interested in the patch as is? Regards, -- Nicolas George
On 9/4/13, Nicolas George <nicolas.george@normalesup.org> wrote:
Le primidi 11 fructidor, an CCXXI, Paul B Mahol a ecrit :
Why adding filter that just repeats frame is (still) ignored?
I do not understand what you mean exactly here. There is vf_fps, of course, but it will not synchronize on another stream.
Btw, I really like multiple input support feature and I'm interested in this feature being applied.
The patch series has progressed to the point it is almost a drop-in replacement for dualinput (it needs an additional _init() call) and the corresponding filters pass FATE (assuming they all are covered). The next step would be to make the filters use the new API directly so they can benefit from the new features.
Is anyone interested in the patch as is?
I'm interested, waiting for remaining TODOs to be resolved.
On 9/6/13, Paul B Mahol <onemda@gmail.com> wrote:
On 9/4/13, Nicolas George <nicolas.george@normalesup.org> wrote:
Le primidi 11 fructidor, an CCXXI, Paul B Mahol a ecrit :
Why adding filter that just repeats frame is (still) ignored?
I do not understand what you mean exactly here. There is vf_fps, of course, but it will not synchronize on another stream.
Btw, I really like multiple input support feature and I'm interested in this feature being applied.
The patch series has progressed to the point it is almost a drop-in replacement for dualinput (it needs an additional _init() call) and the corresponding filters pass FATE (assuming they all are covered). The next step would be to make the filters use the new API directly so they can benefit from the new features.
Is anyone interested in the patch as is?
I'm interested, waiting for remaining TODOs to be resolved.
So you gonna continue working on this or want to outsource it to someone else?
Le duodi 22 fructidor, an CCXXI, Paul B Mahol a écrit :
So you gonna continue working on this or want to outsource it to someone else?
I have a lot of other things to do these days, but it will settle soon. In the meantime, here is the patch series in its current state. It passes FATE, and I believe the code is slightly simpler, but it does not bring any enhancement yet (except, probably, it fixes the "last frame omitted" bug in shortest mode). Adding the new features should be pretty straightforward, but I believe it belongs to another patch series anyway. Regards, -- Nicolas George
Compared to dualinput, this API can handle more than two inputs and can generate frames synchronized to any or all input streams. Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/framesync.c | 329 ++++++++++++++++++++++++++++++++++++++++++++++++ libavfilter/framesync.h | 296 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 625 insertions(+) create mode 100644 libavfilter/framesync.c create mode 100644 libavfilter/framesync.h diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c new file mode 100644 index 0000000..12db50c --- /dev/null +++ b/libavfilter/framesync.c @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2013 Nicolas George + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/avassert.h" +#include "avfilter.h" +#include "bufferqueue.h" +#include "framesync.h" +#include "internal.h" + +#define OFFSET(member) offsetof(FFFrameSync, member) + +static const char *framesync_name(void *ptr) +{ + return "framesync"; +} + +static const AVClass framesync_class = { + .version = LIBAVUTIL_VERSION_INT, + .class_name = "framesync", + .item_name = framesync_name, + .category = AV_CLASS_CATEGORY_FILTER, + .option = NULL, + .parent_log_context_offset = OFFSET(parent), +}; + +enum { + STATE_BOF, + STATE_RUN, + STATE_EOF, +}; + +void ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in) +{ + fs->class = &framesync_class; + fs->parent = parent; + fs->nb_in = nb_in; +} + +static void framesync_sync_level_update(FFFrameSync *fs) +{ + unsigned i, level = 0; + + for (i = 0; i < fs->nb_in; i++) + if (fs->in[i].state != STATE_EOF) + level = FFMAX(level, fs->in[i].sync); + av_assert0(level <= fs->sync_level); + if (level < fs->sync_level) + av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level); + if (level) + fs->sync_level = level; + else + fs->eof = 1; +} + +int ff_framesync_configure(FFFrameSync *fs) +{ + unsigned i; + int64_t gcd, lcm; + + if (!fs->time_base.num) { + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].sync) { + if (fs->time_base.num) { + gcd = av_gcd(fs->time_base.den, fs->in[i].time_base.den); + lcm = (fs->time_base.den / gcd) * fs->in[i].time_base.den; + if (lcm < AV_TIME_BASE / 2) { + fs->time_base.den = lcm; + fs->time_base.num = av_gcd(fs->time_base.num, + fs->in[i].time_base.num); + } else { + fs->time_base.num = 1; + fs->time_base.den = AV_TIME_BASE; + break; + } + } else { + fs->time_base = fs->in[i].time_base; + } + } + } + if (!fs->time_base.num) { + av_log(fs, AV_LOG_ERROR, "Impossible to set time base\n"); + return AVERROR(EINVAL); + } + av_log(fs, AV_LOG_VERBOSE, "Selected %d/%d time base\n", + fs->time_base.num, fs->time_base.den); + } + + for (i = 0; i < fs->nb_in; i++) + fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->sync_level = UINT_MAX; + framesync_sync_level_update(fs); + + return 0; +} + +static void framesync_advance(FFFrameSync *fs) +{ + int latest; + unsigned i; + int64_t pts; + + if (fs->eof) + return; + while (!fs->frame_ready) { + latest = -1; + for (i = 0; i < fs->nb_in; i++) { + if (!fs->in[i].have_next) { + if (latest < 0 || fs->in[i].pts < fs->in[latest].pts) + latest = i; + } + } + if (latest >= 0) { + fs->in_request = latest; + break; + } + + pts = fs->in[0].pts_next; + for (i = 1; i < fs->nb_in; i++) + if (fs->in[i].pts_next < pts) + pts = fs->in[i].pts_next; + if (pts == INT64_MAX) { + fs->eof = 1; + break; + } + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].pts_next == pts || + (fs->in[i].before == EXT_INFINITY && + fs->in[i].state == STATE_BOF)) { + av_frame_free(&fs->in[i].frame); + fs->in[i].frame = fs->in[i].frame_next; + fs->in[i].pts = fs->in[i].pts_next; + fs->in[i].frame_next = NULL; + fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->in[i].have_next = 0; + fs->in[i].state = fs->in[i].frame ? STATE_RUN : STATE_EOF; + if (fs->in[i].sync == fs->sync_level && fs->in[i].frame) + fs->frame_ready = 1; + if (fs->in[i].state == STATE_EOF && + fs->in[i].after == EXT_STOP) + fs->eof = 1; + } + } + if (fs->eof) + fs->frame_ready = 0; + if (fs->frame_ready) + for (i = 0; i < fs->nb_in; i++) + if ((fs->in[i].state == STATE_BOF && + fs->in[i].before == EXT_STOP)) + fs->frame_ready = 0; + fs->pts = pts; + } +} + +static int64_t framesync_pts_extrapolate(FFFrameSync *fs, unsigned in, + int64_t pts) +{ + /* Possible enhancement: use the link's frame rate */ + return pts + 1; +} + +static void framesync_inject_frame(FFFrameSync *fs, unsigned in, AVFrame *frame) +{ + int64_t pts; + + av_assert0(!fs->in[in].have_next); + if (frame) { + pts = av_rescale_q(frame->pts, fs->in[in].time_base, fs->time_base); + frame->pts = pts; + } else { + pts = fs->in[in].state != STATE_RUN || fs->in[in].after == EXT_INFINITY + ? INT64_MAX : framesync_pts_extrapolate(fs, in, fs->in[in].pts); + fs->in[in].sync = 0; + framesync_sync_level_update(fs); + } + fs->in[in].frame_next = frame; + fs->in[in].pts_next = pts; + fs->in[in].have_next = 1; +} + +int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame) +{ + av_assert1(in < fs->nb_in); + if (!fs->in[in].have_next) + framesync_inject_frame(fs, in, frame); + else + ff_bufqueue_add(fs, &fs->in[in].queue, frame); + return 0; +} + +void ff_framesync_next(FFFrameSync *fs) +{ + unsigned i; + + av_assert0(!fs->frame_ready); + for (i = 0; i < fs->nb_in; i++) + if (!fs->in[i].have_next && fs->in[i].queue.available) + framesync_inject_frame(fs, i, ff_bufqueue_get(&fs->in[i].queue)); + fs->frame_ready = 0; + framesync_advance(fs); +} + +void ff_framesync_drop(FFFrameSync *fs) +{ + fs->frame_ready = 0; +} + +int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, + unsigned get) +{ + AVFrame *frame; + unsigned need_copy = 0, i; + int64_t pts_next; + int ret; + + if (!fs->in[in].frame) { + *rframe = NULL; + return 0; + } + frame = fs->in[in].frame; + if (get) { + /* Find out if we need to copy the frame: is there another sync + stream, and do we know if its current frame will outlast this one? */ + pts_next = fs->in[in].have_next ? fs->in[in].pts_next : INT64_MAX; + for (i = 0; i < fs->nb_in && !need_copy; i++) + if (i != in && fs->in[i].sync && + (!fs->in[i].have_next || fs->in[i].pts_next < pts_next)) + need_copy = 1; + if (need_copy) { + if (!(frame = av_frame_clone(frame))) + return AVERROR(ENOMEM); + if ((ret = av_frame_make_writable(frame)) < 0) { + av_frame_free(&frame); + return ret; + } + } else { + fs->in[in].frame = NULL; + } + fs->frame_ready = 0; + } + *rframe = frame; + return 0; +} + +void ff_framesync_uninit(FFFrameSync *fs) +{ + unsigned i; + + for (i = 0; i < fs->nb_in; i++) { + av_frame_free(&fs->in[i].frame); + av_frame_free(&fs->in[i].frame_next); + ff_bufqueue_discard_all(&fs->in[i].queue); + } +} + +int ff_framesync_process_frame(FFFrameSync *fs, unsigned all) +{ + int ret, count = 0; + + av_assert0(fs->on_event); + while (1) { + ff_framesync_next(fs); + if (fs->eof || !fs->frame_ready) + break; + if ((ret = fs->on_event(fs)) < 0) + return ret; + ff_framesync_drop(fs); + count++; + if (!all) + break; + } + if (!count && fs->eof) + return AVERROR_EOF; + return count; +} + +int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink, + AVFrame *in) +{ + int ret; + + if ((ret = ff_framesync_process_frame(fs, 1)) < 0) + return ret; + if ((ret = ff_framesync_add_frame(fs, FF_INLINK_IDX(inlink), in)) < 0) + return ret; + if ((ret = ff_framesync_process_frame(fs, 0)) < 0) + return ret; + return 0; +} + +int ff_framesync_request_frame(FFFrameSync *fs, AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + int input, ret; + + if ((ret = ff_framesync_process_frame(fs, 0)) < 0) + return ret; + if (ret > 0) + return 0; + if (fs->eof) + return AVERROR_EOF; + outlink->flags |= FF_LINK_FLAG_REQUEST_LOOP; + input = fs->in_request; + ret = ff_request_frame(ctx->inputs[input]); + if (ret == AVERROR_EOF) { + if ((ret = ff_framesync_add_frame(fs, input, NULL)) < 0) + return ret; + if ((ret = ff_framesync_process_frame(fs, 0)) < 0) + return ret; + ret = 0; + } + return ret; +} diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h new file mode 100644 index 0000000..2072781 --- /dev/null +++ b/libavfilter/framesync.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2013 Nicolas George + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVFILTER_FRAMESYNC_H +#define AVFILTER_FRAMESYNC_H + +#include "bufferqueue.h" + +/* + * TODO + * Callback-based API similar to dualinput. + * Export convenient options. + */ + +/** + * This API is intended as a helper for filters that have several video + * input and need to combine them somehow. If the inputs have different or + * variable frame rate, getting the input frames to match requires a rather + * complex logic and a few user-tunable options. + * + * In this API, when a set of synchronized input frames is ready to be + * procesed is called a frame event. Frame event can be generated in + * response to input frames on any or all inputs and the handling of + * situations where some stream extend beyond the beginning or the end of + * others can be configured. + * + * The basic working of this API is the following: + * + * - When a frame is available on any input, add it using + * ff_framesync_add_frame(). + * + * - When a frame event is ready to be processed (i.e. after adding a frame + * or when requested on input): + * - call ff_framesync_next(); + * - if fs->frame_ready is true, process the frames; + * - call ff_framesync_drop(). + */ + +/** + * Stream extrapolation mode + * + * Describe how the frames of a stream are extrapolated before the first one + * and after EOF to keep sync with possibly longer other streams. + */ +enum FFFrameSyncExtMode { + + /** + * Completely stop all streams with this one. + */ + EXT_STOP, + + /** + * Ignore this stream and continue processing the other ones. + */ + EXT_NULL, + + /** + * Extend the frame to infinity. + */ + EXT_INFINITY, +}; + +/** + * Input stream structure + */ +typedef struct FFFrameSyncIn { + + /** + * Queue of incoming AVFrame, and NULL to mark EOF + */ + struct FFBufQueue queue; + + /** + * Extrapolation mode for timestamps before the first frame + */ + enum FFFrameSyncExtMode before; + + /** + * Extrapolation mode for timestamps after the last frame + */ + enum FFFrameSyncExtMode after; + + /** + * Time base for the incoming frames + */ + AVRational time_base; + + /** + * Current frame, may be NULL before the first one or after EOF + */ + AVFrame *frame; + + /** + * Next frame, for internal use + */ + AVFrame *frame_next; + + /** + * PTS of the current frame + */ + int64_t pts; + + /** + * PTS of the next frame, for internal use + */ + int64_t pts_next; + + /** + * Boolean flagging the next frame, for internal use + */ + uint8_t have_next; + + /** + * State: before first, in stream or after EOF, for internal use + */ + uint8_t state; + + /** + * Synchronization level: frames on input at the highest sync level will + * generate output frame events. + * + * For example, if inputs #0 and #1 have sync level 2 and input #2 has + * sync level 1, then a frame on either input #0 or #1 will generate a + * frame event, but not a frame on input #2 until both inputs #0 and #1 + * have reached EOF. + * + * If sync is 0, no frame event will be generated. + */ + unsigned sync; + +} FFFrameSyncIn; + +/** + * Frame sync structure. + */ +typedef struct FFFrameSync { + const AVClass *class; + void *parent; + + /** + * Number of input streams + */ + unsigned nb_in; + + /** + * Time base for the output events + */ + AVRational time_base; + + /** + * Timestamp of the current event + */ + int64_t pts; + + /** + * Callback called when a frame event is ready + */ + int (*on_event)(struct FFFrameSync *fs); + + /** + * Opaque pointer, not used by the API + */ + void *opaque; + + /** + * Index of the input that requires a request + */ + unsigned in_request; + + /** + * Synchronization level: only inputs with the same sync level are sync + * sources. + */ + unsigned sync_level; + + /** + * Flag indicating that a frame event is ready + */ + uint8_t frame_ready; + + /** + * Flag indicating that output has reached EOF. + */ + uint8_t eof; + + /** + * Array of inputs; all inputs must be in consecutive memory + */ + FFFrameSyncIn in[1]; /* must be the last field */ + +} FFFrameSync; + +/** + * Initialize a frame sync structure. + * + * The entire structure is expected to be already set to 0. + * + * @param fs frame sync structure to initialize + * @param parent parent object, used for logging + * @param nb_in number of inputs + */ +void ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in); + +/** + * Configure a frame sync structure. + * + * Must be called after all options are set but before all use. + * + * @return >= 0 for success or a negative error code + */ +int ff_framesync_configure(FFFrameSync *fs); + +/** + * Free all memory currently allocated. + */ +void ff_framesync_uninit(FFFrameSync *fs); + +/** + * Add a frame to an input + * + * Typically called from the filter_frame() method. + * + * @param fs frame sync structure + * @param in index of the input + * @param frame input frame, or NULL for EOF + */ +int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame); + +/** + * Prepare the next frame event. + * + * The status of the operation can be found in fs->frame_ready and fs->eof. + */ +void ff_framesync_next(FFFrameSync *fs); + +/** + * Drop the current frame event. + */ +void ff_framesync_drop(FFFrameSync *fs); + +/** + * Get the current frame in an input. + * + * @param fs frame sync structure + * @param in index of the input + * @param rframe used to return the current frame (or NULL) + * @param get if not zero, the calling code needs to get ownership of + * the returned frame; the current frame will either be + * duplicated or removed from the framesync structure + */ +int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, + unsigned get); + +/** + * Process one or several frame using the on_event callback. + * + * @return number of frames processed or negative error code + */ +int ff_framesync_process_frame(FFFrameSync *fs, unsigned all); + + +/** + * Accept a frame on a filter input. + * + * This function can be the complete implementation of all filter_frame + * methods of a filter using framesync. + */ +int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink, + AVFrame *in); + +/** + * Request a frame on the filter output. + * + * This function can be the complete implementation of all filter_frame + * methods of a filter using framesync if it has only one output. + */ +int ff_framesync_request_frame(FFFrameSync *fs, AVFilterLink *outlink); + +#endif /* AVFILTER_FRAMESYNC_H */ -- 1.8.4.rc3
Signed-off-by: Nicolas George <nicolas.george@normalesup.org> --- libavfilter/Makefile | 8 +-- libavfilter/dualinput.c | 149 +++++++++++++---------------------------------- libavfilter/dualinput.h | 8 ++- libavfilter/vf_blend.c | 4 ++ libavfilter/vf_lut3d.c | 4 ++ libavfilter/vf_overlay.c | 5 ++ libavfilter/vf_psnr.c | 4 ++ 7 files changed, 66 insertions(+), 116 deletions(-) diff --git a/libavfilter/Makefile b/libavfilter/Makefile index b57d4c9..c9d8027 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -111,7 +111,7 @@ OBJS-$(CONFIG_ALPHAMERGE_FILTER) += vf_alphamerge.o OBJS-$(CONFIG_BBOX_FILTER) += bbox.o vf_bbox.o OBJS-$(CONFIG_BLACKDETECT_FILTER) += vf_blackdetect.o OBJS-$(CONFIG_BLACKFRAME_FILTER) += vf_blackframe.o -OBJS-$(CONFIG_BLEND_FILTER) += vf_blend.o dualinput.o +OBJS-$(CONFIG_BLEND_FILTER) += vf_blend.o dualinput.o framesync.o OBJS-$(CONFIG_BOXBLUR_FILTER) += vf_boxblur.o OBJS-$(CONFIG_COLORBALANCE_FILTER) += vf_colorbalance.o OBJS-$(CONFIG_COLORCHANNELMIXER_FILTER) += vf_colorchannelmixer.o @@ -139,7 +139,7 @@ OBJS-$(CONFIG_FPS_FILTER) += vf_fps.o OBJS-$(CONFIG_FREI0R_FILTER) += vf_frei0r.o OBJS-$(CONFIG_GEQ_FILTER) += vf_geq.o OBJS-$(CONFIG_GRADFUN_FILTER) += vf_gradfun.o -OBJS-$(CONFIG_HALDCLUT_FILTER) += vf_lut3d.o dualinput.o +OBJS-$(CONFIG_HALDCLUT_FILTER) += vf_lut3d.o dualinput.o framesync.o OBJS-$(CONFIG_HFLIP_FILTER) += vf_hflip.o OBJS-$(CONFIG_HISTEQ_FILTER) += vf_histeq.o OBJS-$(CONFIG_HISTOGRAM_FILTER) += vf_histogram.o @@ -163,7 +163,7 @@ OBJS-$(CONFIG_NOISE_FILTER) += vf_noise.o OBJS-$(CONFIG_NULL_FILTER) += vf_null.o OBJS-$(CONFIG_OCV_FILTER) += vf_libopencv.o OBJS-$(CONFIG_OPENCL) += deshake_opencl.o unsharp_opencl.o -OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o dualinput.o +OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o dualinput.o framesync.o OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o @@ -171,7 +171,7 @@ OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o OBJS-$(CONFIG_PP_FILTER) += vf_pp.o -OBJS-$(CONFIG_PSNR_FILTER) += vf_psnr.o dualinput.o +OBJS-$(CONFIG_PSNR_FILTER) += vf_psnr.o dualinput.o framesync.o OBJS-$(CONFIG_REMOVELOGO_FILTER) += bbox.o lswsutils.o lavfutils.o vf_removelogo.o OBJS-$(CONFIG_ROTATE_FILTER) += vf_rotate.o OBJS-$(CONFIG_SEPARATEFIELDS_FILTER) += vf_separatefields.o diff --git a/libavfilter/dualinput.c b/libavfilter/dualinput.c index 10e3652..179c3ef 100644 --- a/libavfilter/dualinput.c +++ b/libavfilter/dualinput.c @@ -16,144 +16,75 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#define MAIN 0 -#define SECOND 1 - #include "dualinput.h" #include "libavutil/timestamp.h" -static int try_filter_frame(FFDualInputContext *s, - AVFilterContext *ctx, AVFrame *mainpic) +static int process_frame(FFFrameSync *fs) { - int ret; - - /* Discard obsolete second frames: if there is a next second frame with pts - * before the main frame, we can drop the current second. */ - while (1) { - AVFrame *next_overpic = ff_bufqueue_peek(&s->queue[SECOND], 0); - if (!next_overpic && s->second_eof && !s->repeatlast) { - av_frame_free(&s->second_frame); - break; - } - if (!next_overpic || av_compare_ts(next_overpic->pts, ctx->inputs[SECOND]->time_base, - mainpic->pts, ctx->inputs[MAIN]->time_base) > 0) - break; - ff_bufqueue_get(&s->queue[SECOND]); - av_frame_free(&s->second_frame); - s->second_frame = next_overpic; + AVFilterContext *ctx = fs->parent; + FFDualInputContext *s = fs->opaque; + AVFrame *mainpic = NULL, *secondpic = NULL; + int ret = 0; + + if ((ret = ff_framesync_get_frame(&s->fs, 0, &mainpic, 1)) < 0 || + (ret = ff_framesync_get_frame(&s->fs, 1, &secondpic, 0)) < 0) { + av_frame_free(&mainpic); + return ret; } - - /* If there is no next frame and no EOF and the second frame is before - * the main frame, we can not know yet if it will be superseded. */ - if (!s->queue[SECOND].available && !s->second_eof && - (!s->second_frame || av_compare_ts(s->second_frame->pts, ctx->inputs[SECOND]->time_base, - mainpic->pts, ctx->inputs[MAIN]->time_base) < 0)) - return AVERROR(EAGAIN); - - /* At this point, we know that the current second frame extends to the - * time of the main frame. */ - av_dlog(ctx, "main_pts:%s main_pts_time:%s", - av_ts2str(mainpic->pts), av_ts2timestr(mainpic->pts, &ctx->inputs[MAIN]->time_base)); - if (s->second_frame) - av_dlog(ctx, " second_pts:%s second_pts_time:%s", - av_ts2str(s->second_frame->pts), av_ts2timestr(s->second_frame->pts, &ctx->inputs[SECOND]->time_base)); - av_dlog(ctx, "\n"); - - if (s->second_frame && !ctx->is_disabled) - mainpic = s->process(ctx, mainpic, s->second_frame); + av_assert0(mainpic); + mainpic->pts = av_rescale_q(mainpic->pts, s->fs.time_base, ctx->outputs[0]->time_base); + if (secondpic && !ctx->is_disabled) + mainpic = s->process(ctx, mainpic, secondpic); ret = ff_filter_frame(ctx->outputs[0], mainpic); av_assert1(ret != AVERROR(EAGAIN)); s->frame_requested = 0; return ret; } -static int try_filter_next_frame(FFDualInputContext *s, AVFilterContext *ctx) -{ - AVFrame *next_mainpic = ff_bufqueue_peek(&s->queue[MAIN], 0); - int ret; - - if (!next_mainpic) - return AVERROR(EAGAIN); - if ((ret = try_filter_frame(s, ctx, next_mainpic)) == AVERROR(EAGAIN)) - return ret; - ff_bufqueue_get(&s->queue[MAIN]); - return ret; -} - -static int flush_frames(FFDualInputContext *s, AVFilterContext *ctx) +int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s) { - int ret; + FFFrameSyncIn *in = s->fs.in; + + ff_framesync_init(&s->fs, ctx, 2); + s->fs.opaque = s; + s->fs.on_event = process_frame; + in[0].time_base = ctx->inputs[0]->time_base; + in[1].time_base = ctx->inputs[1]->time_base; + in[0].sync = 2; + in[0].before = EXT_STOP; + in[0].after = EXT_INFINITY; + in[1].sync = 1; + in[1].before = EXT_NULL; + in[1].after = EXT_INFINITY; + + if (s->shortest) + in[1].after = EXT_STOP; + if (!s->repeatlast) { + in[0].after = EXT_STOP; + in[1].sync = 0; + } - while (!(ret = try_filter_next_frame(s, ctx))); - return ret == AVERROR(EAGAIN) ? 0 : ret; + return ff_framesync_configure(&s->fs); } int ff_dualinput_filter_frame_main(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in) { - AVFilterContext *ctx = inlink->dst; - int ret; - - if ((ret = flush_frames(s, ctx)) < 0) - return ret; - if ((ret = try_filter_frame(s, ctx, in)) < 0) { - if (ret != AVERROR(EAGAIN)) - return ret; - ff_bufqueue_add(ctx, &s->queue[MAIN], in); - } - - if (!s->second_frame) - return 0; - flush_frames(s, ctx); - - return 0; + return ff_framesync_filter_frame(&s->fs, inlink, in); } int ff_dualinput_filter_frame_second(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in) { - AVFilterContext *ctx = inlink->dst; - int ret; - - if ((ret = flush_frames(s, ctx)) < 0) - return ret; - ff_bufqueue_add(ctx, &s->queue[SECOND], in); - ret = try_filter_next_frame(s, ctx); - return ret == AVERROR(EAGAIN) ? 0 : ret; + return ff_framesync_filter_frame(&s->fs, inlink, in); } int ff_dualinput_request_frame(FFDualInputContext *s, AVFilterLink *outlink) { - AVFilterContext *ctx = outlink->src; - int input, ret; - - if (!try_filter_next_frame(s, ctx)) - return 0; - s->frame_requested = 1; - while (s->frame_requested) { - /* TODO if we had a frame duration, we could guess more accurately */ - input = !s->second_eof && (s->queue[MAIN].available || - s->queue[SECOND].available < 2) ? - SECOND : MAIN; - ret = ff_request_frame(ctx->inputs[input]); - /* EOF on main is reported immediately */ - if (ret == AVERROR_EOF && input == SECOND) { - s->second_eof = 1; - if (s->shortest) - return ret; - if ((ret = try_filter_next_frame(s, ctx)) != AVERROR(EAGAIN)) - return ret; - ret = 0; /* continue requesting frames on main */ - } - if (ret < 0) - return ret; - } - return 0; + return ff_framesync_request_frame(&s->fs, outlink); } void ff_dualinput_uninit(FFDualInputContext *s) { - av_frame_free(&s->second_frame); - ff_bufqueue_discard_all(&s->queue[MAIN]); - ff_bufqueue_discard_all(&s->queue[SECOND]); + ff_framesync_uninit(&s->fs); } diff --git a/libavfilter/dualinput.h b/libavfilter/dualinput.h index 98d0544..aaefd78 100644 --- a/libavfilter/dualinput.h +++ b/libavfilter/dualinput.h @@ -26,18 +26,20 @@ #include <stdint.h> #include "bufferqueue.h" +#include "framesync.h" #include "internal.h" typedef struct { + FFFrameSync fs; + FFFrameSyncIn second_input; /* must be immediately after fs */ + uint8_t frame_requested; - uint8_t second_eof; - AVFrame *second_frame; - struct FFBufQueue queue[2]; AVFrame *(*process)(AVFilterContext *ctx, AVFrame *main, const AVFrame *second); int shortest; ///< terminate stream when the second input terminates int repeatlast; ///< repeat last second frame } FFDualInputContext; +int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s); int ff_dualinput_filter_frame_main(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in); int ff_dualinput_filter_frame_second(FFDualInputContext *s, AVFilterLink *inlink, AVFrame *in); int ff_dualinput_request_frame(FFDualInputContext *s, AVFilterLink *outlink); diff --git a/libavfilter/vf_blend.c b/libavfilter/vf_blend.c index 959fcad..f0552c8 100644 --- a/libavfilter/vf_blend.c +++ b/libavfilter/vf_blend.c @@ -368,6 +368,7 @@ static int config_output(AVFilterLink *outlink) AVFilterLink *bottomlink = ctx->inputs[BOTTOM]; BlendContext *b = ctx->priv; const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(toplink->format); + int ret; if (toplink->format != bottomlink->format) { av_log(ctx, AV_LOG_ERROR, "inputs must be of same pixel format\n"); @@ -399,6 +400,9 @@ static int config_output(AVFilterLink *outlink) b->vsub = pix_desc->log2_chroma_h; b->nb_planes = av_pix_fmt_count_planes(toplink->format); + if ((ret = ff_dualinput_init(ctx, &b->dinput)) < 0) + return ret; + return 0; } diff --git a/libavfilter/vf_lut3d.c b/libavfilter/vf_lut3d.c index 48002b8..dd7f381 100644 --- a/libavfilter/vf_lut3d.c +++ b/libavfilter/vf_lut3d.c @@ -656,10 +656,14 @@ static void update_clut(LUT3DContext *lut3d, const AVFrame *frame) static int config_output(AVFilterLink *outlink) { AVFilterContext *ctx = outlink->src; + LUT3DContext *lut3d = ctx->priv; + int ret; outlink->w = ctx->inputs[0]->w; outlink->h = ctx->inputs[0]->h; outlink->time_base = ctx->inputs[0]->time_base; + if ((ret = ff_dualinput_init(ctx, &lut3d->dinput)) < 0) + return ret; return 0; } diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c index f3e7891..e4aaa5f 100644 --- a/libavfilter/vf_overlay.c +++ b/libavfilter/vf_overlay.c @@ -310,6 +310,11 @@ static int config_input_overlay(AVFilterLink *inlink) static int config_output(AVFilterLink *outlink) { AVFilterContext *ctx = outlink->src; + OverlayContext *s = ctx->priv; + int ret; + + if ((ret = ff_dualinput_init(ctx, &s->dinput)) < 0) + return ret; outlink->w = ctx->inputs[MAIN]->w; outlink->h = ctx->inputs[MAIN]->h; diff --git a/libavfilter/vf_psnr.c b/libavfilter/vf_psnr.c index ef638ff..8f33141 100644 --- a/libavfilter/vf_psnr.c +++ b/libavfilter/vf_psnr.c @@ -255,13 +255,17 @@ static int config_input_ref(AVFilterLink *inlink) static int config_output(AVFilterLink *outlink) { AVFilterContext *ctx = outlink->src; + PSNRContext *s = ctx->priv; AVFilterLink *mainlink = ctx->inputs[0]; + int ret; outlink->w = mainlink->w; outlink->h = mainlink->h; outlink->time_base = mainlink->time_base; outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio; outlink->frame_rate = mainlink->frame_rate; + if ((ret = ff_dualinput_init(ctx, &s->dinput)) < 0) + return ret; return 0; } -- 1.8.4.rc3
participants (4)
-
Michael Niedermayer -
Nicolas George -
Nicolas George -
Paul B Mahol