[PATCH 01/35] fftools/ffmpeg_mux: add private muxer context
Move header_written into it, which is not (and should not be) used by any code outside of ffmpeg_mux. In the future this context will contain more muxer-private state that should not be visible to other code. --- fftools/ffmpeg.h | 6 ++++-- fftools/ffmpeg_mux.c | 26 ++++++++++++++++++++++---- fftools/ffmpeg_opt.c | 6 ++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 69a368b8d1..c6ffe9fdf6 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -581,9 +581,12 @@ typedef struct OutputStream { int64_t error[4]; } OutputStream; +typedef struct Muxer Muxer; + typedef struct OutputFile { int index; + Muxer *mux; const AVOutputFormat *format; AVFormatContext *ctx; @@ -594,8 +597,6 @@ typedef struct OutputFile { uint64_t limit_filesize; /* filesize limit expressed in bytes */ int shortest; - - int header_written; } OutputFile; extern InputStream **input_streams; @@ -694,6 +695,7 @@ int hw_device_setup_for_filter(FilterGraph *fg); int hwaccel_decode_init(AVCodecContext *avctx); +int of_muxer_init(OutputFile *of); /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index a55fd18f8f..e47a55c4e9 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -32,6 +32,10 @@ #include "libavformat/avformat.h" #include "libavformat/avio.h" +struct Muxer { + int header_written; +}; + static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, OSTFinished others) { int i; @@ -64,7 +68,7 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, ost->frame_number++; } - if (!of->header_written) { + if (!of->mux->header_written) { AVPacket *tmp_pkt; /* the muxer is not initialized yet, buffer the packet */ if (!av_fifo_can_write(ost->muxing_queue)) { @@ -182,7 +186,7 @@ static int print_sdp(void) AVFormatContext **avc; for (i = 0; i < nb_output_files; i++) { - if (!output_files[i]->header_written) + if (!output_files[i]->mux->header_written) return 0; } @@ -246,7 +250,7 @@ int of_check_init(OutputFile *of) return ret; } //assert_avoptions(of->opts); - of->header_written = 1; + of->mux->header_written = 1; av_dump_format(of->ctx, of->index, of->ctx->url, 1); nb_output_dumped++; @@ -282,7 +286,7 @@ int of_write_trailer(OutputFile *of) { int ret; - if (!of->header_written) { + if (!of->mux->header_written) { av_log(NULL, AV_LOG_ERROR, "Nothing was written into output file %d (%s), because " "at least one of its streams received no packets.\n", @@ -313,5 +317,19 @@ void of_close(OutputFile **pof) avformat_free_context(s); av_dict_free(&of->opts); + av_freep(&of->mux); + av_freep(pof); } + +int of_muxer_init(OutputFile *of) +{ + Muxer *mux = av_mallocz(sizeof(*mux)); + + if (!mux) + return AVERROR(ENOMEM); + + of->mux = mux; + + return 0; +} diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 398067da96..ab8d307c3c 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2947,6 +2947,12 @@ loop_end: exit_program(1); } + err = of_muxer_init(of); + if (err < 0) { + av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); + exit_program(1); + } + return 0; } -- 2.34.1
Stop accessing muxer internals from outside of ffmpeg_mux. --- fftools/ffmpeg.c | 10 +--------- fftools/ffmpeg.h | 1 + fftools/ffmpeg_mux.c | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 09caa3e3c4..5763f231c6 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1508,8 +1508,7 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti { AVBPrint buf, buf_script; OutputStream *ost; - AVFormatContext *oc; - int64_t total_size; + int64_t total_size = of_filesize(output_files[0]); AVCodecContext *enc; int frame_number, vid, i; double bitrate; @@ -1538,13 +1537,6 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti t = (cur_time-timer_start) / 1000000.0; - - oc = output_files[0]->ctx; - - total_size = avio_size(oc->pb); - if (total_size <= 0) // FIXME improve avio_size() so it works with non seekable output too - total_size = avio_tell(oc->pb); - vid = 0; av_bprint_init(&buf, 0, AV_BPRINT_SIZE_AUTOMATIC); av_bprint_init(&buf_script, 0, AV_BPRINT_SIZE_AUTOMATIC); diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index c6ffe9fdf6..10a51db469 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -703,5 +703,6 @@ void of_close(OutputFile **pof); void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int unqueue); +int64_t of_filesize(OutputFile *of); #endif /* FFTOOLS_FFMPEG_H */ diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index e47a55c4e9..83558f7f7d 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -333,3 +333,17 @@ int of_muxer_init(OutputFile *of) return 0; } + +int64_t of_filesize(OutputFile *of) +{ + AVIOContext *pb = of->ctx->pb; + int64_t ret = -1; + + if (pb) { + ret = avio_size(pb); + if (ret <= 0) // FIXME improve avio_size() so it works with non seekable output too + ret = avio_tell(pb); + } + + return ret; +} -- 2.34.1
The option is parsed as INT64 (signed). It is also compared to the output of avio_tell(), which is also int64_t. --- fftools/ffmpeg.h | 4 ++-- fftools/ffmpeg_opt.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 10a51db469..d126f5d8e6 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -147,7 +147,7 @@ typedef struct OptionsContext { int64_t recording_time; int64_t stop_time; - uint64_t limit_filesize; + int64_t limit_filesize; float mux_preload; float mux_max_delay; int shortest; @@ -594,7 +594,7 @@ typedef struct OutputFile { int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units int64_t start_time; ///< start time in microseconds == AV_TIME_BASE units - uint64_t limit_filesize; /* filesize limit expressed in bytes */ + int64_t limit_filesize; /* filesize limit expressed in bytes */ int shortest; } OutputFile; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index ab8d307c3c..8cd07eddb0 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -230,7 +230,7 @@ static void init_options(OptionsContext *o) o->start_time = AV_NOPTS_VALUE; o->start_time_eof = AV_NOPTS_VALUE; o->recording_time = INT64_MAX; - o->limit_filesize = UINT64_MAX; + o->limit_filesize = INT64_MAX; o->chapters_input_file = INT_MAX; o->accurate_seek = 1; o->thread_queue_size = -1; -- 2.34.1
Move the file size checking code to ffmpeg_mux. Use the recently introduced of_filesize(), making this code consistent with the size shown by print_report(). --- fftools/ffmpeg.c | 4 +--- fftools/ffmpeg.h | 4 ++-- fftools/ffmpeg_mux.c | 11 ++++++++++- fftools/ffmpeg_opt.c | 3 +-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 5763f231c6..8d5f071fd1 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3454,10 +3454,8 @@ static int need_output(void) for (i = 0; i < nb_output_streams; i++) { OutputStream *ost = output_streams[i]; OutputFile *of = output_files[ost->file_index]; - AVFormatContext *os = output_files[ost->file_index]->ctx; - if (ost->finished || - (os->pb && avio_tell(os->pb) >= of->limit_filesize)) + if (ost->finished || of_finished(of)) continue; if (ost->frame_number >= ost->max_frames) { int j; diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index d126f5d8e6..a97cd2a8e4 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -594,7 +594,6 @@ typedef struct OutputFile { int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units int64_t start_time; ///< start time in microseconds == AV_TIME_BASE units - int64_t limit_filesize; /* filesize limit expressed in bytes */ int shortest; } OutputFile; @@ -695,7 +694,7 @@ int hw_device_setup_for_filter(FilterGraph *fg); int hwaccel_decode_init(AVCodecContext *avctx); -int of_muxer_init(OutputFile *of); +int of_muxer_init(OutputFile *of, int64_t limit_filesize); /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); @@ -703,6 +702,7 @@ void of_close(OutputFile **pof); void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int unqueue); +int of_finished(OutputFile *of); int64_t of_filesize(OutputFile *of); #endif /* FFTOOLS_FFMPEG_H */ diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 83558f7f7d..207231f33b 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -33,6 +33,8 @@ #include "libavformat/avio.h" struct Muxer { + /* filesize limit expressed in bytes */ + int64_t limit_filesize; int header_written; }; @@ -322,7 +324,7 @@ void of_close(OutputFile **pof) av_freep(pof); } -int of_muxer_init(OutputFile *of) +int of_muxer_init(OutputFile *of, int64_t limit_filesize) { Muxer *mux = av_mallocz(sizeof(*mux)); @@ -331,9 +333,16 @@ int of_muxer_init(OutputFile *of) of->mux = mux; + mux->limit_filesize = limit_filesize; + return 0; } +int of_finished(OutputFile *of) +{ + return of_filesize(of) >= of->mux->limit_filesize; +} + int64_t of_filesize(OutputFile *of) { AVIOContext *pb = of->ctx->pb; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 8cd07eddb0..28809d0c0b 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2344,7 +2344,6 @@ static int open_output_file(OptionsContext *o, const char *filename) of->ost_index = nb_output_streams; of->recording_time = o->recording_time; of->start_time = o->start_time; - of->limit_filesize = o->limit_filesize; of->shortest = o->shortest; av_dict_copy(&of->opts, o->g->format_opts, 0); @@ -2947,7 +2946,7 @@ loop_end: exit_program(1); } - err = of_muxer_init(of); + err = of_muxer_init(of, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); exit_program(1); -- 2.34.1
Allows making the variable local to ffmpeg_mux. --- fftools/ffmpeg.c | 9 +-------- fftools/ffmpeg.h | 1 - fftools/ffmpeg_mux.c | 5 +++++ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 8d5f071fd1..51616cf51d 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -137,8 +137,6 @@ static int nb_frames_drop = 0; static int64_t decode_error_stat[2]; unsigned nb_output_dumped = 0; -int want_sdp = 1; - static BenchmarkTimeStamps current_time; AVIOContext *progress_avio = NULL; @@ -4505,7 +4503,7 @@ static int64_t getmaxrss(void) int main(int argc, char **argv) { - int i, ret; + int ret; BenchmarkTimeStamps ti; init_dynload(); @@ -4541,11 +4539,6 @@ int main(int argc, char **argv) exit_program(1); } - for (i = 0; i < nb_output_files; i++) { - if (strcmp(output_files[i]->format->name, "rtp")) - want_sdp = 0; - } - current_time = ti = get_benchmark_time_stamps(); if (transcode() < 0) exit_program(1); diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index a97cd2a8e4..e3cf7794e6 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -654,7 +654,6 @@ extern char *qsv_device; #endif extern HWDevice *filter_hw_device; -extern int want_sdp; extern unsigned nb_output_dumped; extern int main_return_code; diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 207231f33b..db04b7858d 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -38,6 +38,8 @@ struct Muxer { int header_written; }; +static int want_sdp = 1; + static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, OSTFinished others) { int i; @@ -335,6 +337,9 @@ int of_muxer_init(OutputFile *of, int64_t limit_filesize) mux->limit_filesize = limit_filesize; + if (strcmp(of->format->name, "rtp")) + want_sdp = 0; + return 0; } -- 2.34.1
There is no reason to delay this. --- fftools/ffmpeg.c | 11 ----------- fftools/ffmpeg_mux.c | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 51616cf51d..2463056f8f 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3266,7 +3266,6 @@ static void report_new_stream(int input_index, AVPacket *pkt) static int transcode_init(void) { int ret = 0, i, j, k; - AVFormatContext *oc; OutputStream *ost; InputStream *ist; char error[1024] = {0}; @@ -3340,16 +3339,6 @@ static int transcode_init(void) } } - /* write headers for files with no streams */ - for (i = 0; i < nb_output_files; i++) { - oc = output_files[i]->ctx; - if (output_files[i]->format->flags & AVFMT_NOSTREAMS && oc->nb_streams == 0) { - ret = of_check_init(output_files[i]); - if (ret < 0) - goto dump_format; - } - } - dump_format: /* dump the stream mapping */ av_log(NULL, AV_LOG_INFO, "Stream mapping:\n"); diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index db04b7858d..396e91184b 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -340,6 +340,13 @@ int of_muxer_init(OutputFile *of, int64_t limit_filesize) if (strcmp(of->format->name, "rtp")) want_sdp = 0; + /* write the header for files with no streams */ + if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { + int ret = of_check_init(of); + if (ret < 0) + return ret; + } + return 0; } -- 2.34.1
The current code postpones closing the files until after printing the final report, which accesses the output file size. Deal with this by storing the final file size before closing the file. --- fftools/ffmpeg.c | 13 ------------- fftools/ffmpeg_mux.c | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 2463056f8f..0364948b1a 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -4304,7 +4304,6 @@ static int transcode_step(void) static int transcode(void) { int ret, i; - AVFormatContext *os; OutputStream *ost; InputStream *ist; int64_t timer_start; @@ -4373,18 +4372,6 @@ static int transcode(void) /* dump report by using the first video and audio streams */ print_report(1, timer_start, av_gettime_relative()); - /* close the output files */ - for (i = 0; i < nb_output_files; i++) { - os = output_files[i]->ctx; - if (os && os->oformat && !(os->oformat->flags & AVFMT_NOFILE)) { - if ((ret = avio_closep(&os->pb)) < 0) { - av_log(NULL, AV_LOG_ERROR, "Error closing file %s: %s\n", os->url, av_err2str(ret)); - if (exit_on_error) - exit_program(1); - } - } - } - /* close each encoder */ for (i = 0; i < nb_output_streams; i++) { ost = output_streams[i]; diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 396e91184b..78395bebb4 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -35,6 +35,7 @@ struct Muxer { /* filesize limit expressed in bytes */ int64_t limit_filesize; + int64_t final_filesize; int header_written; }; @@ -304,6 +305,17 @@ int of_write_trailer(OutputFile *of) return ret; } + of->mux->final_filesize = of_filesize(of); + + if (!(of->format->flags & AVFMT_NOFILE)) { + ret = avio_closep(&of->ctx->pb); + if (ret < 0) { + av_log(NULL, AV_LOG_ERROR, "Error closing file %s: %s\n", + of->ctx->url, av_err2str(ret)); + return ret; + } + } + return 0; } @@ -360,7 +372,9 @@ int64_t of_filesize(OutputFile *of) AVIOContext *pb = of->ctx->pb; int64_t ret = -1; - if (pb) { + if (of->mux->final_filesize) + ret = of->mux->final_filesize; + else if (pb) { ret = avio_size(pb); if (ret <= 0) // FIXME improve avio_size() so it works with non seekable output too ret = avio_tell(pb); -- 2.34.1
Figure out earlier whether the output stream/file should be bitexact and store this information in a flag in OutputFile/OutputStream. Stop accessing the muxer in set_encoder_id(), which will become forbidden in future commits. --- fftools/ffmpeg.c | 21 +-------------------- fftools/ffmpeg.h | 2 ++ fftools/ffmpeg_opt.c | 27 ++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 0364948b1a..75c2322091 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -2815,37 +2815,18 @@ static int init_output_stream_streamcopy(OutputStream *ost) static void set_encoder_id(OutputFile *of, OutputStream *ost) { - const AVDictionaryEntry *e; - uint8_t *encoder_string; int encoder_string_len; - int format_flags = 0; - int codec_flags = ost->enc_ctx->flags; if (av_dict_get(ost->st->metadata, "encoder", NULL, 0)) return; - e = av_dict_get(of->opts, "fflags", NULL, 0); - if (e) { - const AVOption *o = av_opt_find(of->ctx, "fflags", NULL, 0, 0); - if (!o) - return; - av_opt_eval_flags(of->ctx, o, e->value, &format_flags); - } - e = av_dict_get(ost->encoder_opts, "flags", NULL, 0); - if (e) { - const AVOption *o = av_opt_find(ost->enc_ctx, "flags", NULL, 0, 0); - if (!o) - return; - av_opt_eval_flags(ost->enc_ctx, o, e->value, &codec_flags); - } - encoder_string_len = sizeof(LIBAVCODEC_IDENT) + strlen(ost->enc->name) + 2; encoder_string = av_mallocz(encoder_string_len); if (!encoder_string) exit_program(1); - if (!(format_flags & AVFMT_FLAG_BITEXACT) && !(codec_flags & AV_CODEC_FLAG_BITEXACT)) + if (!of->bitexact && !ost->bitexact) av_strlcpy(encoder_string, LIBAVCODEC_IDENT " ", encoder_string_len); else av_strlcpy(encoder_string, "Lavc ", encoder_string_len); diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index e3cf7794e6..ea226eb901 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -496,6 +496,7 @@ typedef struct OutputStream { int top_field_first; int rotate_overridden; int autoscale; + int bitexact; int bits_per_raw_sample; double rotate_override_value; @@ -596,6 +597,7 @@ typedef struct OutputFile { int64_t start_time; ///< start time in microseconds == AV_TIME_BASE units int shortest; + int bitexact; } OutputFile; extern InputStream **input_streams; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 28809d0c0b..6f2bf33d7d 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -1459,6 +1459,22 @@ static int choose_encoder(OptionsContext *o, AVFormatContext *s, OutputStream *o return 0; } +static int check_opt_bitexact(void *ctx, const AVDictionary *opts, + const char *opt_name, int flag) +{ + const AVDictionaryEntry *e = av_dict_get(opts, opt_name, NULL, 0); + + if (e) { + const AVOption *o = av_opt_find(ctx, opt_name, NULL, 0, 0); + int val = 0; + if (!o) + return 0; + av_opt_eval_flags(ctx, o, e->value, &val); + return !!(val & flag); + } + return 0; +} + static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, enum AVMediaType type, int source_index) { OutputStream *ost; @@ -1551,8 +1567,13 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e } - if (o->bitexact) + if (o->bitexact) { ost->enc_ctx->flags |= AV_CODEC_FLAG_BITEXACT; + ost->bitexact = 1; + } else { + ost->bitexact = check_opt_bitexact(ost->enc_ctx, ost->encoder_opts, "flags", + AV_CODEC_FLAG_BITEXACT); + } MATCH_PER_STREAM_OPT(time_bases, str, time_base, oc, st); if (time_base) { @@ -2365,6 +2386,10 @@ static int open_output_file(OptionsContext *o, const char *filename) if (o->bitexact) { oc->flags |= AVFMT_FLAG_BITEXACT; + of->bitexact = 1; + } else { + of->bitexact = check_opt_bitexact(oc, of->opts, "fflags", + AVFMT_FLAG_BITEXACT); } /* create streams for all unlabeled output pads */ -- 2.34.1
Avoid accessing the muxer context directly, as this will become forbidden in future commits. --- fftools/ffmpeg.c | 15 +++++++++------ fftools/ffmpeg.h | 2 ++ fftools/ffmpeg_mux.c | 7 +++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 75c2322091..d8ab0ffb9c 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -2860,12 +2860,15 @@ static void parse_forced_key_frames(char *kf, OutputStream *ost, *next++ = 0; if (!memcmp(p, "chapters", 8)) { - - AVFormatContext *avf = output_files[ost->file_index]->ctx; + OutputFile *of = output_files[ost->file_index]; + AVChapter * const *ch; + unsigned int nb_ch; int j; - if (avf->nb_chapters > INT_MAX - size || - !(pts = av_realloc_f(pts, size += avf->nb_chapters - 1, + ch = of_get_chapters(of, &nb_ch); + + if (nb_ch > INT_MAX - size || + !(pts = av_realloc_f(pts, size += nb_ch - 1, sizeof(*pts)))) { av_log(NULL, AV_LOG_FATAL, "Could not allocate forced key frames array.\n"); @@ -2874,8 +2877,8 @@ static void parse_forced_key_frames(char *kf, OutputStream *ost, t = p[8] ? parse_time_or_die("force_key_frames", p + 8, 1) : 0; t = av_rescale_q(t, AV_TIME_BASE_Q, avctx->time_base); - for (j = 0; j < avf->nb_chapters; j++) { - AVChapter *c = avf->chapters[j]; + for (j = 0; j < nb_ch; j++) { + const AVChapter *c = ch[j]; av_assert1(index < size); pts[index++] = av_rescale_q(c->start, c->time_base, avctx->time_base) + t; diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index ea226eb901..7bc5d2ce2e 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -705,5 +705,7 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int unqueue); int of_finished(OutputFile *of); int64_t of_filesize(OutputFile *of); +AVChapter * const * +of_get_chapters(OutputFile *of, unsigned int *nb_chapters); #endif /* FFTOOLS_FFMPEG_H */ diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 78395bebb4..19519052fe 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -382,3 +382,10 @@ int64_t of_filesize(OutputFile *of) return ret; } + +AVChapter * const * +of_get_chapters(OutputFile *of, unsigned int *nb_chapters) +{ + *nb_chapters = of->ctx->nb_chapters; + return of->ctx->chapters; +} -- 2.34.1
All other logging goes to NULL context. --- fftools/ffmpeg.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index d8ab0ffb9c..8f5901137b 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -2905,7 +2905,6 @@ static void init_encoder_time_base(OutputStream *ost, AVRational default_time_ba { InputStream *ist = get_input_stream(ost); AVCodecContext *enc_ctx = ost->enc_ctx; - AVFormatContext *oc; if (ost->enc_timebase.num > 0) { enc_ctx->time_base = ost->enc_timebase; @@ -2918,8 +2917,9 @@ static void init_encoder_time_base(OutputStream *ost, AVRational default_time_ba return; } - oc = output_files[ost->file_index]->ctx; - av_log(oc, AV_LOG_WARNING, "Input stream data not available, using default time base\n"); + av_log(NULL, AV_LOG_WARNING, + "Input stream data for output stream #%d:%d not available, " + "using default time base\n", ost->file_index, ost->index); } enc_ctx->time_base = default_time_base; @@ -2931,7 +2931,6 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) AVCodecContext *enc_ctx = ost->enc_ctx; AVCodecContext *dec_ctx = NULL; OutputFile *of = output_files[ost->file_index]; - AVFormatContext *oc = of->ctx; int ret; set_encoder_id(output_files[ost->file_index], ost); @@ -2994,7 +2993,7 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) if ( av_q2d(enc_ctx->time_base) < 0.001 && ost->vsync_method != VSYNC_PASSTHROUGH && (ost->vsync_method == VSYNC_CFR || ost->vsync_method == VSYNC_VSCFR || (ost->vsync_method == VSYNC_AUTO && !(of->format->flags & AVFMT_VARIABLE_FPS)))){ - av_log(oc, AV_LOG_WARNING, "Frame rate very high for a muxer not efficiently supporting it.\n" + av_log(NULL, AV_LOG_WARNING, "Frame rate very high for a muxer not efficiently supporting it.\n" "Please consider specifying a lower framerate, a different muxer or " "setting vsync/fps_mode to vfr\n"); } -- 2.34.1
The muxing queue currently lives in OutputStream, which is a very large struct storing the state for both encoding and muxing. The muxing queue is only used by the code in ffmpeg_mux, so it makes sense to restrict it to that file. This makes the first step towards reducing the scope of OutputStream. --- fftools/ffmpeg.c | 7 ---- fftools/ffmpeg.h | 9 ----- fftools/ffmpeg_mux.c | 87 +++++++++++++++++++++++++++++++++++++------- fftools/ffmpeg_opt.c | 9 ----- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 8f5901137b..492c8cbfd2 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -591,13 +591,6 @@ static void ffmpeg_cleanup(int ret) avcodec_free_context(&ost->enc_ctx); avcodec_parameters_free(&ost->ref_par); - if (ost->muxing_queue) { - AVPacket *pkt; - while (av_fifo_read(ost->muxing_queue, &pkt, 1) >= 0) - av_packet_free(&pkt); - av_fifo_freep2(&ost->muxing_queue); - } - av_freep(&output_streams[i]); } #if HAVE_THREADS diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 7bc5d2ce2e..5996fd7858 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -563,15 +563,6 @@ typedef struct OutputStream { int max_muxing_queue_size; - /* the packets are buffered here until the muxer is ready to be initialized */ - AVFifo *muxing_queue; - - /* - * The size of the AVPackets' buffers in queue. - * Updated when a packet is either pushed or pulled from the queue. - */ - size_t muxing_queue_data_size; - /* Threshold after which max_muxing_queue_size will be in effect */ size_t muxing_queue_data_threshold; diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 19519052fe..d49235ee63 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -32,7 +32,20 @@ #include "libavformat/avformat.h" #include "libavformat/avio.h" +typedef struct MuxStream { + /* the packets are buffered here until the muxer is ready to be initialized */ + AVFifo *muxing_queue; + + /* + * The size of the AVPackets' buffers in queue. + * Updated when a packet is either pushed or pulled from the queue. + */ + size_t muxing_queue_data_size; +} MuxStream; + struct Muxer { + MuxStream *streams; + /* filesize limit expressed in bytes */ int64_t limit_filesize; int64_t final_filesize; @@ -55,6 +68,7 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, { AVFormatContext *s = of->ctx; AVStream *st = ost->st; + MuxStream *ms = &of->mux->streams[st->index]; int ret; /* @@ -76,10 +90,10 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, if (!of->mux->header_written) { AVPacket *tmp_pkt; /* the muxer is not initialized yet, buffer the packet */ - if (!av_fifo_can_write(ost->muxing_queue)) { - size_t cur_size = av_fifo_can_read(ost->muxing_queue); + if (!av_fifo_can_write(ms->muxing_queue)) { + size_t cur_size = av_fifo_can_read(ms->muxing_queue); unsigned int are_we_over_size = - (ost->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; + (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; size_t new_size = FFMIN(2 * cur_size, limit); @@ -89,7 +103,7 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, ost->file_index, ost->st->index); exit_program(1); } - ret = av_fifo_grow2(ost->muxing_queue, new_size - cur_size); + ret = av_fifo_grow2(ms->muxing_queue, new_size - cur_size); if (ret < 0) exit_program(1); } @@ -100,8 +114,8 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, if (!tmp_pkt) exit_program(1); av_packet_move_ref(tmp_pkt, pkt); - ost->muxing_queue_data_size += tmp_pkt->size; - av_fifo_write(ost->muxing_queue, &tmp_pkt, 1); + ms->muxing_queue_data_size += tmp_pkt->size; + av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); return; } @@ -270,15 +284,16 @@ int of_check_init(OutputFile *of) /* flush the muxing queues */ for (i = 0; i < of->ctx->nb_streams; i++) { + MuxStream *ms = &of->mux->streams[i]; OutputStream *ost = output_streams[of->ost_index + i]; AVPacket *pkt; /* try to improve muxing time_base (only possible if nothing has been written yet) */ - if (!av_fifo_can_read(ost->muxing_queue)) + if (!av_fifo_can_read(ms->muxing_queue)) ost->mux_timebase = ost->st->time_base; - while (av_fifo_read(ost->muxing_queue, &pkt, 1) >= 0) { - ost->muxing_queue_data_size -= pkt->size; + while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { + ms->muxing_queue_data_size -= pkt->size; of_write_packet(of, pkt, ost, 1); av_packet_free(&pkt); } @@ -319,6 +334,29 @@ int of_write_trailer(OutputFile *of) return 0; } +static void mux_free(Muxer **pmux, int nb_streams) +{ + Muxer *mux = *pmux; + + if (!mux) + return; + + for (int i = 0; i < nb_streams; i++) { + MuxStream *ms = &mux->streams[i]; + AVPacket *pkt; + + if (!ms->muxing_queue) + continue; + + while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) + av_packet_free(&pkt); + av_fifo_freep2(&ms->muxing_queue); + } + av_freep(&mux->streams); + + av_freep(pmux); +} + void of_close(OutputFile **pof) { OutputFile *of = *pof; @@ -328,25 +366,42 @@ void of_close(OutputFile **pof) return; s = of->ctx; + + mux_free(&of->mux, s ? s->nb_streams : 0); + if (s && s->oformat && !(s->oformat->flags & AVFMT_NOFILE)) avio_closep(&s->pb); avformat_free_context(s); av_dict_free(&of->opts); - av_freep(&of->mux); - av_freep(pof); } int of_muxer_init(OutputFile *of, int64_t limit_filesize) { Muxer *mux = av_mallocz(sizeof(*mux)); + int ret = 0; if (!mux) return AVERROR(ENOMEM); + mux->streams = av_calloc(of->ctx->nb_streams, sizeof(*mux->streams)); + if (!mux->streams) { + av_freep(&mux); + return AVERROR(ENOMEM); + } + of->mux = mux; + for (int i = 0; i < of->ctx->nb_streams; i++) { + MuxStream *ms = &mux->streams[i]; + ms->muxing_queue = av_fifo_alloc2(8, sizeof(AVPacket*), 0); + if (!ms->muxing_queue) { + ret = AVERROR(ENOMEM); + goto fail; + } + } + mux->limit_filesize = limit_filesize; if (strcmp(of->format->name, "rtp")) @@ -354,12 +409,16 @@ int of_muxer_init(OutputFile *of, int64_t limit_filesize) /* write the header for files with no streams */ if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { - int ret = of_check_init(of); + ret = of_check_init(of); if (ret < 0) - return ret; + goto fail; } - return 0; +fail: + if (ret < 0) + mux_free(&of->mux, of->ctx->nb_streams); + + return ret; } int of_finished(OutputFile *of) diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 6f2bf33d7d..4d4939c6fd 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -1640,8 +1640,6 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e ost->max_muxing_queue_size = 128; MATCH_PER_STREAM_OPT(max_muxing_queue_size, i, ost->max_muxing_queue_size, oc, st); - ost->muxing_queue_data_size = 0; - ost->muxing_queue_data_threshold = 50*1024*1024; MATCH_PER_STREAM_OPT(muxing_queue_data_threshold, i, ost->muxing_queue_data_threshold, oc, st); @@ -1665,13 +1663,6 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e } ost->last_mux_dts = AV_NOPTS_VALUE; - ost->muxing_queue = av_fifo_alloc2(8, sizeof(AVPacket*), 0); - if (!ost->muxing_queue) - exit_program(1); - - MATCH_PER_STREAM_OPT(copy_initial_nonkeyframes, i, - ost->copy_initial_nonkeyframes, oc, st); - return ost; } -- 2.34.1
--- fftools/ffmpeg_mux.c | 72 +++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index d49235ee63..98637004aa 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -63,12 +63,50 @@ static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, } } +static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +{ + MuxStream *ms = &of->mux->streams[ost->index]; + AVPacket *tmp_pkt; + int ret; + + if (!av_fifo_can_write(ms->muxing_queue)) { + size_t cur_size = av_fifo_can_read(ms->muxing_queue); + unsigned int are_we_over_size = + (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; + size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; + size_t new_size = FFMIN(2 * cur_size, limit); + + if (new_size <= cur_size) { + av_log(NULL, AV_LOG_ERROR, + "Too many packets buffered for output stream %d:%d.\n", + ost->file_index, ost->st->index); + return AVERROR(ENOSPC); + } + ret = av_fifo_grow2(ms->muxing_queue, new_size - cur_size); + if (ret < 0) + return ret; + } + + ret = av_packet_make_refcounted(pkt); + if (ret < 0) + return ret; + + tmp_pkt = av_packet_alloc(); + if (!tmp_pkt) + return AVERROR(ENOMEM); + + av_packet_move_ref(tmp_pkt, pkt); + ms->muxing_queue_data_size += tmp_pkt->size; + av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); + + return 0; +} + void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int unqueue) { AVFormatContext *s = of->ctx; AVStream *st = ost->st; - MuxStream *ms = &of->mux->streams[st->index]; int ret; /* @@ -87,35 +125,13 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, ost->frame_number++; } + /* the muxer is not initialized yet, buffer the packet */ if (!of->mux->header_written) { - AVPacket *tmp_pkt; - /* the muxer is not initialized yet, buffer the packet */ - if (!av_fifo_can_write(ms->muxing_queue)) { - size_t cur_size = av_fifo_can_read(ms->muxing_queue); - unsigned int are_we_over_size = - (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; - size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; - size_t new_size = FFMIN(2 * cur_size, limit); - - if (new_size <= cur_size) { - av_log(NULL, AV_LOG_ERROR, - "Too many packets buffered for output stream %d:%d.\n", - ost->file_index, ost->st->index); - exit_program(1); - } - ret = av_fifo_grow2(ms->muxing_queue, new_size - cur_size); - if (ret < 0) - exit_program(1); - } - ret = av_packet_make_refcounted(pkt); - if (ret < 0) - exit_program(1); - tmp_pkt = av_packet_alloc(); - if (!tmp_pkt) + ret = queue_packet(of, ost, pkt); + if (ret < 0) { + av_packet_unref(pkt); exit_program(1); - av_packet_move_ref(tmp_pkt, pkt); - ms->muxing_queue_data_size += tmp_pkt->size; - av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); + } return; } -- 2.34.1
It is currently called from two places: - output_packet() in ffmpeg.c, which submits the newly available output packet to the muxer - from of_check_init() in ffmpeg_mux.c after the header has been written, to flush the muxing queue Some packets will thus be processed by this function twice, so it requires an extra parameter to indicate the place it is called from and avoid modifying some state twice. This is fragile and hard to follow, so split this function into two. Also rename of_write_packet() to of_submit_packet() to better reflect its new purpose. --- fftools/ffmpeg.c | 4 +-- fftools/ffmpeg.h | 3 +-- fftools/ffmpeg_mux.c | 63 ++++++++++++++++++++++++-------------------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 492c8cbfd2..7496ca1dc6 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -724,11 +724,11 @@ static void output_packet(OutputFile *of, AVPacket *pkt, if (ret < 0) goto finish; while ((ret = av_bsf_receive_packet(ost->bsf_ctx, pkt)) >= 0) - of_write_packet(of, pkt, ost, 0); + of_submit_packet(of, pkt, ost); if (ret == AVERROR(EAGAIN)) ret = 0; } else if (!eof) - of_write_packet(of, pkt, ost, 0); + of_submit_packet(of, pkt, ost); finish: if (ret < 0 && ret != AVERROR_EOF) { diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 5996fd7858..c7db12a640 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -692,8 +692,7 @@ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); void of_close(OutputFile **pof); -void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, - int unqueue); +void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost); int of_finished(OutputFile *of); int64_t of_filesize(OutputFile *of); AVChapter * const * diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 98637004aa..ed091a3d2d 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -102,39 +102,12 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return 0; } -void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, - int unqueue) +static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { AVFormatContext *s = of->ctx; AVStream *st = ost->st; int ret; - /* - * Audio encoders may split the packets -- #frames in != #packets out. - * But there is no reordering, so we can limit the number of output packets - * by simply dropping them here. - * Counting encoded video frames needs to be done separately because of - * reordering, see do_video_out(). - * Do not count the packet when unqueued because it has been counted when queued. - */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed) && !unqueue) { - if (ost->frame_number >= ost->max_frames) { - av_packet_unref(pkt); - return; - } - ost->frame_number++; - } - - /* the muxer is not initialized yet, buffer the packet */ - if (!of->mux->header_written) { - ret = queue_packet(of, ost, pkt); - if (ret < 0) { - av_packet_unref(pkt); - exit_program(1); - } - return; - } - if ((st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->vsync_method == VSYNC_DROP) || (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_sync_method < 0)) pkt->pts = pkt->dts = AV_NOPTS_VALUE; @@ -212,6 +185,38 @@ void of_write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, } } +void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) +{ + AVStream *st = ost->st; + int ret; + + /* + * Audio encoders may split the packets -- #frames in != #packets out. + * But there is no reordering, so we can limit the number of output packets + * by simply dropping them here. + * Counting encoded video frames needs to be done separately because of + * reordering, see do_video_out(). + */ + if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { + if (ost->frame_number >= ost->max_frames) { + av_packet_unref(pkt); + return; + } + ost->frame_number++; + } + + if (of->mux->header_written) { + write_packet(of, ost, pkt); + } else { + /* the muxer is not initialized yet, buffer the packet */ + ret = queue_packet(of, ost, pkt); + if (ret < 0) { + av_packet_unref(pkt); + exit_program(1); + } + } +} + static int print_sdp(void) { char sdp[16384]; @@ -310,7 +315,7 @@ int of_check_init(OutputFile *of) while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { ms->muxing_queue_data_size -= pkt->size; - of_write_packet(of, pkt, ost, 1); + write_packet(of, ost, pkt); av_packet_free(&pkt); } } -- 2.34.1
It is private to the muxer, no reason to access it from outside. --- fftools/ffmpeg.h | 3 +-- fftools/ffmpeg_mux.c | 9 ++++++--- fftools/ffmpeg_opt.c | 12 ++++++------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index c7db12a640..2aa220da29 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -582,7 +582,6 @@ typedef struct OutputFile { const AVOutputFormat *format; AVFormatContext *ctx; - AVDictionary *opts; int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units int64_t start_time; ///< start time in microseconds == AV_TIME_BASE units @@ -686,7 +685,7 @@ int hw_device_setup_for_filter(FilterGraph *fg); int hwaccel_decode_init(AVCodecContext *avctx); -int of_muxer_init(OutputFile *of, int64_t limit_filesize); +int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize); /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index ed091a3d2d..0fd7888d4f 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -46,6 +46,8 @@ typedef struct MuxStream { struct Muxer { MuxStream *streams; + AVDictionary *opts; + /* filesize limit expressed in bytes */ int64_t limit_filesize; int64_t final_filesize; @@ -281,7 +283,7 @@ int of_check_init(OutputFile *of) return 0; } - ret = avformat_write_header(of->ctx, &of->opts); + ret = avformat_write_header(of->ctx, &of->mux->opts); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Could not write header for output file #%d " @@ -374,6 +376,7 @@ static void mux_free(Muxer **pmux, int nb_streams) av_fifo_freep2(&ms->muxing_queue); } av_freep(&mux->streams); + av_dict_free(&mux->opts); av_freep(pmux); } @@ -393,12 +396,11 @@ void of_close(OutputFile **pof) if (s && s->oformat && !(s->oformat->flags & AVFMT_NOFILE)) avio_closep(&s->pb); avformat_free_context(s); - av_dict_free(&of->opts); av_freep(pof); } -int of_muxer_init(OutputFile *of, int64_t limit_filesize) +int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) { Muxer *mux = av_mallocz(sizeof(*mux)); int ret = 0; @@ -424,6 +426,7 @@ int of_muxer_init(OutputFile *of, int64_t limit_filesize) } mux->limit_filesize = limit_filesize; + mux->opts = opts; if (strcmp(of->format->name, "rtp")) want_sdp = 0; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 4d4939c6fd..1bcb7a3006 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2332,7 +2332,7 @@ static int open_output_file(OptionsContext *o, const char *filename) OutputFile *of; OutputStream *ost; InputStream *ist; - AVDictionary *unused_opts = NULL; + AVDictionary *unused_opts = NULL, *format_opts = NULL; const AVDictionaryEntry *e = NULL; if (o->stop_time != INT64_MAX && o->recording_time != INT64_MAX) { @@ -2357,7 +2357,7 @@ static int open_output_file(OptionsContext *o, const char *filename) of->recording_time = o->recording_time; of->start_time = o->start_time; of->shortest = o->shortest; - av_dict_copy(&of->opts, o->g->format_opts, 0); + av_dict_copy(&format_opts, o->g->format_opts, 0); if (!strcmp(filename, "-")) filename = "pipe:"; @@ -2379,7 +2379,7 @@ static int open_output_file(OptionsContext *o, const char *filename) oc->flags |= AVFMT_FLAG_BITEXACT; of->bitexact = 1; } else { - of->bitexact = check_opt_bitexact(oc, of->opts, "fflags", + of->bitexact = check_opt_bitexact(oc, format_opts, "fflags", AVFMT_FLAG_BITEXACT); } @@ -2760,7 +2760,7 @@ loop_end: /* open the file */ if ((err = avio_open2(&oc->pb, filename, AVIO_FLAG_WRITE, &oc->interrupt_callback, - &of->opts)) < 0) { + &format_opts)) < 0) { print_error(filename, err); exit_program(1); } @@ -2768,7 +2768,7 @@ loop_end: assert_file_overwrite(filename); if (o->mux_preload) { - av_dict_set_int(&of->opts, "preload", o->mux_preload*AV_TIME_BASE, 0); + av_dict_set_int(&format_opts, "preload", o->mux_preload*AV_TIME_BASE, 0); } oc->max_delay = (int)(o->mux_max_delay * AV_TIME_BASE); @@ -2962,7 +2962,7 @@ loop_end: exit_program(1); } - err = of_muxer_init(of, o->limit_filesize); + err = of_muxer_init(of, format_opts, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); exit_program(1); -- 2.34.1
The current placement of this free is historical - it used to be followed by avcodec_close(), since removed. The proper place for freeing the stats is currently right before the encoder context itself is freed. --- fftools/ffmpeg.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 7496ca1dc6..f60013eace 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -588,6 +588,8 @@ static void ffmpeg_cleanup(int ret) av_dict_free(&ost->sws_dict); av_dict_free(&ost->swr_opts); + if (ost->enc_ctx) + av_freep(&ost->enc_ctx->stats_in); avcodec_free_context(&ost->enc_ctx); avcodec_parameters_free(&ost->ref_par); @@ -4351,9 +4353,6 @@ static int transcode(void) /* close each encoder */ for (i = 0; i < nb_output_streams; i++) { ost = output_streams[i]; - if (ost->encoding_needed) { - av_freep(&ost->enc_ctx->stats_in); - } total_packets_written += ost->packets_written; if (!ost->packets_written && (abort_on_flags & ABORT_ON_FLAG_EMPTY_OUTPUT_STREAM)) { av_log(NULL, AV_LOG_FATAL, "Empty output on stream %d.\n", i); -- 2.34.1
--- fftools/ffmpeg.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index f60013eace..b8a11818b8 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -140,8 +140,6 @@ unsigned nb_output_dumped = 0; static BenchmarkTimeStamps current_time; AVIOContext *progress_avio = NULL; -static uint8_t *subtitle_out; - InputStream **input_streams = NULL; int nb_input_streams = 0; InputFile **input_files = NULL; @@ -558,8 +556,6 @@ static void ffmpeg_cleanup(int ret) } av_freep(&filtergraphs); - av_freep(&subtitle_out); - /* close files */ for (i = 0; i < nb_output_files; i++) of_close(&output_files[i]); @@ -989,7 +985,7 @@ static void do_subtitle_out(OutputFile *of, AVSubtitle *sub) { int subtitle_out_max_size = 1024 * 1024; - int subtitle_out_size, nb, i; + int subtitle_out_size, nb, i, ret; AVCodecContext *enc; AVPacket *pkt = ost->pkt; int64_t pts; @@ -1003,14 +999,6 @@ static void do_subtitle_out(OutputFile *of, enc = ost->enc_ctx; - if (!subtitle_out) { - subtitle_out = av_malloc(subtitle_out_max_size); - if (!subtitle_out) { - av_log(NULL, AV_LOG_FATAL, "Failed to allocate subtitle_out\n"); - exit_program(1); - } - } - /* Note: DVB subtitle need one packet to draw them and one other packet to clear them */ /* XXX: signal it in the codec context ? */ @@ -1030,6 +1018,12 @@ static void do_subtitle_out(OutputFile *of, if (!check_recording_time(ost)) return; + ret = av_new_packet(pkt, subtitle_out_max_size); + if (ret < 0) { + av_log(NULL, AV_LOG_FATAL, "Failed to allocate subtitle encode buffer\n"); + exit_program(1); + } + sub->pts = pts; // start_display_time is required to be 0 sub->pts += av_rescale_q(sub->start_display_time, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q); @@ -1040,8 +1034,7 @@ static void do_subtitle_out(OutputFile *of, ost->frames_encoded++; - subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out, - subtitle_out_max_size, sub); + subtitle_out_size = avcodec_encode_subtitle(enc, pkt->data, pkt->size, sub); if (i == 1) sub->num_rects = save_num_rects; if (subtitle_out_size < 0) { @@ -1049,8 +1042,6 @@ static void do_subtitle_out(OutputFile *of, exit_program(1); } - av_packet_unref(pkt); - pkt->data = subtitle_out; pkt->size = subtitle_out_size; pkt->pts = av_rescale_q(sub->pts, AV_TIME_BASE_Q, ost->mux_timebase); pkt->duration = av_rescale_q(sub->end_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase); -- 2.34.1
--- fftools/ffmpeg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index b8a11818b8..53ca8c7f7b 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -2516,7 +2516,8 @@ static int process_input_packet(InputStream *ist, const AVPacket *pkt, int no_eo for (i = 0; i < nb_output_streams; i++) { OutputStream *ost = output_streams[i]; - if (!check_output_constraints(ist, ost) || ost->encoding_needed) + if (!check_output_constraints(ist, ost) || ost->encoding_needed || + (!pkt && no_eof)) continue; do_streamcopy(ist, ost, pkt); -- 2.34.1
--- tests/fate/ffmpeg.mak | 8 ++++++++ tests/ref/fate/shortest-sub | 4 ++++ 2 files changed, 12 insertions(+) create mode 100644 tests/ref/fate/shortest-sub diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak index 94f50423be..154af2fac8 100644 --- a/tests/fate/ffmpeg.mak +++ b/tests/fate/ffmpeg.mak @@ -99,6 +99,14 @@ FATE_SAMPLES_FFMPEG-$(call FILTERDEMDEC, AMIX ARESAMPLE SINE, RAWVIDEO, \ fate-shortest: tests/data/vsynth_lena.yuv fate-shortest: CMD = framecrc -auto_conversion_filters -f lavfi -i "sine=3000:d=10" -f lavfi -i "sine=1000:d=1" -sws_flags +accurate_rnd+bitexact -fflags +bitexact -flags +bitexact -idct simple -f rawvideo -s 352x288 -pix_fmt yuv420p -i $(TARGET_PATH)/tests/data/vsynth_lena.yuv -filter_complex "[0:a:0][1:a:0]amix=inputs=2[audio]" -map 2:v:0 -map "[audio]" -sws_flags +accurate_rnd+bitexact -fflags +bitexact -flags +bitexact -idct simple -dct fastint -qscale 10 -threads 1 -c:v mpeg4 -c:a ac3_fixed -shortest +# test interleaving video with a sparse subtitle stream +FATE_SAMPLES_FFMPEG-$(call ALLYES, COLOR_FILTER, VOBSUB_DEMUXER, MATROSKA_DEMUXER,, \ + RAWVIDEO_ENCODER, MATROSKA_MUXER, FRAMECRC_MUXER) += fate-shortest-sub +fate-shortest-sub: CMD = enc_dec \ + vobsub $(TARGET_SAMPLES)/sub/vobsub.idx matroska \ + "-filter_complex 'color=s=1x1:rate=1:duration=400' -pix_fmt rgb24 -allow_raw_vfw 1 -c:s copy -c:v rawvideo" \ + framecrc "-map 0 -c copy -shortest" + # Basic test for fix_sub_duration, which calculates duration based on the # following subtitle's pts. FATE_SAMPLES_FFMPEG-$(call FILTERDEMDECENCMUX, MOVIE, MPEGVIDEO, \ diff --git a/tests/ref/fate/shortest-sub b/tests/ref/fate/shortest-sub new file mode 100644 index 0000000000..3b28898177 --- /dev/null +++ b/tests/ref/fate/shortest-sub @@ -0,0 +1,4 @@ +52bc3d6a0c80e639095a2c28da4ef42c *tests/data/fate/shortest-sub.matroska +139246 tests/data/fate/shortest-sub.matroska +d71f5d359ef788ea689415bc1e4a90df *tests/data/fate/shortest-sub.out.framecrc +stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 26055 -- 2.34.1
This will be needed in the following commit that will add a new buffering stages after encoding and bitstream filtering. --- fftools/ffmpeg.c | 23 ++++++++++++++++++----- fftools/ffmpeg.h | 2 ++ fftools/ffmpeg_opt.c | 1 + 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 53ca8c7f7b..4647555ebf 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1343,6 +1343,12 @@ static int reap_filters(int flush) continue; } + if (filtered_frame->pts != AV_NOPTS_VALUE) { + AVRational tb = av_buffersink_get_time_base(filter); + ost->last_filter_pts = av_rescale_q(filtered_frame->pts, tb, + AV_TIME_BASE_Q); + } + switch (av_buffersink_get_type(filter)) { case AVMEDIA_TYPE_VIDEO: if (!ost->frame_aspect_ratio.num) @@ -3440,13 +3446,20 @@ static OutputStream *choose_output(void) for (i = 0; i < nb_output_streams; i++) { OutputStream *ost = output_streams[i]; - int64_t opts = ost->last_mux_dts == AV_NOPTS_VALUE ? INT64_MIN : + int64_t opts; + + if (ost->filter) { + opts = ost->last_filter_pts == AV_NOPTS_VALUE ? + INT64_MIN : ost->last_filter_pts; + } else { + opts = ost->last_mux_dts == AV_NOPTS_VALUE ? INT64_MIN : av_rescale_q(ost->last_mux_dts, ost->st->time_base, AV_TIME_BASE_Q); - if (ost->last_mux_dts == AV_NOPTS_VALUE) - av_log(NULL, AV_LOG_DEBUG, - "cur_dts is invalid st:%d (%d) [init:%d i_done:%d finish:%d] (this is harmless if it occurs once at the start per stream)\n", - ost->st->index, ost->st->id, ost->initialized, ost->inputs_done, ost->finished); + if (ost->last_mux_dts == AV_NOPTS_VALUE) + av_log(NULL, AV_LOG_DEBUG, + "cur_dts is invalid st:%d (%d) [init:%d i_done:%d finish:%d] (this is harmless if it occurs once at the start per stream)\n", + ost->st->index, ost->st->id, ost->initialized, ost->inputs_done, ost->finished); + } if (!ost->initialized && !ost->inputs_done) return ost->unavailable ? NULL : ost; diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 2aa220da29..861f8140cf 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -468,6 +468,8 @@ typedef struct OutputStream { int64_t first_pts; /* dts of the last packet sent to the muxer */ int64_t last_mux_dts; + /* pts of the last frame received from the filters, in AV_TIME_BASE_Q */ + int64_t last_filter_pts; // the timebase of the packets sent to the muxer AVRational mux_timebase; AVRational enc_timebase; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 1bcb7a3006..09a281dba3 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -1662,6 +1662,7 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e input_streams[source_index]->st->discard = input_streams[source_index]->user_set_discard; } ost->last_mux_dts = AV_NOPTS_VALUE; + ost->last_filter_pts = AV_NOPTS_VALUE; return ost; } -- 2.34.1
On Thu, Jun 16, 2022 at 09:55:18PM +0200, Anton Khirnov wrote:
This will be needed in the following commit that will add a new buffering stages after encoding and bitstream filtering. --- fftools/ffmpeg.c | 23 ++++++++++++++++++----- fftools/ffmpeg.h | 2 ++ fftools/ffmpeg_opt.c | 1 + 3 files changed, 21 insertions(+), 5 deletions(-)
this seems to break this: ffmpeg -i tickets//3015/test_video -filter:a apad -f flv -vcodec flv -ar 22050 -bitexact -shortest file3015-dont-infloop.flv after the patch ffmpeg seems not to exit and keeps increasing the output file size input file seems to be here https://trac.ffmpeg.org/raw-attachment/ticket/3015/test_video thx [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB Republics decline into democracies and democracies degenerate into despotisms. -- Aristotle
This will be needed in following commits that will add new buffering stages after encoding and bitstream filtering. --- also updated my branch ffmpeg_mt/mux --- fftools/ffmpeg.c | 22 +++++++++++++++++----- fftools/ffmpeg.h | 2 ++ fftools/ffmpeg_opt.c | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 53ca8c7f7b..bdbc66fd41 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1343,6 +1343,12 @@ static int reap_filters(int flush) continue; } + if (filtered_frame->pts != AV_NOPTS_VALUE) { + AVRational tb = av_buffersink_get_time_base(filter); + ost->last_filter_pts = av_rescale_q(filtered_frame->pts, tb, + AV_TIME_BASE_Q); + } + switch (av_buffersink_get_type(filter)) { case AVMEDIA_TYPE_VIDEO: if (!ost->frame_aspect_ratio.num) @@ -3440,13 +3446,19 @@ static OutputStream *choose_output(void) for (i = 0; i < nb_output_streams; i++) { OutputStream *ost = output_streams[i]; - int64_t opts = ost->last_mux_dts == AV_NOPTS_VALUE ? INT64_MIN : + int64_t opts; + + if (ost->filter && ost->last_filter_pts != AV_NOPTS_VALUE) { + opts = ost->last_filter_pts; + } else { + opts = ost->last_mux_dts == AV_NOPTS_VALUE ? INT64_MIN : av_rescale_q(ost->last_mux_dts, ost->st->time_base, AV_TIME_BASE_Q); - if (ost->last_mux_dts == AV_NOPTS_VALUE) - av_log(NULL, AV_LOG_DEBUG, - "cur_dts is invalid st:%d (%d) [init:%d i_done:%d finish:%d] (this is harmless if it occurs once at the start per stream)\n", - ost->st->index, ost->st->id, ost->initialized, ost->inputs_done, ost->finished); + if (ost->last_mux_dts == AV_NOPTS_VALUE) + av_log(NULL, AV_LOG_DEBUG, + "cur_dts is invalid st:%d (%d) [init:%d i_done:%d finish:%d] (this is harmless if it occurs once at the start per stream)\n", + ost->st->index, ost->st->id, ost->initialized, ost->inputs_done, ost->finished); + } if (!ost->initialized && !ost->inputs_done) return ost->unavailable ? NULL : ost; diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 2aa220da29..861f8140cf 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -468,6 +468,8 @@ typedef struct OutputStream { int64_t first_pts; /* dts of the last packet sent to the muxer */ int64_t last_mux_dts; + /* pts of the last frame received from the filters, in AV_TIME_BASE_Q */ + int64_t last_filter_pts; // the timebase of the packets sent to the muxer AVRational mux_timebase; AVRational enc_timebase; diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 004db7ac7b..d1e9c0505b 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -1667,6 +1667,7 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e input_streams[source_index]->st->discard = input_streams[source_index]->user_set_discard; } ost->last_mux_dts = AV_NOPTS_VALUE; + ost->last_filter_pts = AV_NOPTS_VALUE; return ost; } -- 2.34.1
The following commits will add a new buffering stage after bitstream filters, which should not be taken into account for choosing next output. OutputStream.last_mux_dts is also used by the muxing code to make up missing DTS values - that field is now moved to the muxer-private MuxStream object. --- fftools/ffmpeg.c | 8 +++++--- fftools/ffmpeg.h | 2 +- fftools/ffmpeg_mux.c | 20 +++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 4647555ebf..6cd471d5cd 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -716,6 +716,9 @@ static void output_packet(OutputFile *of, AVPacket *pkt, { int ret = 0; + if (!eof && pkt->dts != AV_NOPTS_VALUE) + ost->last_mux_dts = av_rescale_q(pkt->dts, ost->mux_timebase, AV_TIME_BASE_Q); + /* apply the output bitstream filters */ if (ost->bsf_ctx) { ret = av_bsf_send_packet(ost->bsf_ctx, eof ? NULL : pkt); @@ -3452,9 +3455,8 @@ static OutputStream *choose_output(void) opts = ost->last_filter_pts == AV_NOPTS_VALUE ? INT64_MIN : ost->last_filter_pts; } else { - opts = ost->last_mux_dts == AV_NOPTS_VALUE ? INT64_MIN : - av_rescale_q(ost->last_mux_dts, ost->st->time_base, - AV_TIME_BASE_Q); + opts = ost->last_mux_dts == AV_NOPTS_VALUE ? + INT64_MIN : ost->last_mux_dts; if (ost->last_mux_dts == AV_NOPTS_VALUE) av_log(NULL, AV_LOG_DEBUG, "cur_dts is invalid st:%d (%d) [init:%d i_done:%d finish:%d] (this is harmless if it occurs once at the start per stream)\n", diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 861f8140cf..5403f9998b 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -466,7 +466,7 @@ typedef struct OutputStream { /* pts of the first frame encoded for this stream, used for limiting * recording time */ int64_t first_pts; - /* dts of the last packet sent to the muxer */ + /* dts of the last packet sent to the muxing queue, in AV_TIME_BASE_Q */ int64_t last_mux_dts; /* pts of the last frame received from the filters, in AV_TIME_BASE_Q */ int64_t last_filter_pts; diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 0fd7888d4f..a3350a73e9 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -41,6 +41,10 @@ typedef struct MuxStream { * Updated when a packet is either pushed or pulled from the queue. */ size_t muxing_queue_data_size; + + /* dts of the last packet sent to the muxer, in the stream timebase + * used for making up missing dts values */ + int64_t last_mux_dts; } MuxStream; struct Muxer { @@ -106,6 +110,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { + MuxStream *ms = &of->mux->streams[ost->index]; AVFormatContext *s = of->ctx; AVStream *st = ost->st; int ret; @@ -133,21 +138,21 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) pkt->dts, pkt->pts, ost->file_index, ost->st->index); pkt->pts = - pkt->dts = pkt->pts + pkt->dts + ost->last_mux_dts + 1 - - FFMIN3(pkt->pts, pkt->dts, ost->last_mux_dts + 1) - - FFMAX3(pkt->pts, pkt->dts, ost->last_mux_dts + 1); + pkt->dts = pkt->pts + pkt->dts + ms->last_mux_dts + 1 + - FFMIN3(pkt->pts, pkt->dts, ms->last_mux_dts + 1) + - FFMAX3(pkt->pts, pkt->dts, ms->last_mux_dts + 1); } if ((st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO || st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO || st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) && pkt->dts != AV_NOPTS_VALUE && - ost->last_mux_dts != AV_NOPTS_VALUE) { - int64_t max = ost->last_mux_dts + !(s->oformat->flags & AVFMT_TS_NONSTRICT); + ms->last_mux_dts != AV_NOPTS_VALUE) { + int64_t max = ms->last_mux_dts + !(s->oformat->flags & AVFMT_TS_NONSTRICT); if (pkt->dts < max) { int loglevel = max - pkt->dts > 2 || st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ? AV_LOG_WARNING : AV_LOG_DEBUG; if (exit_on_error) loglevel = AV_LOG_ERROR; av_log(s, loglevel, "Non-monotonous DTS in output stream " "%d:%d; previous: %"PRId64", current: %"PRId64"; ", - ost->file_index, ost->st->index, ost->last_mux_dts, pkt->dts); + ost->file_index, ost->st->index, ms->last_mux_dts, pkt->dts); if (exit_on_error) { av_log(NULL, AV_LOG_FATAL, "aborting.\n"); exit_program(1); @@ -161,7 +166,7 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } } - ost->last_mux_dts = pkt->dts; + ms->last_mux_dts = pkt->dts; ost->data_size += pkt->size; ost->packets_written++; @@ -423,6 +428,7 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) ret = AVERROR(ENOMEM); goto fail; } + ms->last_mux_dts = AV_NOPTS_VALUE; } mux->limit_filesize = limit_filesize; -- 2.34.1
Allows to avoid constantly allocating and freeing objects like AVFrame or AVPacket. --- fftools/objpool.c | 131 ++++++++++++++++++++++++++++++++++++++++++++++ fftools/objpool.h | 37 +++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 fftools/objpool.c create mode 100644 fftools/objpool.h diff --git a/fftools/objpool.c b/fftools/objpool.c new file mode 100644 index 0000000000..b1561ecd69 --- /dev/null +++ b/fftools/objpool.c @@ -0,0 +1,131 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> + +#include "libavcodec/packet.h" + +#include "libavutil/common.h" +#include "libavutil/error.h" +#include "libavutil/frame.h" +#include "libavutil/mem.h" + +#include "objpool.h" + +struct ObjPool { + void *pool[32]; + unsigned int pool_count; + + ObjPoolCBAlloc alloc; + ObjPoolCBReset reset; + ObjPoolCBFree free; +}; + +ObjPool *objpool_alloc(ObjPoolCBAlloc cb_alloc, ObjPoolCBReset cb_reset, + ObjPoolCBFree cb_free) +{ + ObjPool *op = av_mallocz(sizeof(*op)); + + if (!op) + return NULL; + + op->alloc = cb_alloc; + op->reset = cb_reset; + op->free = cb_free; + + return op; +} + +void objpool_free(ObjPool **pop) +{ + ObjPool *op = *pop; + + if (!op) + return; + + for (unsigned int i = 0; i < op->pool_count; i++) + op->free(&op->pool[i]); + + av_freep(pop); +} + +int objpool_get(ObjPool *op, void **obj) +{ + if (op->pool_count) { + *obj = op->pool[--op->pool_count]; + op->pool[op->pool_count] = NULL; + } else + *obj = op->alloc(); + + return *obj ? 0 : AVERROR(ENOMEM); +} + +void objpool_release(ObjPool *op, void **obj) +{ + if (!*obj) + return; + + op->reset(*obj); + + if (op->pool_count < FF_ARRAY_ELEMS(op->pool)) + op->pool[op->pool_count++] = *obj; + else + op->free(obj); + + *obj = NULL; +} + +static void *alloc_packet(void) +{ + return av_packet_alloc(); +} +static void *alloc_frame(void) +{ + return av_frame_alloc(); +} + +static void reset_packet(void *obj) +{ + return av_packet_unref(obj); +} +static void reset_frame(void *obj) +{ + return av_frame_unref(obj); +} + +static void free_packet(void **obj) +{ + AVPacket *pkt = *obj; + av_packet_free(&pkt); + *obj = NULL; +} +static void free_frame(void **obj) +{ + AVFrame *frame = *obj; + av_frame_free(&frame); + *obj = NULL; +} + +ObjPool *objpool_alloc_packets(void) +{ + return objpool_alloc(alloc_packet, reset_packet, free_packet); +} +ObjPool *objpool_alloc_frames(void) +{ + return objpool_alloc(alloc_frame, reset_frame, free_frame); +} diff --git a/fftools/objpool.h b/fftools/objpool.h new file mode 100644 index 0000000000..1b2aea6aca --- /dev/null +++ b/fftools/objpool.h @@ -0,0 +1,37 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_OBJPOOL_H +#define FFTOOLS_OBJPOOL_H + +typedef struct ObjPool ObjPool; + +typedef void* (*ObjPoolCBAlloc)(void); +typedef void (*ObjPoolCBReset)(void *); +typedef void (*ObjPoolCBFree)(void **); + +void objpool_free(ObjPool **op); +ObjPool *objpool_alloc(ObjPoolCBAlloc cb_alloc, ObjPoolCBReset cb_reset, + ObjPoolCBFree cb_free); +ObjPool *objpool_alloc_packets(void); +ObjPool *objpool_alloc_frames(void); + +int objpool_get(ObjPool *op, void **obj); +void objpool_release(ObjPool *op, void **obj); + +#endif // FFTOOLS_OBJPOOL_H -- 2.34.1
Anton Khirnov:
Allows to avoid constantly allocating and freeing objects like AVFrame or AVPacket. --- fftools/objpool.c | 131 ++++++++++++++++++++++++++++++++++++++++++++++ fftools/objpool.h | 37 +++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 fftools/objpool.c create mode 100644 fftools/objpool.h
diff --git a/fftools/objpool.c b/fftools/objpool.c new file mode 100644 index 0000000000..b1561ecd69 --- /dev/null +++ b/fftools/objpool.c @@ -0,0 +1,131 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> + +#include "libavcodec/packet.h" + +#include "libavutil/common.h" +#include "libavutil/error.h" +#include "libavutil/frame.h" +#include "libavutil/mem.h" + +#include "objpool.h" + +struct ObjPool { + void *pool[32]; + unsigned int pool_count; + + ObjPoolCBAlloc alloc; + ObjPoolCBReset reset; + ObjPoolCBFree free; +}; + +ObjPool *objpool_alloc(ObjPoolCBAlloc cb_alloc, ObjPoolCBReset cb_reset, + ObjPoolCBFree cb_free) +{ + ObjPool *op = av_mallocz(sizeof(*op)); + + if (!op) + return NULL; + + op->alloc = cb_alloc; + op->reset = cb_reset; + op->free = cb_free; + + return op; +} + +void objpool_free(ObjPool **pop) +{ + ObjPool *op = *pop; + + if (!op) + return; + + for (unsigned int i = 0; i < op->pool_count; i++) + op->free(&op->pool[i]); + + av_freep(pop); +} + +int objpool_get(ObjPool *op, void **obj) +{ + if (op->pool_count) { + *obj = op->pool[--op->pool_count]; + op->pool[op->pool_count] = NULL; + } else + *obj = op->alloc(); + + return *obj ? 0 : AVERROR(ENOMEM); +} + +void objpool_release(ObjPool *op, void **obj) +{ + if (!*obj) + return; + + op->reset(*obj); + + if (op->pool_count < FF_ARRAY_ELEMS(op->pool)) + op->pool[op->pool_count++] = *obj; + else + op->free(obj); + + *obj = NULL; +} + +static void *alloc_packet(void) +{ + return av_packet_alloc(); +} +static void *alloc_frame(void) +{ + return av_frame_alloc(); +} + +static void reset_packet(void *obj) +{ + return av_packet_unref(obj); +} +static void reset_frame(void *obj) +{ + return av_frame_unref(obj); +} + +static void free_packet(void **obj) +{ + AVPacket *pkt = *obj; + av_packet_free(&pkt); + *obj = NULL; +} +static void free_frame(void **obj) +{ + AVFrame *frame = *obj; + av_frame_free(&frame); + *obj = NULL; +} + +ObjPool *objpool_alloc_packets(void) +{ + return objpool_alloc(alloc_packet, reset_packet, free_packet); +} +ObjPool *objpool_alloc_frames(void) +{ + return objpool_alloc(alloc_frame, reset_frame, free_frame); +} diff --git a/fftools/objpool.h b/fftools/objpool.h new file mode 100644 index 0000000000..1b2aea6aca --- /dev/null +++ b/fftools/objpool.h @@ -0,0 +1,37 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_OBJPOOL_H +#define FFTOOLS_OBJPOOL_H + +typedef struct ObjPool ObjPool; + +typedef void* (*ObjPoolCBAlloc)(void); +typedef void (*ObjPoolCBReset)(void *); +typedef void (*ObjPoolCBFree)(void **); + +void objpool_free(ObjPool **op); +ObjPool *objpool_alloc(ObjPoolCBAlloc cb_alloc, ObjPoolCBReset cb_reset, + ObjPoolCBFree cb_free); +ObjPool *objpool_alloc_packets(void); +ObjPool *objpool_alloc_frames(void); + +int objpool_get(ObjPool *op, void **obj); +void objpool_release(ObjPool *op, void **obj); + +#endif // FFTOOLS_OBJPOOL_H
AVFifos are often used with non-POD elements that need custom init, reset (unref) and free callbacks (in addition to the move callbacks already supported). So why not add it to AVFifo? The only drawback to this that I see is that the pool could not be shared among multiple AVFifos, but apart from that it should support the usecases that you propose and do so in a way that avoids having to drain the fifos manually when freeing it. - Andreas
Hi, Quoting Andreas Rheinhardt (2022-06-16 23:41:36)
AVFifos are often used with non-POD elements that need custom init, reset (unref) and free callbacks (in addition to the move callbacks already supported). So why not add it to AVFifo? The only drawback to this that I see is that the pool could not be shared among multiple AVFifos, but apart from that it should support the usecases that you propose and do so in a way that avoids having to drain the fifos manually when freeing it.
Since neither of us seems to have much time to deal with this right now --- would you object to me pushing this set as is (Michael confirmed on IRC he is done testing it) and leaving the objpool question for later? Objpool is fully private, so it can always be changed or removed at will. -- Anton Khirnov
Anton Khirnov:
Hi, Quoting Andreas Rheinhardt (2022-06-16 23:41:36)
AVFifos are often used with non-POD elements that need custom init, reset (unref) and free callbacks (in addition to the move callbacks already supported). So why not add it to AVFifo? The only drawback to this that I see is that the pool could not be shared among multiple AVFifos, but apart from that it should support the usecases that you propose and do so in a way that avoids having to drain the fifos manually when freeing it.
Since neither of us seems to have much time to deal with this right now --- would you object to me pushing this set as is (Michael confirmed on IRC he is done testing it) and leaving the objpool question for later? Objpool is fully private, so it can always be changed or removed at will.
I don't object to this approach. Sorry for not having time to do any testing/reviewing. - Andreas
The -shortest option (which finishes the output file at the time the shortest stream ends) is currently implemented by faking the -t option when an output stream ends. This approach is fragile, since it depends on the frames/packets being processed in a specific order. E.g. there are currently some situations in which the output file length will depend unpredictably on unrelated factors like encoder delay. More importantly, the present work aiming at splitting various ffmpeg components into different threads will make this approach completely unworkable, since the frames/packets will arrive in effectively random order. This commit introduces a "sync queue", which is essentially a collection of FIFOs, one per stream. Frames/packets are submitted to these FIFOs and are then released for further processing (encoding or muxing) when it is ensured that the frame in question will not cause its stream to get ahead of the other streams (the logic is similar to libavformat's interleaving queue). These sync queues are then used for encoding and/or muxing when the -shortest option is specified. A new option – -shortest_buf_duration – controls the maximum number of queued packets, to avoid runaway memory usage. This commit changes the results of the following tests: - copy-shortest[12]: the last audio frame is now gone. This is correct, since it actually outlasts the last video frame. - shortest-sub: the video packets following the last subtitle packet are now gone. This is also correct. --- doc/ffmpeg.texi | 16 ++ fftools/Makefile | 2 + fftools/ffmpeg.c | 104 ++++++--- fftools/ffmpeg.h | 9 + fftools/ffmpeg_mux.c | 67 +++++- fftools/ffmpeg_opt.c | 82 +++++++ fftools/sync_queue.c | 427 ++++++++++++++++++++++++++++++++++ fftools/sync_queue.h | 100 ++++++++ tests/fate/ffmpeg.mak | 2 +- tests/ref/fate/copy-shortest1 | 1 - tests/ref/fate/copy-shortest2 | 1 - tests/ref/fate/shortest-sub | 4 +- 12 files changed, 775 insertions(+), 40 deletions(-) create mode 100644 fftools/sync_queue.c create mode 100644 fftools/sync_queue.h diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi index d943f4d6f5..7542832eb3 100644 --- a/doc/ffmpeg.texi +++ b/doc/ffmpeg.texi @@ -1750,6 +1750,22 @@ Default value is 0. Enable bitexact mode for (de)muxer and (de/en)coder @item -shortest (@emph{output}) Finish encoding when the shortest input stream ends. + +Note that this option may require buffering frames, which introduces extra +latency. The maximum amount of this latency may be controlled with the +@code{-shortest_buf_duration} option. + +@item -shortest_buf_duration @var{duration} (@emph{output}) +The @code{-shortest} option may require buffering potentially large amounts +of data when at least one of the streams is "sparse" (i.e. has large gaps +between frames – this is typically the case for subtitles). + +This option controls the maximum duration of buffered frames in seconds. +Larger values may allow the @code{-shortest} option to produce more accurate +results, but increase memory use and latency. + +The default value is 10 seconds. + @item -dts_delta_threshold Timestamp discontinuity delta threshold. @item -dts_error_threshold @var{seconds} diff --git a/fftools/Makefile b/fftools/Makefile index 81ad6c4f4f..bc57ebe748 100644 --- a/fftools/Makefile +++ b/fftools/Makefile @@ -14,6 +14,8 @@ OBJS-ffmpeg += \ fftools/ffmpeg_hw.o \ fftools/ffmpeg_mux.o \ fftools/ffmpeg_opt.o \ + fftools/objpool.o \ + fftools/sync_queue.o \ define DOFFTOOL OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 6cd471d5cd..1a14637ece 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -104,6 +104,7 @@ #include "ffmpeg.h" #include "cmdutils.h" +#include "sync_queue.h" #include "libavutil/avassert.h" @@ -569,6 +570,7 @@ static void ffmpeg_cleanup(int ret) av_bsf_free(&ost->bsf_ctx); av_frame_free(&ost->filtered_frame); + av_frame_free(&ost->sq_frame); av_frame_free(&ost->last_frame); av_packet_free(&ost->pkt); av_dict_free(&ost->encoder_opts); @@ -691,13 +693,10 @@ static void update_benchmark(const char *fmt, ...) static void close_output_stream(OutputStream *ost) { OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - ost->finished |= ENCODER_FINISHED; - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } + + if (ost->sq_idx_encode >= 0) + sq_send(of->sq_encode, ost->sq_idx_encode, SQFRAME(NULL)); } /* @@ -726,10 +725,15 @@ static void output_packet(OutputFile *of, AVPacket *pkt, goto finish; while ((ret = av_bsf_receive_packet(ost->bsf_ctx, pkt)) >= 0) of_submit_packet(of, pkt, ost); + if (ret == AVERROR_EOF) + of_submit_packet(of, NULL, ost); if (ret == AVERROR(EAGAIN)) ret = 0; - } else if (!eof) - of_submit_packet(of, pkt, ost); + } else + of_submit_packet(of, eof ? NULL : pkt, ost); + + if (eof) + ost->finished |= MUXER_FINISHED; finish: if (ret < 0 && ret != AVERROR_EOF) { @@ -963,6 +967,52 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame) av_assert0(0); } +static int submit_encode_frame(OutputFile *of, OutputStream *ost, + AVFrame *frame) +{ + int ret; + + if (ost->sq_idx_encode < 0) + return encode_frame(of, ost, frame); + + if (frame) { + ret = av_frame_ref(ost->sq_frame, frame); + if (ret < 0) + return ret; + frame = ost->sq_frame; + } + + ret = sq_send(of->sq_encode, ost->sq_idx_encode, + SQFRAME(frame)); + if (ret < 0) { + if (frame) + av_frame_unref(frame); + if (ret != AVERROR_EOF) + return ret; + } + + while (1) { + AVFrame *enc_frame = ost->sq_frame; + + ret = sq_receive(of->sq_encode, ost->sq_idx_encode, + SQFRAME(enc_frame)); + if (ret == AVERROR_EOF) { + enc_frame = NULL; + } else if (ret < 0) { + return (ret == AVERROR(EAGAIN)) ? 0 : ret; + } + + ret = encode_frame(of, ost, enc_frame); + if (enc_frame) + av_frame_unref(enc_frame); + if (ret < 0) { + if (ret == AVERROR_EOF) + close_output_stream(ost); + return ret; + } + } +} + static void do_audio_out(OutputFile *of, OutputStream *ost, AVFrame *frame) { @@ -978,8 +1028,8 @@ static void do_audio_out(OutputFile *of, OutputStream *ost, ost->sync_opts = frame->pts + frame->nb_samples; ost->samples_encoded += frame->nb_samples; - ret = encode_frame(of, ost, frame); - if (ret < 0) + ret = submit_encode_frame(of, ost, frame); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); } @@ -1265,8 +1315,8 @@ static void do_video_out(OutputFile *of, av_log(NULL, AV_LOG_DEBUG, "Forced keyframe at time %f\n", pts_time); } - ret = encode_frame(of, ost, in_picture); - if (ret < 0) + ret = submit_encode_frame(of, ost, in_picture); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); ost->sync_opts++; @@ -1278,19 +1328,6 @@ static void do_video_out(OutputFile *of, av_frame_move_ref(ost->last_frame, next_picture); } -static void finish_output_stream(OutputStream *ost) -{ - OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - - ost->finished = ENCODER_FINISHED | MUXER_FINISHED; - - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } -} - /** * Get and encode new output from any of the filtergraphs, without causing * activity. @@ -1758,7 +1795,7 @@ static void flush_encoders(void) exit_program(1); } - finish_output_stream(ost); + output_packet(of, ost->pkt, ost, 1); } init_output_stream_wrapper(ost, NULL, 1); @@ -1767,7 +1804,7 @@ static void flush_encoders(void) if (enc->codec_type != AVMEDIA_TYPE_VIDEO && enc->codec_type != AVMEDIA_TYPE_AUDIO) continue; - ret = encode_frame(of, ost, NULL); + ret = submit_encode_frame(of, ost, NULL); if (ret != AVERROR_EOF) exit_program(1); } @@ -3078,6 +3115,9 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) break; } + if (ost->sq_idx_encode >= 0) + sq_set_tb(of->sq_encode, ost->sq_idx_encode, enc_ctx->time_base); + ost->mux_timebase = enc_ctx->time_base; return 0; @@ -3086,6 +3126,7 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) static int init_output_stream(OutputStream *ost, AVFrame *frame, char *error, int error_len) { + OutputFile *of = output_files[ost->file_index]; int ret = 0; if (ost->encoding_needed) { @@ -3218,6 +3259,9 @@ static int init_output_stream(OutputStream *ost, AVFrame *frame, if (ret < 0) return ret; + if (ost->sq_idx_mux >= 0) + sq_set_tb(of->sq_mux, ost->sq_idx_mux, ost->mux_timebase); + ost->initialized = 1; ret = of_check_init(output_files[ost->file_index]); @@ -3923,8 +3967,10 @@ static int process_input(int file_index) OutputStream *ost = output_streams[j]; if (ost->source_index == ifile->ist_index + i && - (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) - finish_output_stream(ost); + (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) { + OutputFile *of = output_files[ost->file_index]; + output_packet(of, ost->pkt, ost, 1); + } } } diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 5403f9998b..106580adb2 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -26,6 +26,7 @@ #include <signal.h> #include "cmdutils.h" +#include "sync_queue.h" #include "libavformat/avformat.h" #include "libavformat/avio.h" @@ -150,6 +151,7 @@ typedef struct OptionsContext { int64_t limit_filesize; float mux_preload; float mux_max_delay; + float shortest_buf_duration; int shortest; int bitexact; @@ -482,6 +484,7 @@ typedef struct OutputStream { int64_t max_frames; AVFrame *filtered_frame; AVFrame *last_frame; + AVFrame *sq_frame; AVPacket *pkt; int last_dropped; int last_nb0_frames[3]; @@ -573,6 +576,9 @@ typedef struct OutputStream { /* frame encode sum of squared error values */ int64_t error[4]; + + int sq_idx_encode; + int sq_idx_mux; } OutputStream; typedef struct Muxer Muxer; @@ -583,6 +589,9 @@ typedef struct OutputFile { Muxer *mux; const AVOutputFormat *format; + SyncQueue *sq_encode; + SyncQueue *sq_mux; + AVFormatContext *ctx; int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index a3350a73e9..453ccac912 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -20,6 +20,7 @@ #include <string.h> #include "ffmpeg.h" +#include "sync_queue.h" #include "libavutil/fifo.h" #include "libavutil/intreadwrite.h" @@ -56,6 +57,8 @@ struct Muxer { int64_t limit_filesize; int64_t final_filesize; int header_written; + + AVPacket *sq_pkt; }; static int want_sdp = 1; @@ -72,13 +75,14 @@ static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; - AVPacket *tmp_pkt; + AVPacket *tmp_pkt = NULL; int ret; if (!av_fifo_can_write(ms->muxing_queue)) { size_t cur_size = av_fifo_can_read(ms->muxing_queue); + size_t pkt_size = pkt ? pkt->size : 0; unsigned int are_we_over_size = - (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; + (ms->muxing_queue_data_size + pkt_size) > ost->muxing_queue_data_threshold; size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; size_t new_size = FFMIN(2 * cur_size, limit); @@ -93,6 +97,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return ret; } + if (pkt) { ret = av_packet_make_refcounted(pkt); if (ret < 0) return ret; @@ -103,6 +108,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) av_packet_move_ref(tmp_pkt, pkt); ms->muxing_queue_data_size += tmp_pkt->size; + } av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); return 0; @@ -192,11 +198,44 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } +static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +{ + if (ost->sq_idx_mux >= 0) { + int ret = sq_send(of->sq_mux, ost->sq_idx_mux, SQPKT(pkt)); + if (ret < 0) { + if (pkt) + av_packet_unref(pkt); + if (ret == AVERROR_EOF) { + ost->finished |= MUXER_FINISHED; + return; + } else + exit_program(1); + } + + while (1) { + ret = sq_receive(of->sq_mux, -1, SQPKT(of->mux->sq_pkt)); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) + return; + else if (ret < 0) + exit_program(1); + + write_packet(of, output_streams[of->ost_index + ret], + of->mux->sq_pkt); + } + } else { + if (pkt) + write_packet(of, ost, pkt); + else + ost->finished |= MUXER_FINISHED; + } +} + void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) { AVStream *st = ost->st; int ret; + if (pkt) { /* * Audio encoders may split the packets -- #frames in != #packets out. * But there is no reordering, so we can limit the number of output packets @@ -211,9 +250,10 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) } ost->frame_number++; } + } if (of->mux->header_written) { - write_packet(of, ost, pkt); + submit_packet(of, ost, pkt); } else { /* the muxer is not initialized yet, buffer the packet */ ret = queue_packet(of, ost, pkt); @@ -321,9 +361,11 @@ int of_check_init(OutputFile *of) ost->mux_timebase = ost->st->time_base; while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { - ms->muxing_queue_data_size -= pkt->size; - write_packet(of, ost, pkt); - av_packet_free(&pkt); + submit_packet(of, ost, pkt); + if (pkt) { + ms->muxing_queue_data_size -= pkt->size; + av_packet_free(&pkt); + } } } @@ -383,6 +425,8 @@ static void mux_free(Muxer **pmux, int nb_streams) av_freep(&mux->streams); av_dict_free(&mux->opts); + av_packet_free(&mux->sq_pkt); + av_freep(pmux); } @@ -394,6 +438,9 @@ void of_close(OutputFile **pof) if (!of) return; + sq_free(&of->sq_encode); + sq_free(&of->sq_mux); + s = of->ctx; mux_free(&of->mux, s ? s->nb_streams : 0); @@ -437,6 +484,14 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) if (strcmp(of->format->name, "rtp")) want_sdp = 0; + if (of->sq_mux) { + mux->sq_pkt = av_packet_alloc(); + if (!mux->sq_pkt) { + ret = AVERROR(ENOMEM); + goto fail; + } + } + /* write the header for files with no streams */ if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { ret = of_check_init(of); diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 09a281dba3..2cd4d42f2a 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -31,6 +31,7 @@ #include "fopen_utf8.h" #include "cmdutils.h" #include "opt_common.h" +#include "sync_queue.h" #include "libavformat/avformat.h" @@ -234,6 +235,7 @@ static void init_options(OptionsContext *o) o->chapters_input_file = INT_MAX; o->accurate_seek = 1; o->thread_queue_size = -1; + o->shortest_buf_duration = 10.f; } static int show_hwaccels(void *optctx, const char *opt, const char *arg) @@ -2326,6 +2328,78 @@ static int init_complex_filters(void) return 0; } +static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) +{ + int nb_av_enc = 0, nb_interleaved = 0; + +#define IS_AV_ENC(ost, type) \ + (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) +#define IS_INTERLEAVED(type) (type != AVMEDIA_TYPE_ATTACHMENT) + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + ost->sq_idx_encode = -1; + ost->sq_idx_mux = -1; + + nb_interleaved += IS_INTERLEAVED(type); + nb_av_enc += IS_AV_ENC(ost, type); + } + + if (!(nb_interleaved > 1 && of->shortest)) + return 0; + + /* if we have more than one encoded audio/video streams, then we + * synchronize them before encoding */ + if (nb_av_enc > 1) { + of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); + if (!of->sq_encode) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_AV_ENC(ost, type)) + continue; + + ost->sq_idx_encode = sq_add_stream(of->sq_encode); + if (ost->sq_idx_encode < 0) + return ost->sq_idx_encode; + + ost->sq_frame = av_frame_alloc(); + if (!ost->sq_frame) + return AVERROR(ENOMEM); + } + } + + /* if there are any additional interleaved streams, then ALL the streams + * are also synchronized before sending them to the muxer */ + if (nb_interleaved > nb_av_enc) { + of->sq_mux = sq_alloc(SYNC_QUEUE_PACKETS, buf_size_us); + if (!of->sq_mux) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_INTERLEAVED(type)) + continue; + + ost->sq_idx_mux = sq_add_stream(of->sq_mux); + if (ost->sq_idx_mux < 0) + return ost->sq_idx_mux; + } + } + +#undef IS_AV_ENC +#undef IS_INTERLEAVED + + return 0; +} + static int open_output_file(OptionsContext *o, const char *filename) { AVFormatContext *oc; @@ -2963,6 +3037,12 @@ loop_end: exit_program(1); } + err = setup_sync_queues(of, oc, o->shortest_buf_duration * AV_TIME_BASE); + if (err < 0) { + av_log(NULL, AV_LOG_FATAL, "Error setting up output sync queues\n"); + exit_program(1); + } + err = of_muxer_init(of, format_opts, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); @@ -3675,6 +3755,8 @@ const OptionDef options[] = { { "shortest", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest) }, "finish encoding within shortest input" }, + { "shortest_buf_duration", HAS_ARG | OPT_FLOAT | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest_buf_duration) }, + "maximum buffering duration (in seconds) for the -shortest option" }, { "bitexact", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT | OPT_INPUT, { .off = OFFSET(bitexact) }, "bitexact mode" }, diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c new file mode 100644 index 0000000000..49f0600b6c --- /dev/null +++ b/fftools/sync_queue.c @@ -0,0 +1,427 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/error.h" +#include "libavutil/fifo.h" +#include "libavutil/mathematics.h" +#include "libavutil/mem.h" + +#include "objpool.h" +#include "sync_queue.h" + +typedef struct SyncQueueStream { + AVFifo *fifo; + AVRational tb; + + /* stream head: largest timestamp seen */ + int64_t head_ts; + /* no more frames will be sent for this stream */ + int finished; +} SyncQueueStream; + +struct SyncQueue { + enum SyncQueueType type; + + /* no more frames will be sent for any stream */ + int finished; + /* sync head: the stream with the _smallest_ head timestamp + * this stream determines which frames can be output */ + int head_stream; + /* the finished stream with the smallest finish timestamp or -1 */ + int head_finished_stream; + + // maximum buffering duration in microseconds + int64_t buf_size_us; + + SyncQueueStream *streams; + unsigned int nb_streams; + + // pool of preallocated frames to avoid constant allocations + ObjPool *pool; + SyncQueueFrame free_frames[32]; + unsigned int nb_free_frames; +}; + +static void frame_move(const SyncQueue *sq, SyncQueueFrame dst, + SyncQueueFrame src) +{ + if (sq->type == SYNC_QUEUE_PACKETS) + av_packet_move_ref(dst.p, src.p); + else + av_frame_move_ref(dst.f, src.f); +} + +static int64_t frame_ts(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? + frame.p->pts + frame.p->duration : + frame.f->pts + frame.f->pkt_duration; +} + +static int frame_null(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? (frame.p == NULL) : (frame.f == NULL); +} + +static void finish_stream(SyncQueue *sq, unsigned int stream_idx) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + st->finished = 1; + + if (st->head_ts != AV_NOPTS_VALUE) { + /* check if this stream is the new finished head */ + if (sq->head_finished_stream < 0 || + av_compare_ts(st->head_ts, st->tb, + sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb) < 0) { + sq->head_finished_stream = stream_idx; + } + + /* mark as finished all streams that should no longer receive new frames, + * due to them being ahead of some finished stream */ + st = &sq->streams[sq->head_finished_stream]; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + if (st != st1 && st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st->head_ts, st->tb, st1->head_ts, st1->tb) <= 0) + st1->finished = 1; + } + } + + /* mark the whole queue as finished if all streams are finished */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + if (!sq->streams[i].finished) + return; + } + sq->finished = 1; +} + +static void queue_head_update(SyncQueue *sq) +{ + if (sq->head_stream < 0) { + /* wait for one timestamp in each stream before determining + * the queue head */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st = &sq->streams[i]; + if (st->head_ts == AV_NOPTS_VALUE) + return; + } + + // placeholder value, correct one will be found below + sq->head_stream = 0; + } + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st_head = &sq->streams[sq->head_stream]; + SyncQueueStream *st_other = &sq->streams[i]; + if (st_other->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st_other->head_ts, st_other->tb, + st_head->head_ts, st_head->tb) < 0) + sq->head_stream = i; + } +} + +/* update this stream's head timestamp */ +static void stream_update_ts(SyncQueue *sq, unsigned int stream_idx, int64_t ts) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + if (ts == AV_NOPTS_VALUE || + (st->head_ts != AV_NOPTS_VALUE && st->head_ts >= ts)) + return; + + st->head_ts = ts; + + /* if this stream is now ahead of some finished stream, then + * this stream is also finished */ + if (sq->head_finished_stream >= 0 && + av_compare_ts(sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb, + ts, st->tb) <= 0) + finish_stream(sq, stream_idx); + + /* update the overall head timestamp if it could have changed */ + if (sq->head_stream < 0 || sq->head_stream == stream_idx) + queue_head_update(sq); +} + +/* If the queue for the given stream (or all streams when stream_idx=-1) + * is overflowing, trigger a fake heartbeat on lagging streams. + * + * @return 1 if heartbeat triggered, 0 otherwise + */ +static int overflow_heartbeat(SyncQueue *sq, int stream_idx) +{ + SyncQueueStream *st; + SyncQueueFrame frame; + int64_t tail_ts = AV_NOPTS_VALUE; + + /* if no stream specified, pick the one that is most ahead */ + if (stream_idx < 0) { + int64_t ts = AV_NOPTS_VALUE; + + for (int i = 0; i < sq->nb_streams; i++) { + st = &sq->streams[i]; + if (st->head_ts != AV_NOPTS_VALUE && + (ts == AV_NOPTS_VALUE || + av_compare_ts(ts, sq->streams[stream_idx].tb, + st->head_ts, st->tb) < 0)) { + ts = st->head_ts; + stream_idx = i; + } + } + /* no stream has a timestamp yet -> nothing to do */ + if (stream_idx < 0) + return 0; + } + + st = &sq->streams[stream_idx]; + + /* get the chosen stream's tail timestamp */ + for (size_t i = 0; tail_ts == AV_NOPTS_VALUE && + av_fifo_peek(st->fifo, &frame, 1, i) >= 0; i++) + tail_ts = frame_ts(sq, frame); + + /* overflow triggers when the tail is over 10 seconds behind the head */ + if (tail_ts == AV_NOPTS_VALUE || tail_ts >= st->head_ts || + av_rescale_q(st->head_ts - tail_ts, st->tb, AV_TIME_BASE_Q) < sq->buf_size_us) + return 0; + + /* signal a fake timestamp for all streams that prevent tail_ts from being output */ + tail_ts++; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + int64_t ts; + + if (st == st1 || st1->finished || + (st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(tail_ts, st->tb, st1->head_ts, st1->tb) <= 0)) + continue; + + ts = av_rescale_q(tail_ts, st->tb, st1->tb); + if (st1->head_ts != AV_NOPTS_VALUE) + ts = FFMAX(st1->head_ts + 1, ts); + + stream_update_ts(sq, i, ts); + } + + return 1; +} + +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame) +{ + SyncQueueStream *st; + SyncQueueFrame dst; + int64_t ts; + int ret; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(st->tb.num > 0 && st->tb.den > 0); + + if (frame_null(sq, frame)) { + finish_stream(sq, stream_idx); + return 0; + } + if (st->finished) + return AVERROR_EOF; + + ret = objpool_get(sq->pool, (void**)&dst); + if (ret < 0) + return ret; + + frame_move(sq, dst, frame); + + ts = frame_ts(sq, dst); + + ret = av_fifo_write(st->fifo, &dst, 1); + if (ret < 0) { + frame_move(sq, frame, dst); + objpool_release(sq->pool, (void**)&dst); + return ret; + } + + stream_update_ts(sq, stream_idx, ts); + + return 0; +} + +static int receive_for_stream(SyncQueue *sq, unsigned int stream_idx, + SyncQueueFrame frame) +{ + SyncQueueStream *st_head = sq->head_stream >= 0 ? + &sq->streams[sq->head_stream] : NULL; + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + if (av_fifo_can_read(st->fifo)) { + SyncQueueFrame peek; + int64_t ts; + int cmp = 1; + + av_fifo_peek(st->fifo, &peek, 1, 0); + ts = frame_ts(sq, peek); + + /* check if this stream's tail timestamp does not overtake + * the overall queue head */ + if (ts != AV_NOPTS_VALUE && st_head) + cmp = av_compare_ts(ts, st->tb, st_head->head_ts, st_head->tb); + + /* We can release frames that do not end after the queue head. + * Frames with no timestamps are just passed through with no conditions. + */ + if (cmp <= 0 || ts == AV_NOPTS_VALUE) { + frame_move(sq, frame, peek); + objpool_release(sq->pool, (void**)&peek); + av_fifo_drain2(st->fifo, 1); + return 0; + } + } + + return (sq->finished || (st->finished && !av_fifo_can_read(st->fifo))) ? + AVERROR_EOF : AVERROR(EAGAIN); +} + +static int receive_internal(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int nb_eof = 0; + int ret; + + /* read a frame for a specific stream */ + if (stream_idx >= 0) { + ret = receive_for_stream(sq, stream_idx, frame); + return (ret < 0) ? ret : stream_idx; + } + + /* read a frame for any stream with available output */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + ret = receive_for_stream(sq, i, frame); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) { + nb_eof += (ret == AVERROR_EOF); + continue; + } + return (ret < 0) ? ret : i; + } + + return (nb_eof == sq->nb_streams) ? AVERROR_EOF : AVERROR(EAGAIN); +} + +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int ret = receive_internal(sq, stream_idx, frame); + + /* try again if the queue overflowed and triggered a fake heartbeat + * for lagging streams */ + if (ret == AVERROR(EAGAIN) && overflow_heartbeat(sq, stream_idx)) + ret = receive_internal(sq, stream_idx, frame); + + return ret; +} + +int sq_add_stream(SyncQueue *sq) +{ + SyncQueueStream *tmp, *st; + + tmp = av_realloc_array(sq->streams, sq->nb_streams + 1, sizeof(*sq->streams)); + if (!tmp) + return AVERROR(ENOMEM); + sq->streams = tmp; + + st = &sq->streams[sq->nb_streams]; + memset(st, 0, sizeof(*st)); + + st->fifo = av_fifo_alloc2(1, sizeof(SyncQueueFrame), AV_FIFO_FLAG_AUTO_GROW); + if (!st->fifo) + return AVERROR(ENOMEM); + + /* we set a valid default, so that a pathological stream that never + * receives even a real timebase (and no frames) won't stall all other + * streams forever; cf. overflow_heartbeat() */ + st->tb = (AVRational){ 1, 1 }; + st->head_ts = AV_NOPTS_VALUE; + + return sq->nb_streams++; +} + +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(!av_fifo_can_read(st->fifo)); + + if (st->head_ts != AV_NOPTS_VALUE) + st->head_ts = av_rescale_q(st->head_ts, st->tb, tb); + + st->tb = tb; +} + +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) +{ + SyncQueue *sq = av_mallocz(sizeof(*sq)); + + if (!sq) + return NULL; + + sq->type = type; + sq->buf_size_us = buf_size_us; + + sq->head_stream = -1; + sq->head_finished_stream = -1; + + sq->pool = (type == SYNC_QUEUE_PACKETS) ? objpool_alloc_packets() : + objpool_alloc_frames(); + if (!sq->pool) { + av_freep(&sq); + return NULL; + } + + return sq; +} + +void sq_free(SyncQueue **psq) +{ + SyncQueue *sq = *psq; + + if (!sq) + return; + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueFrame frame; + while (av_fifo_read(sq->streams[i].fifo, &frame, 1) >= 0) + objpool_release(sq->pool, (void**)&frame); + + av_fifo_freep2(&sq->streams[i].fifo); + } + + av_freep(&sq->streams); + + objpool_free(&sq->pool); + + av_freep(psq); +} diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h new file mode 100644 index 0000000000..e08780b7bf --- /dev/null +++ b/fftools/sync_queue.h @@ -0,0 +1,100 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_SYNC_QUEUE_H +#define FFTOOLS_SYNC_QUEUE_H + +#include <stdint.h> + +#include "libavcodec/packet.h" + +#include "libavutil/frame.h" + +enum SyncQueueType { + SYNC_QUEUE_PACKETS, + SYNC_QUEUE_FRAMES, +}; + +typedef union SyncQueueFrame { + AVFrame *f; + AVPacket *p; +} SyncQueueFrame; + +#define SQFRAME(frame) ((SyncQueueFrame){ .f = (frame) }) +#define SQPKT(pkt) ((SyncQueueFrame){ .p = (pkt) }) + +typedef struct SyncQueue SyncQueue; + +/** + * Allocate a sync queue of the given type. + * + * @param buf_size_us maximum duration that will be buffered in microseconds + */ +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us); +void sq_free(SyncQueue **sq); + +/** + * Add a new stream to the sync queue. + * + * @return + * - a non-negative stream index on success + * - a negative error code on error + */ +int sq_add_stream(SyncQueue *sq); + +/** + * Set the timebase for the stream with index stream_idx. Should be called + * before sending any frames for this stream. + */ +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb); + +/** + * Submit a frame for the stream with index stream_idx. + * + * On success, the sync queue takes ownership of the frame and will reset the + * contents of the supplied frame. On failure, the frame remains owned by the + * caller. + * + * Sending a frame with NULL contents marks the stream as finished. + * + * @return + * - 0 on success + * - AVERROR_EOF when no more frames should be submitted for this stream + * - another a negative error code on failure + */ +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame); + +/** + * Read a frame from the queue. + * + * @param stream_idx index of the stream to read a frame for. May be -1, then + * try to read a frame from any stream that is ready for + * output. + * @param frame output frame will be written here on success. The frame is owned + * by the caller. + * + * @return + * - a non-negative index of the stream to which the returned frame belongs + * - AVERROR(EAGAIN) when more frames need to be submitted to the queue + * - AVERROR_EOF when no more frames will be available for this stream (for any + * stream if stream_idx is -1) + * - another negative error code on failure + */ +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame); + +#endif // FFTOOLS_SYNC_QUEUE_H diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak index 154af2fac8..38a1ae7ed5 100644 --- a/tests/fate/ffmpeg.mak +++ b/tests/fate/ffmpeg.mak @@ -105,7 +105,7 @@ FATE_SAMPLES_FFMPEG-$(call ALLYES, COLOR_FILTER, VOBSUB_DEMUXER, MATROSKA_DEMUXE fate-shortest-sub: CMD = enc_dec \ vobsub $(TARGET_SAMPLES)/sub/vobsub.idx matroska \ "-filter_complex 'color=s=1x1:rate=1:duration=400' -pix_fmt rgb24 -allow_raw_vfw 1 -c:s copy -c:v rawvideo" \ - framecrc "-map 0 -c copy -shortest" + framecrc "-map 0 -c copy -shortest -shortest_buf_duration 40" # Basic test for fix_sub_duration, which calculates duration based on the # following subtitle's pts. diff --git a/tests/ref/fate/copy-shortest1 b/tests/ref/fate/copy-shortest1 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest1 +++ b/tests/ref/fate/copy-shortest1 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/copy-shortest2 b/tests/ref/fate/copy-shortest2 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest2 +++ b/tests/ref/fate/copy-shortest2 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/shortest-sub b/tests/ref/fate/shortest-sub index 3b28898177..ff223d0fa0 100644 --- a/tests/ref/fate/shortest-sub +++ b/tests/ref/fate/shortest-sub @@ -1,4 +1,4 @@ 52bc3d6a0c80e639095a2c28da4ef42c *tests/data/fate/shortest-sub.matroska 139246 tests/data/fate/shortest-sub.matroska -d71f5d359ef788ea689415bc1e4a90df *tests/data/fate/shortest-sub.out.framecrc -stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 26055 +876ac3fa52e467050ab843969d4cf343 *tests/data/fate/shortest-sub.out.framecrc +stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 23735 -- 2.34.1
Anton Khirnov:
+struct SyncQueue { + enum SyncQueueType type; + + /* no more frames will be sent for any stream */ + int finished; + /* sync head: the stream with the _smallest_ head timestamp + * this stream determines which frames can be output */ + int head_stream; + /* the finished stream with the smallest finish timestamp or -1 */ + int head_finished_stream; + + // maximum buffering duration in microseconds + int64_t buf_size_us; + + SyncQueueStream *streams; + unsigned int nb_streams; + + // pool of preallocated frames to avoid constant allocations + ObjPool *pool; + SyncQueueFrame free_frames[32]; + unsigned int nb_free_frames;
The free_frames stuff seems unused.
+}; +
Quoting Andreas Rheinhardt (2022-06-16 23:05:32)
Anton Khirnov:
+struct SyncQueue { + enum SyncQueueType type; + + /* no more frames will be sent for any stream */ + int finished; + /* sync head: the stream with the _smallest_ head timestamp + * this stream determines which frames can be output */ + int head_stream; + /* the finished stream with the smallest finish timestamp or -1 */ + int head_finished_stream; + + // maximum buffering duration in microseconds + int64_t buf_size_us; + + SyncQueueStream *streams; + unsigned int nb_streams; + + // pool of preallocated frames to avoid constant allocations + ObjPool *pool; + SyncQueueFrame free_frames[32]; + unsigned int nb_free_frames;
The free_frames stuff seems unused.
Right, forgot to remove them after the switch to objpool. Dropped locally. -- Anton Khirnov
The -shortest option (which finishes the output file at the time the shortest stream ends) is currently implemented by faking the -t option when an output stream ends. This approach is fragile, since it depends on the frames/packets being processed in a specific order. E.g. there are currently some situations in which the output file length will depend unpredictably on unrelated factors like encoder delay. More importantly, the present work aiming at splitting various ffmpeg components into different threads will make this approach completely unworkable, since the frames/packets will arrive in effectively random order. This commit introduces a "sync queue", which is essentially a collection of FIFOs, one per stream. Frames/packets are submitted to these FIFOs and are then released for further processing (encoding or muxing) when it is ensured that the frame in question will not cause its stream to get ahead of the other streams (the logic is similar to libavformat's interleaving queue). These sync queues are then used for encoding and/or muxing when the -shortest option is specified. A new option – -shortest_buf_duration – controls the maximum number of queued packets, to avoid runaway memory usage. This commit changes the results of the following tests: - copy-shortest[12]: the last audio frame is now gone. This is correct, since it actually outlasts the last video frame. - shortest-sub: the video packets following the last subtitle packet are now gone. This is also correct. --- Now also moving the samples_encoded increment into encode_frame(), so frames dropped by the sync queue are not counted. The value is only printed at the end of encoding, not used otherwise. --- Changelog | 1 + doc/ffmpeg.texi | 16 ++ fftools/Makefile | 2 + fftools/ffmpeg.c | 106 ++++++--- fftools/ffmpeg.h | 9 + fftools/ffmpeg_mux.c | 67 +++++- fftools/ffmpeg_opt.c | 82 +++++++ fftools/sync_queue.c | 425 ++++++++++++++++++++++++++++++++++ fftools/sync_queue.h | 100 ++++++++ tests/fate/ffmpeg.mak | 2 +- tests/ref/fate/copy-shortest1 | 1 - tests/ref/fate/copy-shortest2 | 1 - tests/ref/fate/shortest-sub | 4 +- 13 files changed, 775 insertions(+), 41 deletions(-) create mode 100644 fftools/sync_queue.c create mode 100644 fftools/sync_queue.h diff --git a/Changelog b/Changelog index ef589705c4..814daff680 100644 --- a/Changelog +++ b/Changelog @@ -21,6 +21,7 @@ version 5.1: - QOI image format support - ffprobe -o option - virtualbass audio filter +- ffmpeg -shortest_buf_duration option version 5.0: diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi index d943f4d6f5..7542832eb3 100644 --- a/doc/ffmpeg.texi +++ b/doc/ffmpeg.texi @@ -1750,6 +1750,22 @@ Default value is 0. Enable bitexact mode for (de)muxer and (de/en)coder @item -shortest (@emph{output}) Finish encoding when the shortest input stream ends. + +Note that this option may require buffering frames, which introduces extra +latency. The maximum amount of this latency may be controlled with the +@code{-shortest_buf_duration} option. + +@item -shortest_buf_duration @var{duration} (@emph{output}) +The @code{-shortest} option may require buffering potentially large amounts +of data when at least one of the streams is "sparse" (i.e. has large gaps +between frames – this is typically the case for subtitles). + +This option controls the maximum duration of buffered frames in seconds. +Larger values may allow the @code{-shortest} option to produce more accurate +results, but increase memory use and latency. + +The default value is 10 seconds. + @item -dts_delta_threshold Timestamp discontinuity delta threshold. @item -dts_error_threshold @var{seconds} diff --git a/fftools/Makefile b/fftools/Makefile index 81ad6c4f4f..bc57ebe748 100644 --- a/fftools/Makefile +++ b/fftools/Makefile @@ -14,6 +14,8 @@ OBJS-ffmpeg += \ fftools/ffmpeg_hw.o \ fftools/ffmpeg_mux.o \ fftools/ffmpeg_opt.o \ + fftools/objpool.o \ + fftools/sync_queue.o \ define DOFFTOOL OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 6cd471d5cd..b2d3e73cc7 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -104,6 +104,7 @@ #include "ffmpeg.h" #include "cmdutils.h" +#include "sync_queue.h" #include "libavutil/avassert.h" @@ -569,6 +570,7 @@ static void ffmpeg_cleanup(int ret) av_bsf_free(&ost->bsf_ctx); av_frame_free(&ost->filtered_frame); + av_frame_free(&ost->sq_frame); av_frame_free(&ost->last_frame); av_packet_free(&ost->pkt); av_dict_free(&ost->encoder_opts); @@ -691,13 +693,10 @@ static void update_benchmark(const char *fmt, ...) static void close_output_stream(OutputStream *ost) { OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - ost->finished |= ENCODER_FINISHED; - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } + + if (ost->sq_idx_encode >= 0) + sq_send(of->sq_encode, ost->sq_idx_encode, SQFRAME(NULL)); } /* @@ -726,10 +725,15 @@ static void output_packet(OutputFile *of, AVPacket *pkt, goto finish; while ((ret = av_bsf_receive_packet(ost->bsf_ctx, pkt)) >= 0) of_submit_packet(of, pkt, ost); + if (ret == AVERROR_EOF) + of_submit_packet(of, NULL, ost); if (ret == AVERROR(EAGAIN)) ret = 0; - } else if (!eof) - of_submit_packet(of, pkt, ost); + } else + of_submit_packet(of, eof ? NULL : pkt, ost); + + if (eof) + ost->finished |= MUXER_FINISHED; finish: if (ret < 0 && ret != AVERROR_EOF) { @@ -891,6 +895,7 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame) if (frame) { ost->frames_encoded++; + ost->samples_encoded += frame->nb_samples; if (debug_ts) { av_log(NULL, AV_LOG_INFO, "encoder <- type:%s " @@ -963,6 +968,52 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame) av_assert0(0); } +static int submit_encode_frame(OutputFile *of, OutputStream *ost, + AVFrame *frame) +{ + int ret; + + if (ost->sq_idx_encode < 0) + return encode_frame(of, ost, frame); + + if (frame) { + ret = av_frame_ref(ost->sq_frame, frame); + if (ret < 0) + return ret; + frame = ost->sq_frame; + } + + ret = sq_send(of->sq_encode, ost->sq_idx_encode, + SQFRAME(frame)); + if (ret < 0) { + if (frame) + av_frame_unref(frame); + if (ret != AVERROR_EOF) + return ret; + } + + while (1) { + AVFrame *enc_frame = ost->sq_frame; + + ret = sq_receive(of->sq_encode, ost->sq_idx_encode, + SQFRAME(enc_frame)); + if (ret == AVERROR_EOF) { + enc_frame = NULL; + } else if (ret < 0) { + return (ret == AVERROR(EAGAIN)) ? 0 : ret; + } + + ret = encode_frame(of, ost, enc_frame); + if (enc_frame) + av_frame_unref(enc_frame); + if (ret < 0) { + if (ret == AVERROR_EOF) + close_output_stream(ost); + return ret; + } + } +} + static void do_audio_out(OutputFile *of, OutputStream *ost, AVFrame *frame) { @@ -976,10 +1027,9 @@ static void do_audio_out(OutputFile *of, OutputStream *ost, if (frame->pts == AV_NOPTS_VALUE || audio_sync_method < 0) frame->pts = ost->sync_opts; ost->sync_opts = frame->pts + frame->nb_samples; - ost->samples_encoded += frame->nb_samples; - ret = encode_frame(of, ost, frame); - if (ret < 0) + ret = submit_encode_frame(of, ost, frame); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); } @@ -1265,8 +1315,8 @@ static void do_video_out(OutputFile *of, av_log(NULL, AV_LOG_DEBUG, "Forced keyframe at time %f\n", pts_time); } - ret = encode_frame(of, ost, in_picture); - if (ret < 0) + ret = submit_encode_frame(of, ost, in_picture); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); ost->sync_opts++; @@ -1278,19 +1328,6 @@ static void do_video_out(OutputFile *of, av_frame_move_ref(ost->last_frame, next_picture); } -static void finish_output_stream(OutputStream *ost) -{ - OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - - ost->finished = ENCODER_FINISHED | MUXER_FINISHED; - - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } -} - /** * Get and encode new output from any of the filtergraphs, without causing * activity. @@ -1758,7 +1795,7 @@ static void flush_encoders(void) exit_program(1); } - finish_output_stream(ost); + output_packet(of, ost->pkt, ost, 1); } init_output_stream_wrapper(ost, NULL, 1); @@ -1767,7 +1804,7 @@ static void flush_encoders(void) if (enc->codec_type != AVMEDIA_TYPE_VIDEO && enc->codec_type != AVMEDIA_TYPE_AUDIO) continue; - ret = encode_frame(of, ost, NULL); + ret = submit_encode_frame(of, ost, NULL); if (ret != AVERROR_EOF) exit_program(1); } @@ -3078,6 +3115,9 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) break; } + if (ost->sq_idx_encode >= 0) + sq_set_tb(of->sq_encode, ost->sq_idx_encode, enc_ctx->time_base); + ost->mux_timebase = enc_ctx->time_base; return 0; @@ -3086,6 +3126,7 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) static int init_output_stream(OutputStream *ost, AVFrame *frame, char *error, int error_len) { + OutputFile *of = output_files[ost->file_index]; int ret = 0; if (ost->encoding_needed) { @@ -3218,6 +3259,9 @@ static int init_output_stream(OutputStream *ost, AVFrame *frame, if (ret < 0) return ret; + if (ost->sq_idx_mux >= 0) + sq_set_tb(of->sq_mux, ost->sq_idx_mux, ost->mux_timebase); + ost->initialized = 1; ret = of_check_init(output_files[ost->file_index]); @@ -3923,8 +3967,10 @@ static int process_input(int file_index) OutputStream *ost = output_streams[j]; if (ost->source_index == ifile->ist_index + i && - (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) - finish_output_stream(ost); + (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) { + OutputFile *of = output_files[ost->file_index]; + output_packet(of, ost->pkt, ost, 1); + } } } diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 5403f9998b..106580adb2 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -26,6 +26,7 @@ #include <signal.h> #include "cmdutils.h" +#include "sync_queue.h" #include "libavformat/avformat.h" #include "libavformat/avio.h" @@ -150,6 +151,7 @@ typedef struct OptionsContext { int64_t limit_filesize; float mux_preload; float mux_max_delay; + float shortest_buf_duration; int shortest; int bitexact; @@ -482,6 +484,7 @@ typedef struct OutputStream { int64_t max_frames; AVFrame *filtered_frame; AVFrame *last_frame; + AVFrame *sq_frame; AVPacket *pkt; int last_dropped; int last_nb0_frames[3]; @@ -573,6 +576,9 @@ typedef struct OutputStream { /* frame encode sum of squared error values */ int64_t error[4]; + + int sq_idx_encode; + int sq_idx_mux; } OutputStream; typedef struct Muxer Muxer; @@ -583,6 +589,9 @@ typedef struct OutputFile { Muxer *mux; const AVOutputFormat *format; + SyncQueue *sq_encode; + SyncQueue *sq_mux; + AVFormatContext *ctx; int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index a3350a73e9..453ccac912 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -20,6 +20,7 @@ #include <string.h> #include "ffmpeg.h" +#include "sync_queue.h" #include "libavutil/fifo.h" #include "libavutil/intreadwrite.h" @@ -56,6 +57,8 @@ struct Muxer { int64_t limit_filesize; int64_t final_filesize; int header_written; + + AVPacket *sq_pkt; }; static int want_sdp = 1; @@ -72,13 +75,14 @@ static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; - AVPacket *tmp_pkt; + AVPacket *tmp_pkt = NULL; int ret; if (!av_fifo_can_write(ms->muxing_queue)) { size_t cur_size = av_fifo_can_read(ms->muxing_queue); + size_t pkt_size = pkt ? pkt->size : 0; unsigned int are_we_over_size = - (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; + (ms->muxing_queue_data_size + pkt_size) > ost->muxing_queue_data_threshold; size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; size_t new_size = FFMIN(2 * cur_size, limit); @@ -93,6 +97,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return ret; } + if (pkt) { ret = av_packet_make_refcounted(pkt); if (ret < 0) return ret; @@ -103,6 +108,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) av_packet_move_ref(tmp_pkt, pkt); ms->muxing_queue_data_size += tmp_pkt->size; + } av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); return 0; @@ -192,11 +198,44 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } +static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +{ + if (ost->sq_idx_mux >= 0) { + int ret = sq_send(of->sq_mux, ost->sq_idx_mux, SQPKT(pkt)); + if (ret < 0) { + if (pkt) + av_packet_unref(pkt); + if (ret == AVERROR_EOF) { + ost->finished |= MUXER_FINISHED; + return; + } else + exit_program(1); + } + + while (1) { + ret = sq_receive(of->sq_mux, -1, SQPKT(of->mux->sq_pkt)); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) + return; + else if (ret < 0) + exit_program(1); + + write_packet(of, output_streams[of->ost_index + ret], + of->mux->sq_pkt); + } + } else { + if (pkt) + write_packet(of, ost, pkt); + else + ost->finished |= MUXER_FINISHED; + } +} + void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) { AVStream *st = ost->st; int ret; + if (pkt) { /* * Audio encoders may split the packets -- #frames in != #packets out. * But there is no reordering, so we can limit the number of output packets @@ -211,9 +250,10 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) } ost->frame_number++; } + } if (of->mux->header_written) { - write_packet(of, ost, pkt); + submit_packet(of, ost, pkt); } else { /* the muxer is not initialized yet, buffer the packet */ ret = queue_packet(of, ost, pkt); @@ -321,9 +361,11 @@ int of_check_init(OutputFile *of) ost->mux_timebase = ost->st->time_base; while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { - ms->muxing_queue_data_size -= pkt->size; - write_packet(of, ost, pkt); - av_packet_free(&pkt); + submit_packet(of, ost, pkt); + if (pkt) { + ms->muxing_queue_data_size -= pkt->size; + av_packet_free(&pkt); + } } } @@ -383,6 +425,8 @@ static void mux_free(Muxer **pmux, int nb_streams) av_freep(&mux->streams); av_dict_free(&mux->opts); + av_packet_free(&mux->sq_pkt); + av_freep(pmux); } @@ -394,6 +438,9 @@ void of_close(OutputFile **pof) if (!of) return; + sq_free(&of->sq_encode); + sq_free(&of->sq_mux); + s = of->ctx; mux_free(&of->mux, s ? s->nb_streams : 0); @@ -437,6 +484,14 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) if (strcmp(of->format->name, "rtp")) want_sdp = 0; + if (of->sq_mux) { + mux->sq_pkt = av_packet_alloc(); + if (!mux->sq_pkt) { + ret = AVERROR(ENOMEM); + goto fail; + } + } + /* write the header for files with no streams */ if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { ret = of_check_init(of); diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 09a281dba3..2cd4d42f2a 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -31,6 +31,7 @@ #include "fopen_utf8.h" #include "cmdutils.h" #include "opt_common.h" +#include "sync_queue.h" #include "libavformat/avformat.h" @@ -234,6 +235,7 @@ static void init_options(OptionsContext *o) o->chapters_input_file = INT_MAX; o->accurate_seek = 1; o->thread_queue_size = -1; + o->shortest_buf_duration = 10.f; } static int show_hwaccels(void *optctx, const char *opt, const char *arg) @@ -2326,6 +2328,78 @@ static int init_complex_filters(void) return 0; } +static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) +{ + int nb_av_enc = 0, nb_interleaved = 0; + +#define IS_AV_ENC(ost, type) \ + (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) +#define IS_INTERLEAVED(type) (type != AVMEDIA_TYPE_ATTACHMENT) + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + ost->sq_idx_encode = -1; + ost->sq_idx_mux = -1; + + nb_interleaved += IS_INTERLEAVED(type); + nb_av_enc += IS_AV_ENC(ost, type); + } + + if (!(nb_interleaved > 1 && of->shortest)) + return 0; + + /* if we have more than one encoded audio/video streams, then we + * synchronize them before encoding */ + if (nb_av_enc > 1) { + of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); + if (!of->sq_encode) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_AV_ENC(ost, type)) + continue; + + ost->sq_idx_encode = sq_add_stream(of->sq_encode); + if (ost->sq_idx_encode < 0) + return ost->sq_idx_encode; + + ost->sq_frame = av_frame_alloc(); + if (!ost->sq_frame) + return AVERROR(ENOMEM); + } + } + + /* if there are any additional interleaved streams, then ALL the streams + * are also synchronized before sending them to the muxer */ + if (nb_interleaved > nb_av_enc) { + of->sq_mux = sq_alloc(SYNC_QUEUE_PACKETS, buf_size_us); + if (!of->sq_mux) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_INTERLEAVED(type)) + continue; + + ost->sq_idx_mux = sq_add_stream(of->sq_mux); + if (ost->sq_idx_mux < 0) + return ost->sq_idx_mux; + } + } + +#undef IS_AV_ENC +#undef IS_INTERLEAVED + + return 0; +} + static int open_output_file(OptionsContext *o, const char *filename) { AVFormatContext *oc; @@ -2963,6 +3037,12 @@ loop_end: exit_program(1); } + err = setup_sync_queues(of, oc, o->shortest_buf_duration * AV_TIME_BASE); + if (err < 0) { + av_log(NULL, AV_LOG_FATAL, "Error setting up output sync queues\n"); + exit_program(1); + } + err = of_muxer_init(of, format_opts, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); @@ -3675,6 +3755,8 @@ const OptionDef options[] = { { "shortest", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest) }, "finish encoding within shortest input" }, + { "shortest_buf_duration", HAS_ARG | OPT_FLOAT | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest_buf_duration) }, + "maximum buffering duration (in seconds) for the -shortest option" }, { "bitexact", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT | OPT_INPUT, { .off = OFFSET(bitexact) }, "bitexact mode" }, diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c new file mode 100644 index 0000000000..ab654ca790 --- /dev/null +++ b/fftools/sync_queue.c @@ -0,0 +1,425 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/error.h" +#include "libavutil/fifo.h" +#include "libavutil/mathematics.h" +#include "libavutil/mem.h" + +#include "objpool.h" +#include "sync_queue.h" + +typedef struct SyncQueueStream { + AVFifo *fifo; + AVRational tb; + + /* stream head: largest timestamp seen */ + int64_t head_ts; + /* no more frames will be sent for this stream */ + int finished; +} SyncQueueStream; + +struct SyncQueue { + enum SyncQueueType type; + + /* no more frames will be sent for any stream */ + int finished; + /* sync head: the stream with the _smallest_ head timestamp + * this stream determines which frames can be output */ + int head_stream; + /* the finished stream with the smallest finish timestamp or -1 */ + int head_finished_stream; + + // maximum buffering duration in microseconds + int64_t buf_size_us; + + SyncQueueStream *streams; + unsigned int nb_streams; + + // pool of preallocated frames to avoid constant allocations + ObjPool *pool; +}; + +static void frame_move(const SyncQueue *sq, SyncQueueFrame dst, + SyncQueueFrame src) +{ + if (sq->type == SYNC_QUEUE_PACKETS) + av_packet_move_ref(dst.p, src.p); + else + av_frame_move_ref(dst.f, src.f); +} + +static int64_t frame_ts(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? + frame.p->pts + frame.p->duration : + frame.f->pts + frame.f->pkt_duration; +} + +static int frame_null(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? (frame.p == NULL) : (frame.f == NULL); +} + +static void finish_stream(SyncQueue *sq, unsigned int stream_idx) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + st->finished = 1; + + if (st->head_ts != AV_NOPTS_VALUE) { + /* check if this stream is the new finished head */ + if (sq->head_finished_stream < 0 || + av_compare_ts(st->head_ts, st->tb, + sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb) < 0) { + sq->head_finished_stream = stream_idx; + } + + /* mark as finished all streams that should no longer receive new frames, + * due to them being ahead of some finished stream */ + st = &sq->streams[sq->head_finished_stream]; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + if (st != st1 && st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st->head_ts, st->tb, st1->head_ts, st1->tb) <= 0) + st1->finished = 1; + } + } + + /* mark the whole queue as finished if all streams are finished */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + if (!sq->streams[i].finished) + return; + } + sq->finished = 1; +} + +static void queue_head_update(SyncQueue *sq) +{ + if (sq->head_stream < 0) { + /* wait for one timestamp in each stream before determining + * the queue head */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st = &sq->streams[i]; + if (st->head_ts == AV_NOPTS_VALUE) + return; + } + + // placeholder value, correct one will be found below + sq->head_stream = 0; + } + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st_head = &sq->streams[sq->head_stream]; + SyncQueueStream *st_other = &sq->streams[i]; + if (st_other->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st_other->head_ts, st_other->tb, + st_head->head_ts, st_head->tb) < 0) + sq->head_stream = i; + } +} + +/* update this stream's head timestamp */ +static void stream_update_ts(SyncQueue *sq, unsigned int stream_idx, int64_t ts) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + if (ts == AV_NOPTS_VALUE || + (st->head_ts != AV_NOPTS_VALUE && st->head_ts >= ts)) + return; + + st->head_ts = ts; + + /* if this stream is now ahead of some finished stream, then + * this stream is also finished */ + if (sq->head_finished_stream >= 0 && + av_compare_ts(sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb, + ts, st->tb) <= 0) + finish_stream(sq, stream_idx); + + /* update the overall head timestamp if it could have changed */ + if (sq->head_stream < 0 || sq->head_stream == stream_idx) + queue_head_update(sq); +} + +/* If the queue for the given stream (or all streams when stream_idx=-1) + * is overflowing, trigger a fake heartbeat on lagging streams. + * + * @return 1 if heartbeat triggered, 0 otherwise + */ +static int overflow_heartbeat(SyncQueue *sq, int stream_idx) +{ + SyncQueueStream *st; + SyncQueueFrame frame; + int64_t tail_ts = AV_NOPTS_VALUE; + + /* if no stream specified, pick the one that is most ahead */ + if (stream_idx < 0) { + int64_t ts = AV_NOPTS_VALUE; + + for (int i = 0; i < sq->nb_streams; i++) { + st = &sq->streams[i]; + if (st->head_ts != AV_NOPTS_VALUE && + (ts == AV_NOPTS_VALUE || + av_compare_ts(ts, sq->streams[stream_idx].tb, + st->head_ts, st->tb) < 0)) { + ts = st->head_ts; + stream_idx = i; + } + } + /* no stream has a timestamp yet -> nothing to do */ + if (stream_idx < 0) + return 0; + } + + st = &sq->streams[stream_idx]; + + /* get the chosen stream's tail timestamp */ + for (size_t i = 0; tail_ts == AV_NOPTS_VALUE && + av_fifo_peek(st->fifo, &frame, 1, i) >= 0; i++) + tail_ts = frame_ts(sq, frame); + + /* overflow triggers when the tail is over specified duration behind the head */ + if (tail_ts == AV_NOPTS_VALUE || tail_ts >= st->head_ts || + av_rescale_q(st->head_ts - tail_ts, st->tb, AV_TIME_BASE_Q) < sq->buf_size_us) + return 0; + + /* signal a fake timestamp for all streams that prevent tail_ts from being output */ + tail_ts++; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + int64_t ts; + + if (st == st1 || st1->finished || + (st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(tail_ts, st->tb, st1->head_ts, st1->tb) <= 0)) + continue; + + ts = av_rescale_q(tail_ts, st->tb, st1->tb); + if (st1->head_ts != AV_NOPTS_VALUE) + ts = FFMAX(st1->head_ts + 1, ts); + + stream_update_ts(sq, i, ts); + } + + return 1; +} + +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame) +{ + SyncQueueStream *st; + SyncQueueFrame dst; + int64_t ts; + int ret; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(st->tb.num > 0 && st->tb.den > 0); + + if (frame_null(sq, frame)) { + finish_stream(sq, stream_idx); + return 0; + } + if (st->finished) + return AVERROR_EOF; + + ret = objpool_get(sq->pool, (void**)&dst); + if (ret < 0) + return ret; + + frame_move(sq, dst, frame); + + ts = frame_ts(sq, dst); + + ret = av_fifo_write(st->fifo, &dst, 1); + if (ret < 0) { + frame_move(sq, frame, dst); + objpool_release(sq->pool, (void**)&dst); + return ret; + } + + stream_update_ts(sq, stream_idx, ts); + + return 0; +} + +static int receive_for_stream(SyncQueue *sq, unsigned int stream_idx, + SyncQueueFrame frame) +{ + SyncQueueStream *st_head = sq->head_stream >= 0 ? + &sq->streams[sq->head_stream] : NULL; + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + if (av_fifo_can_read(st->fifo)) { + SyncQueueFrame peek; + int64_t ts; + int cmp = 1; + + av_fifo_peek(st->fifo, &peek, 1, 0); + ts = frame_ts(sq, peek); + + /* check if this stream's tail timestamp does not overtake + * the overall queue head */ + if (ts != AV_NOPTS_VALUE && st_head) + cmp = av_compare_ts(ts, st->tb, st_head->head_ts, st_head->tb); + + /* We can release frames that do not end after the queue head. + * Frames with no timestamps are just passed through with no conditions. + */ + if (cmp <= 0 || ts == AV_NOPTS_VALUE) { + frame_move(sq, frame, peek); + objpool_release(sq->pool, (void**)&peek); + av_fifo_drain2(st->fifo, 1); + return 0; + } + } + + return (sq->finished || (st->finished && !av_fifo_can_read(st->fifo))) ? + AVERROR_EOF : AVERROR(EAGAIN); +} + +static int receive_internal(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int nb_eof = 0; + int ret; + + /* read a frame for a specific stream */ + if (stream_idx >= 0) { + ret = receive_for_stream(sq, stream_idx, frame); + return (ret < 0) ? ret : stream_idx; + } + + /* read a frame for any stream with available output */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + ret = receive_for_stream(sq, i, frame); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) { + nb_eof += (ret == AVERROR_EOF); + continue; + } + return (ret < 0) ? ret : i; + } + + return (nb_eof == sq->nb_streams) ? AVERROR_EOF : AVERROR(EAGAIN); +} + +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int ret = receive_internal(sq, stream_idx, frame); + + /* try again if the queue overflowed and triggered a fake heartbeat + * for lagging streams */ + if (ret == AVERROR(EAGAIN) && overflow_heartbeat(sq, stream_idx)) + ret = receive_internal(sq, stream_idx, frame); + + return ret; +} + +int sq_add_stream(SyncQueue *sq) +{ + SyncQueueStream *tmp, *st; + + tmp = av_realloc_array(sq->streams, sq->nb_streams + 1, sizeof(*sq->streams)); + if (!tmp) + return AVERROR(ENOMEM); + sq->streams = tmp; + + st = &sq->streams[sq->nb_streams]; + memset(st, 0, sizeof(*st)); + + st->fifo = av_fifo_alloc2(1, sizeof(SyncQueueFrame), AV_FIFO_FLAG_AUTO_GROW); + if (!st->fifo) + return AVERROR(ENOMEM); + + /* we set a valid default, so that a pathological stream that never + * receives even a real timebase (and no frames) won't stall all other + * streams forever; cf. overflow_heartbeat() */ + st->tb = (AVRational){ 1, 1 }; + st->head_ts = AV_NOPTS_VALUE; + + return sq->nb_streams++; +} + +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(!av_fifo_can_read(st->fifo)); + + if (st->head_ts != AV_NOPTS_VALUE) + st->head_ts = av_rescale_q(st->head_ts, st->tb, tb); + + st->tb = tb; +} + +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) +{ + SyncQueue *sq = av_mallocz(sizeof(*sq)); + + if (!sq) + return NULL; + + sq->type = type; + sq->buf_size_us = buf_size_us; + + sq->head_stream = -1; + sq->head_finished_stream = -1; + + sq->pool = (type == SYNC_QUEUE_PACKETS) ? objpool_alloc_packets() : + objpool_alloc_frames(); + if (!sq->pool) { + av_freep(&sq); + return NULL; + } + + return sq; +} + +void sq_free(SyncQueue **psq) +{ + SyncQueue *sq = *psq; + + if (!sq) + return; + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueFrame frame; + while (av_fifo_read(sq->streams[i].fifo, &frame, 1) >= 0) + objpool_release(sq->pool, (void**)&frame); + + av_fifo_freep2(&sq->streams[i].fifo); + } + + av_freep(&sq->streams); + + objpool_free(&sq->pool); + + av_freep(psq); +} diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h new file mode 100644 index 0000000000..e08780b7bf --- /dev/null +++ b/fftools/sync_queue.h @@ -0,0 +1,100 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_SYNC_QUEUE_H +#define FFTOOLS_SYNC_QUEUE_H + +#include <stdint.h> + +#include "libavcodec/packet.h" + +#include "libavutil/frame.h" + +enum SyncQueueType { + SYNC_QUEUE_PACKETS, + SYNC_QUEUE_FRAMES, +}; + +typedef union SyncQueueFrame { + AVFrame *f; + AVPacket *p; +} SyncQueueFrame; + +#define SQFRAME(frame) ((SyncQueueFrame){ .f = (frame) }) +#define SQPKT(pkt) ((SyncQueueFrame){ .p = (pkt) }) + +typedef struct SyncQueue SyncQueue; + +/** + * Allocate a sync queue of the given type. + * + * @param buf_size_us maximum duration that will be buffered in microseconds + */ +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us); +void sq_free(SyncQueue **sq); + +/** + * Add a new stream to the sync queue. + * + * @return + * - a non-negative stream index on success + * - a negative error code on error + */ +int sq_add_stream(SyncQueue *sq); + +/** + * Set the timebase for the stream with index stream_idx. Should be called + * before sending any frames for this stream. + */ +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb); + +/** + * Submit a frame for the stream with index stream_idx. + * + * On success, the sync queue takes ownership of the frame and will reset the + * contents of the supplied frame. On failure, the frame remains owned by the + * caller. + * + * Sending a frame with NULL contents marks the stream as finished. + * + * @return + * - 0 on success + * - AVERROR_EOF when no more frames should be submitted for this stream + * - another a negative error code on failure + */ +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame); + +/** + * Read a frame from the queue. + * + * @param stream_idx index of the stream to read a frame for. May be -1, then + * try to read a frame from any stream that is ready for + * output. + * @param frame output frame will be written here on success. The frame is owned + * by the caller. + * + * @return + * - a non-negative index of the stream to which the returned frame belongs + * - AVERROR(EAGAIN) when more frames need to be submitted to the queue + * - AVERROR_EOF when no more frames will be available for this stream (for any + * stream if stream_idx is -1) + * - another negative error code on failure + */ +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame); + +#endif // FFTOOLS_SYNC_QUEUE_H diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak index 154af2fac8..38a1ae7ed5 100644 --- a/tests/fate/ffmpeg.mak +++ b/tests/fate/ffmpeg.mak @@ -105,7 +105,7 @@ FATE_SAMPLES_FFMPEG-$(call ALLYES, COLOR_FILTER, VOBSUB_DEMUXER, MATROSKA_DEMUXE fate-shortest-sub: CMD = enc_dec \ vobsub $(TARGET_SAMPLES)/sub/vobsub.idx matroska \ "-filter_complex 'color=s=1x1:rate=1:duration=400' -pix_fmt rgb24 -allow_raw_vfw 1 -c:s copy -c:v rawvideo" \ - framecrc "-map 0 -c copy -shortest" + framecrc "-map 0 -c copy -shortest -shortest_buf_duration 40" # Basic test for fix_sub_duration, which calculates duration based on the # following subtitle's pts. diff --git a/tests/ref/fate/copy-shortest1 b/tests/ref/fate/copy-shortest1 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest1 +++ b/tests/ref/fate/copy-shortest1 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/copy-shortest2 b/tests/ref/fate/copy-shortest2 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest2 +++ b/tests/ref/fate/copy-shortest2 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/shortest-sub b/tests/ref/fate/shortest-sub index 3b28898177..ff223d0fa0 100644 --- a/tests/ref/fate/shortest-sub +++ b/tests/ref/fate/shortest-sub @@ -1,4 +1,4 @@ 52bc3d6a0c80e639095a2c28da4ef42c *tests/data/fate/shortest-sub.matroska 139246 tests/data/fate/shortest-sub.matroska -d71f5d359ef788ea689415bc1e4a90df *tests/data/fate/shortest-sub.out.framecrc -stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 26055 +876ac3fa52e467050ab843969d4cf343 *tests/data/fate/shortest-sub.out.framecrc +stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 23735 -- 2.34.1
The -shortest option (which finishes the output file at the time the shortest stream ends) is currently implemented by faking the -t option when an output stream ends. This approach is fragile, since it depends on the frames/packets being processed in a specific order. E.g. there are currently some situations in which the output file length will depend unpredictably on unrelated factors like encoder delay. More importantly, the present work aiming at splitting various ffmpeg components into different threads will make this approach completely unworkable, since the frames/packets will arrive in effectively random order. This commit introduces a "sync queue", which is essentially a collection of FIFOs, one per stream. Frames/packets are submitted to these FIFOs and are then released for further processing (encoding or muxing) when it is ensured that the frame in question will not cause its stream to get ahead of the other streams (the logic is similar to libavformat's interleaving queue). These sync queues are then used for encoding and/or muxing when the -shortest option is specified. A new option – -shortest_buf_duration – controls the maximum number of queued packets, to avoid runaway memory usage. This commit changes the results of the following tests: - copy-shortest[12]: the last audio frame is now gone. This is correct, since it actually outlasts the last video frame. - shortest-sub: the video packets following the last subtitle packet are now gone. This is also correct. --- Now setting video frame durations in do_video_out(), so the sync queue can account for them correctly. --- Changelog | 1 + doc/ffmpeg.texi | 16 ++ fftools/Makefile | 2 + fftools/ffmpeg.c | 109 ++++++--- fftools/ffmpeg.h | 9 + fftools/ffmpeg_mux.c | 67 +++++- fftools/ffmpeg_opt.c | 82 +++++++ fftools/sync_queue.c | 425 ++++++++++++++++++++++++++++++++++ fftools/sync_queue.h | 100 ++++++++ tests/fate/ffmpeg.mak | 2 +- tests/ref/fate/copy-shortest1 | 1 - tests/ref/fate/copy-shortest2 | 1 - tests/ref/fate/shortest-sub | 4 +- 13 files changed, 778 insertions(+), 41 deletions(-) create mode 100644 fftools/sync_queue.c create mode 100644 fftools/sync_queue.h diff --git a/Changelog b/Changelog index ba679aa64e..9f0af86c9b 100644 --- a/Changelog +++ b/Changelog @@ -23,6 +23,7 @@ version 5.1: - virtualbass audio filter - VDPAU AV1 hwaccel - PHM image format support +- ffmpeg -shortest_buf_duration option version 5.0: diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi index 1a534ff1cc..e0186abcc2 100644 --- a/doc/ffmpeg.texi +++ b/doc/ffmpeg.texi @@ -1750,6 +1750,22 @@ Default value is 0. Enable bitexact mode for (de)muxer and (de/en)coder @item -shortest (@emph{output}) Finish encoding when the shortest output stream ends. + +Note that this option may require buffering frames, which introduces extra +latency. The maximum amount of this latency may be controlled with the +@code{-shortest_buf_duration} option. + +@item -shortest_buf_duration @var{duration} (@emph{output}) +The @code{-shortest} option may require buffering potentially large amounts +of data when at least one of the streams is "sparse" (i.e. has large gaps +between frames – this is typically the case for subtitles). + +This option controls the maximum duration of buffered frames in seconds. +Larger values may allow the @code{-shortest} option to produce more accurate +results, but increase memory use and latency. + +The default value is 10 seconds. + @item -dts_delta_threshold Timestamp discontinuity delta threshold. @item -dts_error_threshold @var{seconds} diff --git a/fftools/Makefile b/fftools/Makefile index 81ad6c4f4f..bc57ebe748 100644 --- a/fftools/Makefile +++ b/fftools/Makefile @@ -14,6 +14,8 @@ OBJS-ffmpeg += \ fftools/ffmpeg_hw.o \ fftools/ffmpeg_mux.o \ fftools/ffmpeg_opt.o \ + fftools/objpool.o \ + fftools/sync_queue.o \ define DOFFTOOL OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 9ddd292512..b69f40cc8e 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -104,6 +104,7 @@ #include "ffmpeg.h" #include "cmdutils.h" +#include "sync_queue.h" #include "libavutil/avassert.h" @@ -569,6 +570,7 @@ static void ffmpeg_cleanup(int ret) av_bsf_free(&ost->bsf_ctx); av_frame_free(&ost->filtered_frame); + av_frame_free(&ost->sq_frame); av_frame_free(&ost->last_frame); av_packet_free(&ost->pkt); av_dict_free(&ost->encoder_opts); @@ -691,13 +693,10 @@ static void update_benchmark(const char *fmt, ...) static void close_output_stream(OutputStream *ost) { OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - ost->finished |= ENCODER_FINISHED; - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } + + if (ost->sq_idx_encode >= 0) + sq_send(of->sq_encode, ost->sq_idx_encode, SQFRAME(NULL)); } /* @@ -726,10 +725,15 @@ static void output_packet(OutputFile *of, AVPacket *pkt, goto finish; while ((ret = av_bsf_receive_packet(ost->bsf_ctx, pkt)) >= 0) of_submit_packet(of, pkt, ost); + if (ret == AVERROR_EOF) + of_submit_packet(of, NULL, ost); if (ret == AVERROR(EAGAIN)) ret = 0; - } else if (!eof) - of_submit_packet(of, pkt, ost); + } else + of_submit_packet(of, eof ? NULL : pkt, ost); + + if (eof) + ost->finished |= MUXER_FINISHED; finish: if (ret < 0 && ret != AVERROR_EOF) { @@ -891,6 +895,7 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame) if (frame) { ost->frames_encoded++; + ost->samples_encoded += frame->nb_samples; if (debug_ts) { av_log(NULL, AV_LOG_INFO, "encoder <- type:%s " @@ -963,6 +968,52 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame) av_assert0(0); } +static int submit_encode_frame(OutputFile *of, OutputStream *ost, + AVFrame *frame) +{ + int ret; + + if (ost->sq_idx_encode < 0) + return encode_frame(of, ost, frame); + + if (frame) { + ret = av_frame_ref(ost->sq_frame, frame); + if (ret < 0) + return ret; + frame = ost->sq_frame; + } + + ret = sq_send(of->sq_encode, ost->sq_idx_encode, + SQFRAME(frame)); + if (ret < 0) { + if (frame) + av_frame_unref(frame); + if (ret != AVERROR_EOF) + return ret; + } + + while (1) { + AVFrame *enc_frame = ost->sq_frame; + + ret = sq_receive(of->sq_encode, ost->sq_idx_encode, + SQFRAME(enc_frame)); + if (ret == AVERROR_EOF) { + enc_frame = NULL; + } else if (ret < 0) { + return (ret == AVERROR(EAGAIN)) ? 0 : ret; + } + + ret = encode_frame(of, ost, enc_frame); + if (enc_frame) + av_frame_unref(enc_frame); + if (ret < 0) { + if (ret == AVERROR_EOF) + close_output_stream(ost); + return ret; + } + } +} + static void do_audio_out(OutputFile *of, OutputStream *ost, AVFrame *frame) { @@ -976,10 +1027,9 @@ static void do_audio_out(OutputFile *of, OutputStream *ost, if (frame->pts == AV_NOPTS_VALUE || audio_sync_method < 0) frame->pts = ost->sync_opts; ost->sync_opts = frame->pts + frame->nb_samples; - ost->samples_encoded += frame->nb_samples; - ret = encode_frame(of, ost, frame); - if (ret < 0) + ret = submit_encode_frame(of, ost, frame); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); } @@ -1143,15 +1193,18 @@ static void do_video_out(OutputFile *of, if (delta0 > 1.1) nb0_frames = llrintf(delta0 - 0.6); } + next_picture->pkt_duration = 1; break; case VSYNC_VFR: if (delta <= -0.6) nb_frames = 0; else if (delta > 0.6) ost->sync_opts = llrint(sync_ipts); + next_picture->pkt_duration = duration; break; case VSYNC_DROP: case VSYNC_PASSTHROUGH: + next_picture->pkt_duration = duration; ost->sync_opts = llrint(sync_ipts); break; default: @@ -1265,8 +1318,8 @@ static void do_video_out(OutputFile *of, av_log(NULL, AV_LOG_DEBUG, "Forced keyframe at time %f\n", pts_time); } - ret = encode_frame(of, ost, in_picture); - if (ret < 0) + ret = submit_encode_frame(of, ost, in_picture); + if (ret < 0 && ret != AVERROR_EOF) exit_program(1); ost->sync_opts++; @@ -1278,19 +1331,6 @@ static void do_video_out(OutputFile *of, av_frame_move_ref(ost->last_frame, next_picture); } -static void finish_output_stream(OutputStream *ost) -{ - OutputFile *of = output_files[ost->file_index]; - AVRational time_base = ost->stream_copy ? ost->mux_timebase : ost->enc_ctx->time_base; - - ost->finished = ENCODER_FINISHED | MUXER_FINISHED; - - if (of->shortest) { - int64_t end = av_rescale_q(ost->sync_opts - ost->first_pts, time_base, AV_TIME_BASE_Q); - of->recording_time = FFMIN(of->recording_time, end); - } -} - /** * Get and encode new output from any of the filtergraphs, without causing * activity. @@ -1758,7 +1798,7 @@ static void flush_encoders(void) exit_program(1); } - finish_output_stream(ost); + output_packet(of, ost->pkt, ost, 1); } init_output_stream_wrapper(ost, NULL, 1); @@ -1767,7 +1807,7 @@ static void flush_encoders(void) if (enc->codec_type != AVMEDIA_TYPE_VIDEO && enc->codec_type != AVMEDIA_TYPE_AUDIO) continue; - ret = encode_frame(of, ost, NULL); + ret = submit_encode_frame(of, ost, NULL); if (ret != AVERROR_EOF) exit_program(1); } @@ -3078,6 +3118,9 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) break; } + if (ost->sq_idx_encode >= 0) + sq_set_tb(of->sq_encode, ost->sq_idx_encode, enc_ctx->time_base); + ost->mux_timebase = enc_ctx->time_base; return 0; @@ -3086,6 +3129,7 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame) static int init_output_stream(OutputStream *ost, AVFrame *frame, char *error, int error_len) { + OutputFile *of = output_files[ost->file_index]; int ret = 0; if (ost->encoding_needed) { @@ -3218,6 +3262,9 @@ static int init_output_stream(OutputStream *ost, AVFrame *frame, if (ret < 0) return ret; + if (ost->sq_idx_mux >= 0) + sq_set_tb(of->sq_mux, ost->sq_idx_mux, ost->mux_timebase); + ost->initialized = 1; ret = of_check_init(output_files[ost->file_index]); @@ -3922,8 +3969,10 @@ static int process_input(int file_index) OutputStream *ost = output_streams[j]; if (ost->source_index == ifile->ist_index + i && - (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) - finish_output_stream(ost); + (ost->stream_copy || ost->enc->type == AVMEDIA_TYPE_SUBTITLE)) { + OutputFile *of = output_files[ost->file_index]; + output_packet(of, ost->pkt, ost, 1); + } } } diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 0135d28e89..4223c9958c 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -26,6 +26,7 @@ #include <signal.h> #include "cmdutils.h" +#include "sync_queue.h" #include "libavformat/avformat.h" #include "libavformat/avio.h" @@ -150,6 +151,7 @@ typedef struct OptionsContext { int64_t limit_filesize; float mux_preload; float mux_max_delay; + float shortest_buf_duration; int shortest; int bitexact; @@ -482,6 +484,7 @@ typedef struct OutputStream { int64_t max_frames; AVFrame *filtered_frame; AVFrame *last_frame; + AVFrame *sq_frame; AVPacket *pkt; int64_t last_dropped; int64_t last_nb0_frames[3]; @@ -573,6 +576,9 @@ typedef struct OutputStream { /* frame encode sum of squared error values */ int64_t error[4]; + + int sq_idx_encode; + int sq_idx_mux; } OutputStream; typedef struct Muxer Muxer; @@ -583,6 +589,9 @@ typedef struct OutputFile { Muxer *mux; const AVOutputFormat *format; + SyncQueue *sq_encode; + SyncQueue *sq_mux; + AVFormatContext *ctx; int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index a3350a73e9..453ccac912 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -20,6 +20,7 @@ #include <string.h> #include "ffmpeg.h" +#include "sync_queue.h" #include "libavutil/fifo.h" #include "libavutil/intreadwrite.h" @@ -56,6 +57,8 @@ struct Muxer { int64_t limit_filesize; int64_t final_filesize; int header_written; + + AVPacket *sq_pkt; }; static int want_sdp = 1; @@ -72,13 +75,14 @@ static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; - AVPacket *tmp_pkt; + AVPacket *tmp_pkt = NULL; int ret; if (!av_fifo_can_write(ms->muxing_queue)) { size_t cur_size = av_fifo_can_read(ms->muxing_queue); + size_t pkt_size = pkt ? pkt->size : 0; unsigned int are_we_over_size = - (ms->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold; + (ms->muxing_queue_data_size + pkt_size) > ost->muxing_queue_data_threshold; size_t limit = are_we_over_size ? ost->max_muxing_queue_size : SIZE_MAX; size_t new_size = FFMIN(2 * cur_size, limit); @@ -93,6 +97,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return ret; } + if (pkt) { ret = av_packet_make_refcounted(pkt); if (ret < 0) return ret; @@ -103,6 +108,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) av_packet_move_ref(tmp_pkt, pkt); ms->muxing_queue_data_size += tmp_pkt->size; + } av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); return 0; @@ -192,11 +198,44 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } +static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +{ + if (ost->sq_idx_mux >= 0) { + int ret = sq_send(of->sq_mux, ost->sq_idx_mux, SQPKT(pkt)); + if (ret < 0) { + if (pkt) + av_packet_unref(pkt); + if (ret == AVERROR_EOF) { + ost->finished |= MUXER_FINISHED; + return; + } else + exit_program(1); + } + + while (1) { + ret = sq_receive(of->sq_mux, -1, SQPKT(of->mux->sq_pkt)); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) + return; + else if (ret < 0) + exit_program(1); + + write_packet(of, output_streams[of->ost_index + ret], + of->mux->sq_pkt); + } + } else { + if (pkt) + write_packet(of, ost, pkt); + else + ost->finished |= MUXER_FINISHED; + } +} + void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) { AVStream *st = ost->st; int ret; + if (pkt) { /* * Audio encoders may split the packets -- #frames in != #packets out. * But there is no reordering, so we can limit the number of output packets @@ -211,9 +250,10 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) } ost->frame_number++; } + } if (of->mux->header_written) { - write_packet(of, ost, pkt); + submit_packet(of, ost, pkt); } else { /* the muxer is not initialized yet, buffer the packet */ ret = queue_packet(of, ost, pkt); @@ -321,9 +361,11 @@ int of_check_init(OutputFile *of) ost->mux_timebase = ost->st->time_base; while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { - ms->muxing_queue_data_size -= pkt->size; - write_packet(of, ost, pkt); - av_packet_free(&pkt); + submit_packet(of, ost, pkt); + if (pkt) { + ms->muxing_queue_data_size -= pkt->size; + av_packet_free(&pkt); + } } } @@ -383,6 +425,8 @@ static void mux_free(Muxer **pmux, int nb_streams) av_freep(&mux->streams); av_dict_free(&mux->opts); + av_packet_free(&mux->sq_pkt); + av_freep(pmux); } @@ -394,6 +438,9 @@ void of_close(OutputFile **pof) if (!of) return; + sq_free(&of->sq_encode); + sq_free(&of->sq_mux); + s = of->ctx; mux_free(&of->mux, s ? s->nb_streams : 0); @@ -437,6 +484,14 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) if (strcmp(of->format->name, "rtp")) want_sdp = 0; + if (of->sq_mux) { + mux->sq_pkt = av_packet_alloc(); + if (!mux->sq_pkt) { + ret = AVERROR(ENOMEM); + goto fail; + } + } + /* write the header for files with no streams */ if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { ret = of_check_init(of); diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index d1e9c0505b..be88743463 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -31,6 +31,7 @@ #include "fopen_utf8.h" #include "cmdutils.h" #include "opt_common.h" +#include "sync_queue.h" #include "libavformat/avformat.h" @@ -235,6 +236,7 @@ static void init_options(OptionsContext *o) o->chapters_input_file = INT_MAX; o->accurate_seek = 1; o->thread_queue_size = -1; + o->shortest_buf_duration = 10.f; } static int show_hwaccels(void *optctx, const char *opt, const char *arg) @@ -2331,6 +2333,78 @@ static int init_complex_filters(void) return 0; } +static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) +{ + int nb_av_enc = 0, nb_interleaved = 0; + +#define IS_AV_ENC(ost, type) \ + (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) +#define IS_INTERLEAVED(type) (type != AVMEDIA_TYPE_ATTACHMENT) + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + ost->sq_idx_encode = -1; + ost->sq_idx_mux = -1; + + nb_interleaved += IS_INTERLEAVED(type); + nb_av_enc += IS_AV_ENC(ost, type); + } + + if (!(nb_interleaved > 1 && of->shortest)) + return 0; + + /* if we have more than one encoded audio/video streams, then we + * synchronize them before encoding */ + if (nb_av_enc > 1) { + of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); + if (!of->sq_encode) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_AV_ENC(ost, type)) + continue; + + ost->sq_idx_encode = sq_add_stream(of->sq_encode); + if (ost->sq_idx_encode < 0) + return ost->sq_idx_encode; + + ost->sq_frame = av_frame_alloc(); + if (!ost->sq_frame) + return AVERROR(ENOMEM); + } + } + + /* if there are any additional interleaved streams, then ALL the streams + * are also synchronized before sending them to the muxer */ + if (nb_interleaved > nb_av_enc) { + of->sq_mux = sq_alloc(SYNC_QUEUE_PACKETS, buf_size_us); + if (!of->sq_mux) + return AVERROR(ENOMEM); + + for (int i = 0; i < oc->nb_streams; i++) { + OutputStream *ost = output_streams[of->ost_index + i]; + enum AVMediaType type = ost->st->codecpar->codec_type; + + if (!IS_INTERLEAVED(type)) + continue; + + ost->sq_idx_mux = sq_add_stream(of->sq_mux); + if (ost->sq_idx_mux < 0) + return ost->sq_idx_mux; + } + } + +#undef IS_AV_ENC +#undef IS_INTERLEAVED + + return 0; +} + static int open_output_file(OptionsContext *o, const char *filename) { AVFormatContext *oc; @@ -2968,6 +3042,12 @@ loop_end: exit_program(1); } + err = setup_sync_queues(of, oc, o->shortest_buf_duration * AV_TIME_BASE); + if (err < 0) { + av_log(NULL, AV_LOG_FATAL, "Error setting up output sync queues\n"); + exit_program(1); + } + err = of_muxer_init(of, format_opts, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); @@ -3680,6 +3760,8 @@ const OptionDef options[] = { { "shortest", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest) }, "finish encoding within shortest input" }, + { "shortest_buf_duration", HAS_ARG | OPT_FLOAT | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT, { .off = OFFSET(shortest_buf_duration) }, + "maximum buffering duration (in seconds) for the -shortest option" }, { "bitexact", OPT_BOOL | OPT_EXPERT | OPT_OFFSET | OPT_OUTPUT | OPT_INPUT, { .off = OFFSET(bitexact) }, "bitexact mode" }, diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c new file mode 100644 index 0000000000..ab654ca790 --- /dev/null +++ b/fftools/sync_queue.c @@ -0,0 +1,425 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/error.h" +#include "libavutil/fifo.h" +#include "libavutil/mathematics.h" +#include "libavutil/mem.h" + +#include "objpool.h" +#include "sync_queue.h" + +typedef struct SyncQueueStream { + AVFifo *fifo; + AVRational tb; + + /* stream head: largest timestamp seen */ + int64_t head_ts; + /* no more frames will be sent for this stream */ + int finished; +} SyncQueueStream; + +struct SyncQueue { + enum SyncQueueType type; + + /* no more frames will be sent for any stream */ + int finished; + /* sync head: the stream with the _smallest_ head timestamp + * this stream determines which frames can be output */ + int head_stream; + /* the finished stream with the smallest finish timestamp or -1 */ + int head_finished_stream; + + // maximum buffering duration in microseconds + int64_t buf_size_us; + + SyncQueueStream *streams; + unsigned int nb_streams; + + // pool of preallocated frames to avoid constant allocations + ObjPool *pool; +}; + +static void frame_move(const SyncQueue *sq, SyncQueueFrame dst, + SyncQueueFrame src) +{ + if (sq->type == SYNC_QUEUE_PACKETS) + av_packet_move_ref(dst.p, src.p); + else + av_frame_move_ref(dst.f, src.f); +} + +static int64_t frame_ts(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? + frame.p->pts + frame.p->duration : + frame.f->pts + frame.f->pkt_duration; +} + +static int frame_null(const SyncQueue *sq, SyncQueueFrame frame) +{ + return (sq->type == SYNC_QUEUE_PACKETS) ? (frame.p == NULL) : (frame.f == NULL); +} + +static void finish_stream(SyncQueue *sq, unsigned int stream_idx) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + st->finished = 1; + + if (st->head_ts != AV_NOPTS_VALUE) { + /* check if this stream is the new finished head */ + if (sq->head_finished_stream < 0 || + av_compare_ts(st->head_ts, st->tb, + sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb) < 0) { + sq->head_finished_stream = stream_idx; + } + + /* mark as finished all streams that should no longer receive new frames, + * due to them being ahead of some finished stream */ + st = &sq->streams[sq->head_finished_stream]; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + if (st != st1 && st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st->head_ts, st->tb, st1->head_ts, st1->tb) <= 0) + st1->finished = 1; + } + } + + /* mark the whole queue as finished if all streams are finished */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + if (!sq->streams[i].finished) + return; + } + sq->finished = 1; +} + +static void queue_head_update(SyncQueue *sq) +{ + if (sq->head_stream < 0) { + /* wait for one timestamp in each stream before determining + * the queue head */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st = &sq->streams[i]; + if (st->head_ts == AV_NOPTS_VALUE) + return; + } + + // placeholder value, correct one will be found below + sq->head_stream = 0; + } + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st_head = &sq->streams[sq->head_stream]; + SyncQueueStream *st_other = &sq->streams[i]; + if (st_other->head_ts != AV_NOPTS_VALUE && + av_compare_ts(st_other->head_ts, st_other->tb, + st_head->head_ts, st_head->tb) < 0) + sq->head_stream = i; + } +} + +/* update this stream's head timestamp */ +static void stream_update_ts(SyncQueue *sq, unsigned int stream_idx, int64_t ts) +{ + SyncQueueStream *st = &sq->streams[stream_idx]; + + if (ts == AV_NOPTS_VALUE || + (st->head_ts != AV_NOPTS_VALUE && st->head_ts >= ts)) + return; + + st->head_ts = ts; + + /* if this stream is now ahead of some finished stream, then + * this stream is also finished */ + if (sq->head_finished_stream >= 0 && + av_compare_ts(sq->streams[sq->head_finished_stream].head_ts, + sq->streams[sq->head_finished_stream].tb, + ts, st->tb) <= 0) + finish_stream(sq, stream_idx); + + /* update the overall head timestamp if it could have changed */ + if (sq->head_stream < 0 || sq->head_stream == stream_idx) + queue_head_update(sq); +} + +/* If the queue for the given stream (or all streams when stream_idx=-1) + * is overflowing, trigger a fake heartbeat on lagging streams. + * + * @return 1 if heartbeat triggered, 0 otherwise + */ +static int overflow_heartbeat(SyncQueue *sq, int stream_idx) +{ + SyncQueueStream *st; + SyncQueueFrame frame; + int64_t tail_ts = AV_NOPTS_VALUE; + + /* if no stream specified, pick the one that is most ahead */ + if (stream_idx < 0) { + int64_t ts = AV_NOPTS_VALUE; + + for (int i = 0; i < sq->nb_streams; i++) { + st = &sq->streams[i]; + if (st->head_ts != AV_NOPTS_VALUE && + (ts == AV_NOPTS_VALUE || + av_compare_ts(ts, sq->streams[stream_idx].tb, + st->head_ts, st->tb) < 0)) { + ts = st->head_ts; + stream_idx = i; + } + } + /* no stream has a timestamp yet -> nothing to do */ + if (stream_idx < 0) + return 0; + } + + st = &sq->streams[stream_idx]; + + /* get the chosen stream's tail timestamp */ + for (size_t i = 0; tail_ts == AV_NOPTS_VALUE && + av_fifo_peek(st->fifo, &frame, 1, i) >= 0; i++) + tail_ts = frame_ts(sq, frame); + + /* overflow triggers when the tail is over specified duration behind the head */ + if (tail_ts == AV_NOPTS_VALUE || tail_ts >= st->head_ts || + av_rescale_q(st->head_ts - tail_ts, st->tb, AV_TIME_BASE_Q) < sq->buf_size_us) + return 0; + + /* signal a fake timestamp for all streams that prevent tail_ts from being output */ + tail_ts++; + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueStream *st1 = &sq->streams[i]; + int64_t ts; + + if (st == st1 || st1->finished || + (st1->head_ts != AV_NOPTS_VALUE && + av_compare_ts(tail_ts, st->tb, st1->head_ts, st1->tb) <= 0)) + continue; + + ts = av_rescale_q(tail_ts, st->tb, st1->tb); + if (st1->head_ts != AV_NOPTS_VALUE) + ts = FFMAX(st1->head_ts + 1, ts); + + stream_update_ts(sq, i, ts); + } + + return 1; +} + +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame) +{ + SyncQueueStream *st; + SyncQueueFrame dst; + int64_t ts; + int ret; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(st->tb.num > 0 && st->tb.den > 0); + + if (frame_null(sq, frame)) { + finish_stream(sq, stream_idx); + return 0; + } + if (st->finished) + return AVERROR_EOF; + + ret = objpool_get(sq->pool, (void**)&dst); + if (ret < 0) + return ret; + + frame_move(sq, dst, frame); + + ts = frame_ts(sq, dst); + + ret = av_fifo_write(st->fifo, &dst, 1); + if (ret < 0) { + frame_move(sq, frame, dst); + objpool_release(sq->pool, (void**)&dst); + return ret; + } + + stream_update_ts(sq, stream_idx, ts); + + return 0; +} + +static int receive_for_stream(SyncQueue *sq, unsigned int stream_idx, + SyncQueueFrame frame) +{ + SyncQueueStream *st_head = sq->head_stream >= 0 ? + &sq->streams[sq->head_stream] : NULL; + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + if (av_fifo_can_read(st->fifo)) { + SyncQueueFrame peek; + int64_t ts; + int cmp = 1; + + av_fifo_peek(st->fifo, &peek, 1, 0); + ts = frame_ts(sq, peek); + + /* check if this stream's tail timestamp does not overtake + * the overall queue head */ + if (ts != AV_NOPTS_VALUE && st_head) + cmp = av_compare_ts(ts, st->tb, st_head->head_ts, st_head->tb); + + /* We can release frames that do not end after the queue head. + * Frames with no timestamps are just passed through with no conditions. + */ + if (cmp <= 0 || ts == AV_NOPTS_VALUE) { + frame_move(sq, frame, peek); + objpool_release(sq->pool, (void**)&peek); + av_fifo_drain2(st->fifo, 1); + return 0; + } + } + + return (sq->finished || (st->finished && !av_fifo_can_read(st->fifo))) ? + AVERROR_EOF : AVERROR(EAGAIN); +} + +static int receive_internal(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int nb_eof = 0; + int ret; + + /* read a frame for a specific stream */ + if (stream_idx >= 0) { + ret = receive_for_stream(sq, stream_idx, frame); + return (ret < 0) ? ret : stream_idx; + } + + /* read a frame for any stream with available output */ + for (unsigned int i = 0; i < sq->nb_streams; i++) { + ret = receive_for_stream(sq, i, frame); + if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) { + nb_eof += (ret == AVERROR_EOF); + continue; + } + return (ret < 0) ? ret : i; + } + + return (nb_eof == sq->nb_streams) ? AVERROR_EOF : AVERROR(EAGAIN); +} + +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) +{ + int ret = receive_internal(sq, stream_idx, frame); + + /* try again if the queue overflowed and triggered a fake heartbeat + * for lagging streams */ + if (ret == AVERROR(EAGAIN) && overflow_heartbeat(sq, stream_idx)) + ret = receive_internal(sq, stream_idx, frame); + + return ret; +} + +int sq_add_stream(SyncQueue *sq) +{ + SyncQueueStream *tmp, *st; + + tmp = av_realloc_array(sq->streams, sq->nb_streams + 1, sizeof(*sq->streams)); + if (!tmp) + return AVERROR(ENOMEM); + sq->streams = tmp; + + st = &sq->streams[sq->nb_streams]; + memset(st, 0, sizeof(*st)); + + st->fifo = av_fifo_alloc2(1, sizeof(SyncQueueFrame), AV_FIFO_FLAG_AUTO_GROW); + if (!st->fifo) + return AVERROR(ENOMEM); + + /* we set a valid default, so that a pathological stream that never + * receives even a real timebase (and no frames) won't stall all other + * streams forever; cf. overflow_heartbeat() */ + st->tb = (AVRational){ 1, 1 }; + st->head_ts = AV_NOPTS_VALUE; + + return sq->nb_streams++; +} + +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + av_assert0(!av_fifo_can_read(st->fifo)); + + if (st->head_ts != AV_NOPTS_VALUE) + st->head_ts = av_rescale_q(st->head_ts, st->tb, tb); + + st->tb = tb; +} + +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) +{ + SyncQueue *sq = av_mallocz(sizeof(*sq)); + + if (!sq) + return NULL; + + sq->type = type; + sq->buf_size_us = buf_size_us; + + sq->head_stream = -1; + sq->head_finished_stream = -1; + + sq->pool = (type == SYNC_QUEUE_PACKETS) ? objpool_alloc_packets() : + objpool_alloc_frames(); + if (!sq->pool) { + av_freep(&sq); + return NULL; + } + + return sq; +} + +void sq_free(SyncQueue **psq) +{ + SyncQueue *sq = *psq; + + if (!sq) + return; + + for (unsigned int i = 0; i < sq->nb_streams; i++) { + SyncQueueFrame frame; + while (av_fifo_read(sq->streams[i].fifo, &frame, 1) >= 0) + objpool_release(sq->pool, (void**)&frame); + + av_fifo_freep2(&sq->streams[i].fifo); + } + + av_freep(&sq->streams); + + objpool_free(&sq->pool); + + av_freep(psq); +} diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h new file mode 100644 index 0000000000..e08780b7bf --- /dev/null +++ b/fftools/sync_queue.h @@ -0,0 +1,100 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_SYNC_QUEUE_H +#define FFTOOLS_SYNC_QUEUE_H + +#include <stdint.h> + +#include "libavcodec/packet.h" + +#include "libavutil/frame.h" + +enum SyncQueueType { + SYNC_QUEUE_PACKETS, + SYNC_QUEUE_FRAMES, +}; + +typedef union SyncQueueFrame { + AVFrame *f; + AVPacket *p; +} SyncQueueFrame; + +#define SQFRAME(frame) ((SyncQueueFrame){ .f = (frame) }) +#define SQPKT(pkt) ((SyncQueueFrame){ .p = (pkt) }) + +typedef struct SyncQueue SyncQueue; + +/** + * Allocate a sync queue of the given type. + * + * @param buf_size_us maximum duration that will be buffered in microseconds + */ +SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us); +void sq_free(SyncQueue **sq); + +/** + * Add a new stream to the sync queue. + * + * @return + * - a non-negative stream index on success + * - a negative error code on error + */ +int sq_add_stream(SyncQueue *sq); + +/** + * Set the timebase for the stream with index stream_idx. Should be called + * before sending any frames for this stream. + */ +void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb); + +/** + * Submit a frame for the stream with index stream_idx. + * + * On success, the sync queue takes ownership of the frame and will reset the + * contents of the supplied frame. On failure, the frame remains owned by the + * caller. + * + * Sending a frame with NULL contents marks the stream as finished. + * + * @return + * - 0 on success + * - AVERROR_EOF when no more frames should be submitted for this stream + * - another a negative error code on failure + */ +int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame); + +/** + * Read a frame from the queue. + * + * @param stream_idx index of the stream to read a frame for. May be -1, then + * try to read a frame from any stream that is ready for + * output. + * @param frame output frame will be written here on success. The frame is owned + * by the caller. + * + * @return + * - a non-negative index of the stream to which the returned frame belongs + * - AVERROR(EAGAIN) when more frames need to be submitted to the queue + * - AVERROR_EOF when no more frames will be available for this stream (for any + * stream if stream_idx is -1) + * - another negative error code on failure + */ +int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame); + +#endif // FFTOOLS_SYNC_QUEUE_H diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak index 154af2fac8..38a1ae7ed5 100644 --- a/tests/fate/ffmpeg.mak +++ b/tests/fate/ffmpeg.mak @@ -105,7 +105,7 @@ FATE_SAMPLES_FFMPEG-$(call ALLYES, COLOR_FILTER, VOBSUB_DEMUXER, MATROSKA_DEMUXE fate-shortest-sub: CMD = enc_dec \ vobsub $(TARGET_SAMPLES)/sub/vobsub.idx matroska \ "-filter_complex 'color=s=1x1:rate=1:duration=400' -pix_fmt rgb24 -allow_raw_vfw 1 -c:s copy -c:v rawvideo" \ - framecrc "-map 0 -c copy -shortest" + framecrc "-map 0 -c copy -shortest -shortest_buf_duration 40" # Basic test for fix_sub_duration, which calculates duration based on the # following subtitle's pts. diff --git a/tests/ref/fate/copy-shortest1 b/tests/ref/fate/copy-shortest1 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest1 +++ b/tests/ref/fate/copy-shortest1 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/copy-shortest2 b/tests/ref/fate/copy-shortest2 index 5038973e4e..87bee4c41f 100644 --- a/tests/ref/fate/copy-shortest2 +++ b/tests/ref/fate/copy-shortest2 @@ -120,4 +120,3 @@ 0, 98304, 98304, 2048, 11182, e35a2ab846029effdbca0e43639717f2 1, 85760, 85760, 1536, 418, cf52ea7fc69e4c5bc8f75b354dfe60af 0, 100352, 100352, 2048, 1423, f480272c7d0b97834bc8ea36cceca61d -1, 87296, 87296, 1536, 418, 78ab22657a1b6c8a0e5b8612ceb8081d diff --git a/tests/ref/fate/shortest-sub b/tests/ref/fate/shortest-sub index be0922fd56..0da4ba2e95 100644 --- a/tests/ref/fate/shortest-sub +++ b/tests/ref/fate/shortest-sub @@ -1,4 +1,4 @@ 145b9b48d56f9c966bf41657f7569954 *tests/data/fate/shortest-sub.matroska 139232 tests/data/fate/shortest-sub.matroska -d71f5d359ef788ea689415bc1e4a90df *tests/data/fate/shortest-sub.out.framecrc -stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 26055 +876ac3fa52e467050ab843969d4cf343 *tests/data/fate/shortest-sub.out.framecrc +stddev:11541.12 PSNR: 15.08 MAXDIFF:22854 bytes: 2591/ 23735 -- 2.34.1
--- fftools/ffmpeg_mux.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 453ccac912..641bdb98b0 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -98,16 +98,16 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } if (pkt) { - ret = av_packet_make_refcounted(pkt); - if (ret < 0) - return ret; + ret = av_packet_make_refcounted(pkt); + if (ret < 0) + return ret; - tmp_pkt = av_packet_alloc(); - if (!tmp_pkt) - return AVERROR(ENOMEM); + tmp_pkt = av_packet_alloc(); + if (!tmp_pkt) + return AVERROR(ENOMEM); - av_packet_move_ref(tmp_pkt, pkt); - ms->muxing_queue_data_size += tmp_pkt->size; + av_packet_move_ref(tmp_pkt, pkt); + ms->muxing_queue_data_size += tmp_pkt->size; } av_fifo_write(ms->muxing_queue, &tmp_pkt, 1); @@ -236,20 +236,20 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) int ret; if (pkt) { - /* - * Audio encoders may split the packets -- #frames in != #packets out. - * But there is no reordering, so we can limit the number of output packets - * by simply dropping them here. - * Counting encoded video frames needs to be done separately because of - * reordering, see do_video_out(). - */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { - if (ost->frame_number >= ost->max_frames) { - av_packet_unref(pkt); - return; + /* + * Audio encoders may split the packets -- #frames in != #packets out. + * But there is no reordering, so we can limit the number of output packets + * by simply dropping them here. + * Counting encoded video frames needs to be done separately because of + * reordering, see do_video_out(). + */ + if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { + if (ost->frame_number >= ost->max_frames) { + av_packet_unref(pkt); + return; + } + ost->frame_number++; } - ost->frame_number++; - } } if (of->mux->header_written) { -- 2.34.1
Same issues apply to it as to -shortest. Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable. - mkv-1242 The test streamcopies one video and one audio stream, video is limited to 11 frames. The new result shortens the audio stream so that it is not longer than the video. --- fftools/ffmpeg.c | 6 ------ fftools/ffmpeg_mux.c | 10 +--------- fftools/ffmpeg_opt.c | 18 ++++++++++++++--- fftools/sync_queue.c | 20 +++++++++++++++++++ fftools/sync_queue.h | 7 +++++++ tests/ref/fate/matroska-flac-extradata-update | 16 +++++---------- tests/ref/fate/mkv-1242 | 3 --- 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 1a14637ece..a471fa0f8f 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3467,12 +3467,6 @@ static int need_output(void) if (ost->finished || of_finished(of)) continue; - if (ost->frame_number >= ost->max_frames) { - int j; - for (j = 0; j < of->ctx->nb_streams; j++) - close_output_stream(output_streams[of->ost_index + j]); - continue; - } return 1; } diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 641bdb98b0..56444770bf 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -237,19 +237,11 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) if (pkt) { /* - * Audio encoders may split the packets -- #frames in != #packets out. - * But there is no reordering, so we can limit the number of output packets - * by simply dropping them here. * Counting encoded video frames needs to be done separately because of * reordering, see do_video_out(). */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { - if (ost->frame_number >= ost->max_frames) { - av_packet_unref(pkt); - return; - } + if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) ost->frame_number++; - } } if (of->mux->header_written) { diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 2cd4d42f2a..22eb558ff3 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2331,6 +2331,7 @@ static int init_complex_filters(void) static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) { int nb_av_enc = 0, nb_interleaved = 0; + int limit_frames = 0, limit_frames_av_enc = 0; #define IS_AV_ENC(ost, type) \ (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) @@ -2345,14 +2346,19 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si nb_interleaved += IS_INTERLEAVED(type); nb_av_enc += IS_AV_ENC(ost, type); + + limit_frames |= ost->max_frames < INT64_MAX; + limit_frames_av_enc |= (ost->max_frames < INT64_MAX) && IS_AV_ENC(ost, type); } - if (!(nb_interleaved > 1 && of->shortest)) + if (!((nb_interleaved > 1 && of->shortest) || + (nb_interleaved > 0 && limit_frames))) return 0; - /* if we have more than one encoded audio/video streams, then we + /* if we have more than one encoded audio/video streams, or at least + * one encoded audio/video stream is frame-limited, then we * synchronize them before encoding */ - if (nb_av_enc > 1) { + if ((of->shortest && nb_av_enc > 1) || limit_frames_av_enc) { of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); if (!of->sq_encode) return AVERROR(ENOMEM); @@ -2371,6 +2377,9 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si ost->sq_frame = av_frame_alloc(); if (!ost->sq_frame) return AVERROR(ENOMEM); + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_encode, ost->sq_idx_encode, ost->max_frames); } } @@ -2391,6 +2400,9 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si ost->sq_idx_mux = sq_add_stream(of->sq_mux); if (ost->sq_idx_mux < 0) return ost->sq_idx_mux; + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_mux, ost->sq_idx_mux, ost->max_frames); } } diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c index 49f0600b6c..08c3cf4f2a 100644 --- a/fftools/sync_queue.c +++ b/fftools/sync_queue.c @@ -36,6 +36,9 @@ typedef struct SyncQueueStream { int64_t head_ts; /* no more frames will be sent for this stream */ int finished; + + uint64_t frames_sent; + uint64_t frames_max; } SyncQueueStream; struct SyncQueue { @@ -264,6 +267,10 @@ int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame) stream_update_ts(sq, stream_idx, ts); + st->frames_sent++; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); + return 0; } @@ -362,6 +369,7 @@ int sq_add_stream(SyncQueue *sq) * streams forever; cf. overflow_heartbeat() */ st->tb = (AVRational){ 1, 1 }; st->head_ts = AV_NOPTS_VALUE; + st->frames_max = UINT64_MAX; return sq->nb_streams++; } @@ -381,6 +389,18 @@ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) st->tb = tb; } +void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, uint64_t frames) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + st->frames_max = frames; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); +} + SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) { SyncQueue *sq = av_mallocz(sizeof(*sq)); diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h index e08780b7bf..7beddf8c4a 100644 --- a/fftools/sync_queue.h +++ b/fftools/sync_queue.h @@ -63,6 +63,13 @@ int sq_add_stream(SyncQueue *sq); */ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb); +/** + * Limit the number of output frames for stream with index stream_idx + * to max_frames. + */ +void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, + uint64_t max_frames); + /** * Submit a frame for the stream with index stream_idx. * diff --git a/tests/ref/fate/matroska-flac-extradata-update b/tests/ref/fate/matroska-flac-extradata-update index b0276f734d..84dcc9c1d0 100644 --- a/tests/ref/fate/matroska-flac-extradata-update +++ b/tests/ref/fate/matroska-flac-extradata-update @@ -1,8 +1,8 @@ -56ff5763fd81ad3bc02c22402cd685e2 *tests/data/fate/matroska-flac-extradata-update.matroska -2008 tests/data/fate/matroska-flac-extradata-update.matroska -#extradata 0: 34, 0x7acb09e7 -#extradata 1: 34, 0x7acb09e7 -#extradata 2: 34, 0x443402dd +8ec02dffd603f44e08b2ae3b81a0d5a0 *tests/data/fate/matroska-flac-extradata-update.matroska +1816 tests/data/fate/matroska-flac-extradata-update.matroska +#extradata 0: 34, 0x93650c81 +#extradata 1: 34, 0x93650c81 +#extradata 2: 34, 0x93650c81 #tb 0: 1/1000 #media_type 0: audio #codec_id 0: flac @@ -42,9 +42,3 @@ 0, 672, 672, 96, 26, 0x50dd042e 1, 672, 672, 96, 26, 0x50dd042e 2, 672, 672, 96, 26, 0x50dd042e -0, 768, 768, 96, 26, 0x53de0499 -1, 768, 768, 96, 26, 0x53de0499 -0, 864, 864, 96, 26, 0x53df04b4 -1, 864, 864, 96, 26, 0x53df04b4 -0, 960, 960, 42, 26, 0x5740044b -1, 960, 960, 42, 26, 0x5740044b diff --git a/tests/ref/fate/mkv-1242 b/tests/ref/fate/mkv-1242 index e025701093..1d1a227832 100644 --- a/tests/ref/fate/mkv-1242 +++ b/tests/ref/fate/mkv-1242 @@ -42,6 +42,3 @@ 1, 383, 383, 21, 325, 0xcd7a9fd6 1, 404, 404, 22, 359, 0x6edeb91c 1, 426, 426, 21, 333, 0xb8999fb7 -1, 447, 447, 21, 317, 0xf2589e1a -1, 468, 468, 21, 319, 0x82ed9572 -1, 489, 489, 22, 473, 0xea54e696 -- 2.34.1
Anton Khirnov:
Same issues apply to it as to -shortest.
Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable.
The point of the test was that only one stream is limited so that one can see the extradata update directly in the test result: The unlimited streams have a different extradata than the limited stream (because said extradata contains an md5 of the decoded data). So it is expected that the extradata hashes of the first two streams coincide and differ from the hash of the last stream. (The current test results btw show an imperfection: The extradata of the last stream is not updated, as the encoder is not flushed (or the flush packet does not arrive at the muxer). Fixing this (as seems to be the case) is good.)
- mkv-1242 The test streamcopies one video and one audio stream, video is limited to 11 frames. The new result shortens the audio stream so that it is not longer than the video. --- fftools/ffmpeg.c | 6 ------ fftools/ffmpeg_mux.c | 10 +--------- fftools/ffmpeg_opt.c | 18 ++++++++++++++--- fftools/sync_queue.c | 20 +++++++++++++++++++ fftools/sync_queue.h | 7 +++++++ tests/ref/fate/matroska-flac-extradata-update | 16 +++++---------- tests/ref/fate/mkv-1242 | 3 --- 7 files changed, 48 insertions(+), 32 deletions(-)
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 1a14637ece..a471fa0f8f 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3467,12 +3467,6 @@ static int need_output(void)
if (ost->finished || of_finished(of)) continue; - if (ost->frame_number >= ost->max_frames) { - int j; - for (j = 0; j < of->ctx->nb_streams; j++) - close_output_stream(output_streams[of->ost_index + j]); - continue; - }
return 1; } diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 641bdb98b0..56444770bf 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -237,19 +237,11 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost)
if (pkt) { /* - * Audio encoders may split the packets -- #frames in != #packets out. - * But there is no reordering, so we can limit the number of output packets - * by simply dropping them here. * Counting encoded video frames needs to be done separately because of * reordering, see do_video_out(). */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { - if (ost->frame_number >= ost->max_frames) { - av_packet_unref(pkt); - return; - } + if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) ost->frame_number++; - } }
if (of->mux->header_written) { diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 2cd4d42f2a..22eb558ff3 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2331,6 +2331,7 @@ static int init_complex_filters(void) static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) { int nb_av_enc = 0, nb_interleaved = 0; + int limit_frames = 0, limit_frames_av_enc = 0;
#define IS_AV_ENC(ost, type) \ (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) @@ -2345,14 +2346,19 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si
nb_interleaved += IS_INTERLEAVED(type); nb_av_enc += IS_AV_ENC(ost, type); + + limit_frames |= ost->max_frames < INT64_MAX; + limit_frames_av_enc |= (ost->max_frames < INT64_MAX) && IS_AV_ENC(ost, type); }
- if (!(nb_interleaved > 1 && of->shortest)) + if (!((nb_interleaved > 1 && of->shortest) || + (nb_interleaved > 0 && limit_frames))) return 0;
- /* if we have more than one encoded audio/video streams, then we + /* if we have more than one encoded audio/video streams, or at least + * one encoded audio/video stream is frame-limited, then we * synchronize them before encoding */ - if (nb_av_enc > 1) { + if ((of->shortest && nb_av_enc > 1) || limit_frames_av_enc) { of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); if (!of->sq_encode) return AVERROR(ENOMEM); @@ -2371,6 +2377,9 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si ost->sq_frame = av_frame_alloc(); if (!ost->sq_frame) return AVERROR(ENOMEM); + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_encode, ost->sq_idx_encode, ost->max_frames); } }
@@ -2391,6 +2400,9 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si ost->sq_idx_mux = sq_add_stream(of->sq_mux); if (ost->sq_idx_mux < 0) return ost->sq_idx_mux; + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_mux, ost->sq_idx_mux, ost->max_frames); } }
diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c index 49f0600b6c..08c3cf4f2a 100644 --- a/fftools/sync_queue.c +++ b/fftools/sync_queue.c @@ -36,6 +36,9 @@ typedef struct SyncQueueStream { int64_t head_ts; /* no more frames will be sent for this stream */ int finished; + + uint64_t frames_sent; + uint64_t frames_max; } SyncQueueStream;
struct SyncQueue { @@ -264,6 +267,10 @@ int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame)
stream_update_ts(sq, stream_idx, ts);
+ st->frames_sent++; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); + return 0; }
@@ -362,6 +369,7 @@ int sq_add_stream(SyncQueue *sq) * streams forever; cf. overflow_heartbeat() */ st->tb = (AVRational){ 1, 1 }; st->head_ts = AV_NOPTS_VALUE; + st->frames_max = UINT64_MAX;
return sq->nb_streams++; } @@ -381,6 +389,18 @@ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) st->tb = tb; }
+void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, uint64_t frames) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + st->frames_max = frames; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); +} + SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) { SyncQueue *sq = av_mallocz(sizeof(*sq)); diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h index e08780b7bf..7beddf8c4a 100644 --- a/fftools/sync_queue.h +++ b/fftools/sync_queue.h @@ -63,6 +63,13 @@ int sq_add_stream(SyncQueue *sq); */ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb);
+/** + * Limit the number of output frames for stream with index stream_idx + * to max_frames. + */ +void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, + uint64_t max_frames); + /** * Submit a frame for the stream with index stream_idx. * diff --git a/tests/ref/fate/matroska-flac-extradata-update b/tests/ref/fate/matroska-flac-extradata-update index b0276f734d..84dcc9c1d0 100644 --- a/tests/ref/fate/matroska-flac-extradata-update +++ b/tests/ref/fate/matroska-flac-extradata-update @@ -1,8 +1,8 @@ -56ff5763fd81ad3bc02c22402cd685e2 *tests/data/fate/matroska-flac-extradata-update.matroska -2008 tests/data/fate/matroska-flac-extradata-update.matroska -#extradata 0: 34, 0x7acb09e7 -#extradata 1: 34, 0x7acb09e7 -#extradata 2: 34, 0x443402dd +8ec02dffd603f44e08b2ae3b81a0d5a0 *tests/data/fate/matroska-flac-extradata-update.matroska +1816 tests/data/fate/matroska-flac-extradata-update.matroska +#extradata 0: 34, 0x93650c81 +#extradata 1: 34, 0x93650c81 +#extradata 2: 34, 0x93650c81 #tb 0: 1/1000 #media_type 0: audio #codec_id 0: flac @@ -42,9 +42,3 @@ 0, 672, 672, 96, 26, 0x50dd042e 1, 672, 672, 96, 26, 0x50dd042e 2, 672, 672, 96, 26, 0x50dd042e -0, 768, 768, 96, 26, 0x53de0499 -1, 768, 768, 96, 26, 0x53de0499 -0, 864, 864, 96, 26, 0x53df04b4 -1, 864, 864, 96, 26, 0x53df04b4 -0, 960, 960, 42, 26, 0x5740044b -1, 960, 960, 42, 26, 0x5740044b diff --git a/tests/ref/fate/mkv-1242 b/tests/ref/fate/mkv-1242 index e025701093..1d1a227832 100644 --- a/tests/ref/fate/mkv-1242 +++ b/tests/ref/fate/mkv-1242 @@ -42,6 +42,3 @@ 1, 383, 383, 21, 325, 0xcd7a9fd6 1, 404, 404, 22, 359, 0x6edeb91c 1, 426, 426, 21, 333, 0xb8999fb7 -1, 447, 447, 21, 317, 0xf2589e1a -1, 468, 468, 21, 319, 0x82ed9572 -1, 489, 489, 22, 473, 0xea54e696
Quoting Andreas Rheinhardt (2022-06-16 22:33:46)
Anton Khirnov:
Same issues apply to it as to -shortest.
Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable.
The point of the test was that only one stream is limited so that one can see the extradata update directly in the test result: The unlimited streams have a different extradata than the limited stream (because said extradata contains an md5 of the decoded data). So it is expected that the extradata hashes of the first two streams coincide and differ from the hash of the last stream.
Right, but my point is that the amount of data that ends up in those unlimited streams is largely an accident of how the code happens to work currently and is not guaranteed by anything. Are you suggesting any specific changes to the test or the patch? E.g. the atrim filter could be used to replicate previous behaviour if you'd like to keep it.
(The current test results btw show an imperfection: The extradata of the last stream is not updated, as the encoder is not flushed (or the flush packet does not arrive at the muxer). Fixing this (as seems to be the case) is good.)
Correct - frame-limiting is now done before sending frames to the encoder, so all packets, including the one from flushing the encoder, get to the muxer. -- Anton Khirnov
Anton Khirnov:
Quoting Andreas Rheinhardt (2022-06-16 22:33:46)
Anton Khirnov:
Same issues apply to it as to -shortest.
Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable.
The point of the test was that only one stream is limited so that one can see the extradata update directly in the test result: The unlimited streams have a different extradata than the limited stream (because said extradata contains an md5 of the decoded data). So it is expected that the extradata hashes of the first two streams coincide and differ from the hash of the last stream.
Right, but my point is that the amount of data that ends up in those unlimited streams is largely an accident of how the code happens to work currently and is not guaranteed by anything.
The documentation of frames reads: "-frames[:stream_specifier] framecount (output,per-stream) Stop writing to the stream after framecount frames." It does not say that the other streams end after one stream has reached its framecount. So it is guaranteed by the documentation that the other streams don't end prematurely. (Why do you think that this is an accident?)
Are you suggesting any specific changes to the test or the patch? E.g. the atrim filter could be used to replicate previous behaviour if you'd like to keep it.
(The current test results btw show an imperfection: The extradata of the last stream is not updated, as the encoder is not flushed (or the flush packet does not arrive at the muxer). Fixing this (as seems to be the case) is good.)
Correct - frame-limiting is now done before sending frames to the encoder, so all packets, including the one from flushing the encoder, get to the muxer.
Quoting Andreas Rheinhardt (2022-06-22 10:27:30)
Anton Khirnov:
Quoting Andreas Rheinhardt (2022-06-16 22:33:46)
Anton Khirnov:
Same issues apply to it as to -shortest.
Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable.
The point of the test was that only one stream is limited so that one can see the extradata update directly in the test result: The unlimited streams have a different extradata than the limited stream (because said extradata contains an md5 of the decoded data). So it is expected that the extradata hashes of the first two streams coincide and differ from the hash of the last stream.
Right, but my point is that the amount of data that ends up in those unlimited streams is largely an accident of how the code happens to work currently and is not guaranteed by anything.
The documentation of frames reads: "-frames[:stream_specifier] framecount (output,per-stream) Stop writing to the stream after framecount frames." It does not say that the other streams end after one stream has reached its framecount. So it is guaranteed by the documentation that the other streams don't end prematurely. (Why do you think that this is an accident?)
1) Documentation not saying what happens to the other streams does not imply any guarantees IMO, it implies a lack of any guarantees. 2) Documentation for ffmpeg has always been less than fully descriptive (to put it extremely mildly), so I would not rely on it as the ultimate source of truth. 3) The original commit adding this option in 2004 would terminate ALL output to the file on reaching the specified frame limit, not just the affected stream. This was broken by me in 2f51ec2b9438e211f5b8abb2fcf5d8be678e7e8c, because need_output() terminates early and does not check max_frames for further streams if an earlier stream still allows output. I would consider this a bug, because it makes no sense to treat the option differently based on stream ordering. Presumably nobody noticed because nobody relies on this option producing consistent output with multiple streams. This is also the reason the test outputs all frames for the first two streams. If you changed "-frames:a:2 8" to "-frames:a:0 8", you would get: Output stream #0:0 (audio): 8 frames encoded (36864 samples); 8 packets muxed (208 bytes); Output stream #0:1 (audio): 7 frames encoded (32256 samples); 8 packets muxed (182 bytes); Output stream #0:2 (audio): 8 frames encoded (36864 samples); 9 packets muxed (208 bytes); -- Anton Khirnov
On Thu, Jun 16, 2022 at 09:55:23PM +0200, Anton Khirnov wrote:
Same issues apply to it as to -shortest.
Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable. - mkv-1242 The test streamcopies one video and one audio stream, video is limited to 11 frames. The new result shortens the audio stream so that it is not longer than the video.
seems to break: ./ffmpeg -i tspacket_size_changeback.ts -vframes 2 -y -qscale 2 sizechangback.avi 0, 0, 0, 1, 81175, 0xa6efb96202 bitrate= 114.8kbits/s speed=N/A 1, 0, 0, 1, 384, 0xdfabb6ef 1, 1, 1, 1, 384, 0x5abbbe00 1, 2, 2, 1, 384, 0x9930b9a2 1, 3, 3, 1, 384, 0x7c0cb08d 1, 4, 4, 1, 384, 0x0c54b25e 1, 5, 5, 1, 384, 0x8dafb427 1, 6, 6, 1, 384, 0x6c7eb911 1, 7, 7, 1, 384, 0x336fd87e 0, 51, 51, 1, 43825, 0x8dc20601, F=0x0 frame= 2 fps=0.0 q=-1.0 Lsize= 1kB time=00:00:01.04 bitrate= 6.7kbits/s speed=1.03e+04x video:122kB audio:3kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: unknown after: 1, 0, 0, 1, 384, 0xdfabb6ef02 bitrate= 84.0kbits/s speed=N/A 1, 1, 1, 1, 384, 0x5abbbe00 1, 2, 2, 1, 384, 0x9930b9a2 1, 3, 3, 1, 384, 0x7c0cb08d 1, 4, 4, 1, 384, 0x0c54b25e 1, 5, 5, 1, 384, 0x8dafb427 1, 6, 6, 1, 384, 0x6c7eb911 1, 7, 7, 1, 384, 0x336fd87e frame= 0 fps=0.0 q=-1.0 Lsize= 1kB time=00:00:00.19 bitrate= 29.8kbits/s speed=4.47e+03x video:0kB audio:3kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: unknown will mail the file privatly [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB The educated differ from the uneducated as much as the living from the dead. -- Aristotle
Same issues apply to it as to -shortest. Changes the results of the following tests: - matroska-flac-extradata-update The test reencodes two input FLAC streams into three output FLAC streams. The last output stream is limited to 8 frames. The current code results in the first two output streams having 12 frames, after this commit all three streams have 8 frames and are the same length. This new result is better, since it is predictable. - mkv-1242 The test streamcopies one video and one audio stream, video is limited to 11 frames. The new result shortens the audio stream so that it is not longer than the video. --- fftools/ffmpeg.c | 6 ---- fftools/ffmpeg_mux.c | 10 +----- fftools/ffmpeg_opt.c | 24 +++++++++++--- fftools/sync_queue.c | 33 ++++++++++++++++--- fftools/sync_queue.h | 11 ++++++- tests/ref/fate/matroska-flac-extradata-update | 16 +++------ tests/ref/fate/mkv-1242 | 3 -- 7 files changed, 63 insertions(+), 40 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 15e350f8fd..503a148103 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -3467,12 +3467,6 @@ static int need_output(void) if (ost->finished || of_finished(of)) continue; - if (ost->frame_number >= ost->max_frames) { - int j; - for (j = 0; j < of->ctx->nb_streams; j++) - close_output_stream(output_streams[of->ost_index + j]); - continue; - } return 1; } diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 641bdb98b0..56444770bf 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -237,19 +237,11 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) if (pkt) { /* - * Audio encoders may split the packets -- #frames in != #packets out. - * But there is no reordering, so we can limit the number of output packets - * by simply dropping them here. * Counting encoded video frames needs to be done separately because of * reordering, see do_video_out(). */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) { - if (ost->frame_number >= ost->max_frames) { - av_packet_unref(pkt); - return; - } + if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) ost->frame_number++; - } } if (of->mux->header_written) { diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index be88743463..12f5d16157 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2336,6 +2336,7 @@ static int init_complex_filters(void) static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_size_us) { int nb_av_enc = 0, nb_interleaved = 0; + int limit_frames = 0, limit_frames_av_enc = 0; #define IS_AV_ENC(ost, type) \ (ost->encoding_needed && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)) @@ -2350,14 +2351,19 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si nb_interleaved += IS_INTERLEAVED(type); nb_av_enc += IS_AV_ENC(ost, type); + + limit_frames |= ost->max_frames < INT64_MAX; + limit_frames_av_enc |= (ost->max_frames < INT64_MAX) && IS_AV_ENC(ost, type); } - if (!(nb_interleaved > 1 && of->shortest)) + if (!((nb_interleaved > 1 && of->shortest) || + (nb_interleaved > 0 && limit_frames))) return 0; - /* if we have more than one encoded audio/video streams, then we + /* if we have more than one encoded audio/video streams, or at least + * one encoded audio/video stream is frame-limited, then we * synchronize them before encoding */ - if (nb_av_enc > 1) { + if ((of->shortest && nb_av_enc > 1) || limit_frames_av_enc) { of->sq_encode = sq_alloc(SYNC_QUEUE_FRAMES, buf_size_us); if (!of->sq_encode) return AVERROR(ENOMEM); @@ -2369,13 +2375,17 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si if (!IS_AV_ENC(ost, type)) continue; - ost->sq_idx_encode = sq_add_stream(of->sq_encode); + ost->sq_idx_encode = sq_add_stream(of->sq_encode, + of->shortest || ost->max_frames < INT64_MAX); if (ost->sq_idx_encode < 0) return ost->sq_idx_encode; ost->sq_frame = av_frame_alloc(); if (!ost->sq_frame) return AVERROR(ENOMEM); + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_encode, ost->sq_idx_encode, ost->max_frames); } } @@ -2393,9 +2403,13 @@ static int setup_sync_queues(OutputFile *of, AVFormatContext *oc, int64_t buf_si if (!IS_INTERLEAVED(type)) continue; - ost->sq_idx_mux = sq_add_stream(of->sq_mux); + ost->sq_idx_mux = sq_add_stream(of->sq_mux, + of->shortest || ost->max_frames < INT64_MAX); if (ost->sq_idx_mux < 0) return ost->sq_idx_mux; + + if (ost->max_frames != INT64_MAX) + sq_limit_frames(of->sq_mux, ost->sq_idx_mux, ost->max_frames); } } diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c index ab654ca790..3143d12cf1 100644 --- a/fftools/sync_queue.c +++ b/fftools/sync_queue.c @@ -34,8 +34,12 @@ typedef struct SyncQueueStream { /* stream head: largest timestamp seen */ int64_t head_ts; + int limiting; /* no more frames will be sent for this stream */ int finished; + + uint64_t frames_sent; + uint64_t frames_max; } SyncQueueStream; struct SyncQueue { @@ -86,7 +90,7 @@ static void finish_stream(SyncQueue *sq, unsigned int stream_idx) st->finished = 1; - if (st->head_ts != AV_NOPTS_VALUE) { + if (st->limiting && st->head_ts != AV_NOPTS_VALUE) { /* check if this stream is the new finished head */ if (sq->head_finished_stream < 0 || av_compare_ts(st->head_ts, st->tb, @@ -121,7 +125,7 @@ static void queue_head_update(SyncQueue *sq) * the queue head */ for (unsigned int i = 0; i < sq->nb_streams; i++) { SyncQueueStream *st = &sq->streams[i]; - if (st->head_ts == AV_NOPTS_VALUE) + if (st->limiting && st->head_ts == AV_NOPTS_VALUE) return; } @@ -132,7 +136,7 @@ static void queue_head_update(SyncQueue *sq) for (unsigned int i = 0; i < sq->nb_streams; i++) { SyncQueueStream *st_head = &sq->streams[sq->head_stream]; SyncQueueStream *st_other = &sq->streams[i]; - if (st_other->head_ts != AV_NOPTS_VALUE && + if (st_other->limiting && st_other->head_ts != AV_NOPTS_VALUE && av_compare_ts(st_other->head_ts, st_other->tb, st_head->head_ts, st_head->tb) < 0) sq->head_stream = i; @@ -159,7 +163,8 @@ static void stream_update_ts(SyncQueue *sq, unsigned int stream_idx, int64_t ts) finish_stream(sq, stream_idx); /* update the overall head timestamp if it could have changed */ - if (sq->head_stream < 0 || sq->head_stream == stream_idx) + if (st->limiting && + (sq->head_stream < 0 || sq->head_stream == stream_idx)) queue_head_update(sq); } @@ -262,6 +267,10 @@ int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame) stream_update_ts(sq, stream_idx, ts); + st->frames_sent++; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); + return 0; } @@ -339,7 +348,7 @@ int sq_receive(SyncQueue *sq, int stream_idx, SyncQueueFrame frame) return ret; } -int sq_add_stream(SyncQueue *sq) +int sq_add_stream(SyncQueue *sq, int limiting) { SyncQueueStream *tmp, *st; @@ -360,6 +369,8 @@ int sq_add_stream(SyncQueue *sq) * streams forever; cf. overflow_heartbeat() */ st->tb = (AVRational){ 1, 1 }; st->head_ts = AV_NOPTS_VALUE; + st->frames_max = UINT64_MAX; + st->limiting = limiting; return sq->nb_streams++; } @@ -379,6 +390,18 @@ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb) st->tb = tb; } +void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, uint64_t frames) +{ + SyncQueueStream *st; + + av_assert0(stream_idx < sq->nb_streams); + st = &sq->streams[stream_idx]; + + st->frames_max = frames; + if (st->frames_sent >= st->frames_max) + finish_stream(sq, stream_idx); +} + SyncQueue *sq_alloc(enum SyncQueueType type, int64_t buf_size_us) { SyncQueue *sq = av_mallocz(sizeof(*sq)); diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h index e08780b7bf..3f823ff0d9 100644 --- a/fftools/sync_queue.h +++ b/fftools/sync_queue.h @@ -51,11 +51,13 @@ void sq_free(SyncQueue **sq); /** * Add a new stream to the sync queue. * + * @param limiting whether the stream is limiting, i.e. no other stream can be + * longer than this one * @return * - a non-negative stream index on success * - a negative error code on error */ -int sq_add_stream(SyncQueue *sq); +int sq_add_stream(SyncQueue *sq, int limiting); /** * Set the timebase for the stream with index stream_idx. Should be called @@ -63,6 +65,13 @@ int sq_add_stream(SyncQueue *sq); */ void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb); +/** + * Limit the number of output frames for stream with index stream_idx + * to max_frames. + */ +void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, + uint64_t max_frames); + /** * Submit a frame for the stream with index stream_idx. * diff --git a/tests/ref/fate/matroska-flac-extradata-update b/tests/ref/fate/matroska-flac-extradata-update index e8812f51b5..d5814925f5 100644 --- a/tests/ref/fate/matroska-flac-extradata-update +++ b/tests/ref/fate/matroska-flac-extradata-update @@ -1,8 +1,8 @@ -732446e97bae29037ff0cd9963d4ac08 *tests/data/fate/matroska-flac-extradata-update.matroska -1987 tests/data/fate/matroska-flac-extradata-update.matroska -#extradata 0: 34, 0x7acb09e7 -#extradata 1: 34, 0x7acb09e7 -#extradata 2: 34, 0x443402dd +28bc0ded5dc520d955caf29db80d35da *tests/data/fate/matroska-flac-extradata-update.matroska +1795 tests/data/fate/matroska-flac-extradata-update.matroska +#extradata 0: 34, 0x93650c81 +#extradata 1: 34, 0x93650c81 +#extradata 2: 34, 0x93650c81 #tb 0: 1/1000 #media_type 0: audio #codec_id 0: flac @@ -42,9 +42,3 @@ 0, 672, 672, 96, 26, 0x50dd042e 1, 672, 672, 96, 26, 0x50dd042e 2, 672, 672, 96, 26, 0x50dd042e -0, 768, 768, 96, 26, 0x53de0499 -1, 768, 768, 96, 26, 0x53de0499 -0, 864, 864, 96, 26, 0x53df04b4 -1, 864, 864, 96, 26, 0x53df04b4 -0, 960, 960, 42, 26, 0x5740044b -1, 960, 960, 42, 26, 0x5740044b diff --git a/tests/ref/fate/mkv-1242 b/tests/ref/fate/mkv-1242 index e025701093..1d1a227832 100644 --- a/tests/ref/fate/mkv-1242 +++ b/tests/ref/fate/mkv-1242 @@ -42,6 +42,3 @@ 1, 383, 383, 21, 325, 0xcd7a9fd6 1, 404, 404, 22, 359, 0x6edeb91c 1, 426, 426, 21, 333, 0xb8999fb7 -1, 447, 447, 21, 317, 0xf2589e1a -1, 468, 468, 21, 319, 0x82ed9572 -1, 489, 489, 22, 473, 0xea54e696 -- 2.34.1
This field means different things when the video is encoded (number of frames emitted to the encoding sync queue/encoder by the video sync code) or copied (number of packets sent to the muxer sync queue). Print the value of packets_written instead, which means the same thing in both cases. It is also more accurate, since packets may be dropped by the sync queue or bitstream filters. --- fftools/ffmpeg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index a471fa0f8f..beb07928bf 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1585,7 +1585,7 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti if (!vid && enc->codec_type == AVMEDIA_TYPE_VIDEO) { float fps; - frame_number = ost->frame_number; + frame_number = ost->packets_written; fps = t > 1 ? frame_number / t : 0; av_bprintf(&buf, "frame=%5d fps=%3.*f q=%3.1f ", frame_number, fps < 9.95, fps, q); -- 2.34.1
It is unused otherwise. Rename the field to vsync_frame_number to better reflect its current purpose. --- fftools/ffmpeg.c | 10 +++++----- fftools/ffmpeg.h | 3 ++- fftools/ffmpeg_mux.c | 10 ---------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index beb07928bf..ab3665e53c 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1176,7 +1176,7 @@ static void do_video_out(OutputFile *of, switch (ost->vsync_method) { case VSYNC_VSCFR: - if (ost->frame_number == 0 && delta0 >= 0.5) { + if (ost->vsync_frame_number == 0 && delta0 >= 0.5) { av_log(NULL, AV_LOG_DEBUG, "Not duplicating %d initial frames\n", (int)lrintf(delta0)); delta = duration; delta0 = 0; @@ -1184,7 +1184,7 @@ static void do_video_out(OutputFile *of, } case VSYNC_CFR: // FIXME set to 0.5 after we fix some dts/pts bugs like in avidec.c - if (frame_drop_threshold && delta < frame_drop_threshold && ost->frame_number) { + if (frame_drop_threshold && delta < frame_drop_threshold && ost->vsync_frame_number) { nb_frames = 0; } else if (delta < -1.1) nb_frames = 0; @@ -1214,7 +1214,7 @@ static void do_video_out(OutputFile *of, * But there may be reordering, so we can't throw away frames on encoder * flush, we need to limit them here, before they go into encoder. */ - nb_frames = FFMIN(nb_frames, ost->max_frames - ost->frame_number); + nb_frames = FFMIN(nb_frames, ost->max_frames - ost->vsync_frame_number); nb0_frames = FFMIN(nb0_frames, nb_frames); memmove(ost->last_nb0_frames + 1, @@ -1226,7 +1226,7 @@ static void do_video_out(OutputFile *of, nb_frames_drop++; av_log(NULL, AV_LOG_VERBOSE, "*** dropping frame %d from stream %d at ts %"PRId64"\n", - ost->frame_number, ost->st->index, ost->last_frame->pts); + ost->vsync_frame_number, ost->st->index, ost->last_frame->pts); } if (nb_frames > (nb0_frames && ost->last_dropped) + (nb_frames > nb0_frames)) { if (nb_frames > dts_error_threshold * 30) { @@ -1320,7 +1320,7 @@ static void do_video_out(OutputFile *of, exit_program(1); ost->sync_opts++; - ost->frame_number++; + ost->vsync_frame_number++; } av_frame_unref(ost->last_frame); diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 106580adb2..3d55370d3a 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -460,7 +460,8 @@ typedef struct OutputStream { int source_index; /* InputStream index */ AVStream *st; /* stream in the output file */ int encoding_needed; /* true if encoding needed for this stream */ - int frame_number; + /* number of frames emitted by the video-encoding sync code */ + int vsync_frame_number; /* input pts and corresponding output pts for A/V sync */ struct InputStream *sync_ist; /* input stream to sync against */ diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 56444770bf..87c7bdc4e3 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -232,18 +232,8 @@ static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) { - AVStream *st = ost->st; int ret; - if (pkt) { - /* - * Counting encoded video frames needs to be done separately because of - * reordering, see do_video_out(). - */ - if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed)) - ost->frame_number++; - } - if (of->mux->header_written) { submit_packet(of, ost, pkt); } else { -- 2.34.1
Since the muxer will operate in a separate thread in the future, the muxer context should not be accessed from the outside. --- fftools/ffmpeg.c | 4 +-- fftools/ffmpeg.h | 6 ++-- fftools/ffmpeg_filter.c | 6 ++-- fftools/ffmpeg_mux.c | 80 +++++++++++++++++++++++++---------------- fftools/ffmpeg_opt.c | 6 ++-- 5 files changed, 62 insertions(+), 40 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index ab3665e53c..4321652848 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1496,9 +1496,9 @@ static void print_final_stats(int64_t total_size) uint64_t total_packets = 0, total_size = 0; av_log(NULL, AV_LOG_VERBOSE, "Output file #%d (%s):\n", - i, of->ctx->url); + i, of->url); - for (j = 0; j < of->ctx->nb_streams; j++) { + for (j = 0; j < of->nb_streams; j++) { OutputStream *ost = output_streams[of->ost_index + j]; enum AVMediaType type = ost->enc_ctx->codec_type; diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 3d55370d3a..c757f72c0e 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -589,11 +589,12 @@ typedef struct OutputFile { Muxer *mux; const AVOutputFormat *format; + const char *url; SyncQueue *sq_encode; SyncQueue *sq_mux; - AVFormatContext *ctx; + int nb_streams; int ost_index; /* index of the first stream in output_streams */ int64_t recording_time; ///< desired length of the resulting file in microseconds == AV_TIME_BASE units int64_t start_time; ///< start time in microseconds == AV_TIME_BASE units @@ -697,7 +698,8 @@ int hw_device_setup_for_filter(FilterGraph *fg); int hwaccel_decode_init(AVCodecContext *avctx); -int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize); +int of_muxer_init(OutputFile *of, AVFormatContext *fc, + AVDictionary *opts, int64_t limit_filesize); /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c index 0845c631a5..122b17686a 100644 --- a/fftools/ffmpeg_filter.c +++ b/fftools/ffmpeg_filter.c @@ -603,11 +603,11 @@ static int configure_output_audio_filter(FilterGraph *fg, OutputFilter *ofilter, if (ost->apad && of->shortest) { int i; - for (i=0; i<of->ctx->nb_streams; i++) - if (of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + for (i = 0; i < of->nb_streams; i++) + if (output_streams[of->ost_index + i]->st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) break; - if (i<of->ctx->nb_streams) { + if (i < of->nb_streams) { AUTO_INSERT_FILTER("-apad", "apad", ost->apad); } } diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 87c7bdc4e3..c44ff0f1df 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -49,6 +49,8 @@ typedef struct MuxStream { } MuxStream; struct Muxer { + AVFormatContext *fc; + MuxStream *streams; AVDictionary *opts; @@ -117,7 +119,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; - AVFormatContext *s = of->ctx; + AVFormatContext *s = of->mux->fc; AVStream *st = ost->st; int ret; @@ -263,8 +265,8 @@ static int print_sdp(void) if (!avc) exit_program(1); for (i = 0, j = 0; i < nb_output_files; i++) { - if (!strcmp(output_files[i]->ctx->oformat->name, "rtp")) { - avc[j] = output_files[i]->ctx; + if (!strcmp(output_files[i]->format->name, "rtp")) { + avc[j] = output_files[i]->mux->fc; j++; } } @@ -302,15 +304,16 @@ fail: /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of) { + AVFormatContext *fc = of->mux->fc; int ret, i; - for (i = 0; i < of->ctx->nb_streams; i++) { + for (i = 0; i < fc->nb_streams; i++) { OutputStream *ost = output_streams[of->ost_index + i]; if (!ost->initialized) return 0; } - ret = avformat_write_header(of->ctx, &of->mux->opts); + ret = avformat_write_header(fc, &of->mux->opts); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Could not write header for output file #%d " @@ -321,7 +324,7 @@ int of_check_init(OutputFile *of) //assert_avoptions(of->opts); of->mux->header_written = 1; - av_dump_format(of->ctx, of->index, of->ctx->url, 1); + av_dump_format(fc, of->index, fc->url, 1); nb_output_dumped++; if (sdp_filename || want_sdp) { @@ -333,7 +336,7 @@ int of_check_init(OutputFile *of) } /* flush the muxing queues */ - for (i = 0; i < of->ctx->nb_streams; i++) { + for (i = 0; i < fc->nb_streams; i++) { MuxStream *ms = &of->mux->streams[i]; OutputStream *ost = output_streams[of->ost_index + i]; AVPacket *pkt; @@ -356,29 +359,30 @@ int of_check_init(OutputFile *of) int of_write_trailer(OutputFile *of) { + AVFormatContext *fc = of->mux->fc; int ret; if (!of->mux->header_written) { av_log(NULL, AV_LOG_ERROR, "Nothing was written into output file %d (%s), because " "at least one of its streams received no packets.\n", - of->index, of->ctx->url); + of->index, fc->url); return AVERROR(EINVAL); } - ret = av_write_trailer(of->ctx); + ret = av_write_trailer(fc); if (ret < 0) { - av_log(NULL, AV_LOG_ERROR, "Error writing trailer of %s: %s\n", of->ctx->url, av_err2str(ret)); + av_log(NULL, AV_LOG_ERROR, "Error writing trailer of %s: %s\n", fc->url, av_err2str(ret)); return ret; } of->mux->final_filesize = of_filesize(of); if (!(of->format->flags & AVFMT_NOFILE)) { - ret = avio_closep(&of->ctx->pb); + ret = avio_closep(&fc->pb); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Error closing file %s: %s\n", - of->ctx->url, av_err2str(ret)); + fc->url, av_err2str(ret)); return ret; } } @@ -386,14 +390,28 @@ int of_write_trailer(OutputFile *of) return 0; } -static void mux_free(Muxer **pmux, int nb_streams) +static void fc_close(AVFormatContext **pfc) +{ + AVFormatContext *fc = *pfc; + + if (!fc) + return; + + if (!(fc->oformat->flags & AVFMT_NOFILE)) + avio_closep(&fc->pb); + avformat_free_context(fc); + + *pfc = NULL; +} + +static void mux_free(Muxer **pmux) { Muxer *mux = *pmux; if (!mux) return; - for (int i = 0; i < nb_streams; i++) { + for (int i = 0; i < mux->fc->nb_streams; i++) { MuxStream *ms = &mux->streams[i]; AVPacket *pkt; @@ -409,13 +427,14 @@ static void mux_free(Muxer **pmux, int nb_streams) av_packet_free(&mux->sq_pkt); + fc_close(&mux->fc); + av_freep(pmux); } void of_close(OutputFile **pof) { OutputFile *of = *pof; - AVFormatContext *s; if (!of) return; @@ -423,34 +442,33 @@ void of_close(OutputFile **pof) sq_free(&of->sq_encode); sq_free(&of->sq_mux); - s = of->ctx; - - mux_free(&of->mux, s ? s->nb_streams : 0); - - if (s && s->oformat && !(s->oformat->flags & AVFMT_NOFILE)) - avio_closep(&s->pb); - avformat_free_context(s); + mux_free(&of->mux); av_freep(pof); } -int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) +int of_muxer_init(OutputFile *of, AVFormatContext *fc, + AVDictionary *opts, int64_t limit_filesize) { Muxer *mux = av_mallocz(sizeof(*mux)); int ret = 0; - if (!mux) + if (!mux) { + fc_close(&fc); return AVERROR(ENOMEM); + } - mux->streams = av_calloc(of->ctx->nb_streams, sizeof(*mux->streams)); + mux->streams = av_calloc(fc->nb_streams, sizeof(*mux->streams)); if (!mux->streams) { + fc_close(&fc); av_freep(&mux); return AVERROR(ENOMEM); } of->mux = mux; + mux->fc = fc; - for (int i = 0; i < of->ctx->nb_streams; i++) { + for (int i = 0; i < fc->nb_streams; i++) { MuxStream *ms = &mux->streams[i]; ms->muxing_queue = av_fifo_alloc2(8, sizeof(AVPacket*), 0); if (!ms->muxing_queue) { @@ -475,7 +493,7 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) } /* write the header for files with no streams */ - if (of->format->flags & AVFMT_NOSTREAMS && of->ctx->nb_streams == 0) { + if (of->format->flags & AVFMT_NOSTREAMS && fc->nb_streams == 0) { ret = of_check_init(of); if (ret < 0) goto fail; @@ -483,7 +501,7 @@ int of_muxer_init(OutputFile *of, AVDictionary *opts, int64_t limit_filesize) fail: if (ret < 0) - mux_free(&of->mux, of->ctx->nb_streams); + mux_free(&of->mux); return ret; } @@ -495,7 +513,7 @@ int of_finished(OutputFile *of) int64_t of_filesize(OutputFile *of) { - AVIOContext *pb = of->ctx->pb; + AVIOContext *pb = of->mux->fc->pb; int64_t ret = -1; if (of->mux->final_filesize) @@ -512,6 +530,6 @@ int64_t of_filesize(OutputFile *of) AVChapter * const * of_get_chapters(OutputFile *of, unsigned int *nb_chapters) { - *nb_chapters = of->ctx->nb_chapters; - return of->ctx->chapters; + *nb_chapters = of->mux->fc->nb_chapters; + return of->mux->fc->chapters; } diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 22eb558ff3..5ef4115cdb 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2455,7 +2455,6 @@ static int open_output_file(OptionsContext *o, const char *filename) exit_program(1); } - of->ctx = oc; of->format = oc->oformat; if (o->recording_time != INT64_MAX) oc->duration = o->recording_time; @@ -3055,7 +3054,10 @@ loop_end: exit_program(1); } - err = of_muxer_init(of, format_opts, o->limit_filesize); + of->nb_streams = oc->nb_streams; + of->url = filename; + + err = of_muxer_init(of, oc, format_opts, o->limit_filesize); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); exit_program(1); -- 2.34.1
Do not call exit_program(), as that would conflict with moving this code into a separate thread. --- fftools/ffmpeg.c | 54 +++++++++++++++++++++++++++++++------------- fftools/ffmpeg.h | 2 +- fftools/ffmpeg_mux.c | 6 +++-- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 4321652848..5df453c2d9 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -713,6 +713,7 @@ static void close_output_stream(OutputStream *ost) static void output_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int eof) { + const char *err_msg; int ret = 0; if (!eof && pkt->dts != AV_NOPTS_VALUE) @@ -720,28 +721,49 @@ static void output_packet(OutputFile *of, AVPacket *pkt, /* apply the output bitstream filters */ if (ost->bsf_ctx) { + int bsf_eof = 0; + ret = av_bsf_send_packet(ost->bsf_ctx, eof ? NULL : pkt); + if (ret < 0) { + err_msg = "submitting a packet for bitstream filtering"; + goto fail; + } + + while (!bsf_eof) { + ret = av_bsf_receive_packet(ost->bsf_ctx, pkt); + if (ret == AVERROR(EAGAIN)) + return; + else if (ret == AVERROR_EOF) + bsf_eof = 1; + else if (ret < 0) { + err_msg = "applying bitstream filters to a packet"; + goto fail; + } + + ret = of_submit_packet(of, bsf_eof ? NULL : pkt, ost); + if (ret < 0) + goto mux_fail; + } + } else { + ret = of_submit_packet(of, eof ? NULL : pkt, ost); if (ret < 0) - goto finish; - while ((ret = av_bsf_receive_packet(ost->bsf_ctx, pkt)) >= 0) - of_submit_packet(of, pkt, ost); - if (ret == AVERROR_EOF) - of_submit_packet(of, NULL, ost); - if (ret == AVERROR(EAGAIN)) - ret = 0; - } else - of_submit_packet(of, eof ? NULL : pkt, ost); + goto mux_fail; + } if (eof) ost->finished |= MUXER_FINISHED; -finish: - if (ret < 0 && ret != AVERROR_EOF) { - av_log(NULL, AV_LOG_ERROR, "Error applying bitstream filters to an output " - "packet for stream #%d:%d.\n", ost->file_index, ost->index); - if(exit_on_error) - exit_program(1); - } + return; + +mux_fail: + err_msg = "submitting a packet to the muxer"; + +fail: + av_log(NULL, AV_LOG_ERROR, "Error %s for output stream #%d:%d.\n", + err_msg, ost->file_index, ost->index); + if (exit_on_error) + exit_program(1); + } static int check_recording_time(OutputStream *ost) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index c757f72c0e..91f5b4024b 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -705,7 +705,7 @@ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); void of_close(OutputFile **pof); -void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost); +int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost); int of_finished(OutputFile *of); int64_t of_filesize(OutputFile *of); AVChapter * const * diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index c44ff0f1df..490e0e54eb 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -232,7 +232,7 @@ static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } -void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) +int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) { int ret; @@ -243,9 +243,11 @@ void of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) ret = queue_packet(of, ost, pkt); if (ret < 0) { av_packet_unref(pkt); - exit_program(1); + return ret; } } + + return 0; } static int print_sdp(void) -- 2.34.1
Do not call exit_program(), as that would conflict with moving this code into a separate thread. --- fftools/ffmpeg_mux.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 490e0e54eb..98d3adaeac 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -200,7 +200,7 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) } } -static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +static int submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { if (ost->sq_idx_mux >= 0) { int ret = sq_send(of->sq_mux, ost->sq_idx_mux, SQPKT(pkt)); @@ -209,17 +209,17 @@ static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) av_packet_unref(pkt); if (ret == AVERROR_EOF) { ost->finished |= MUXER_FINISHED; - return; + return 0; } else - exit_program(1); + return ret; } while (1) { ret = sq_receive(of->sq_mux, -1, SQPKT(of->mux->sq_pkt)); if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) - return; + return 0; else if (ret < 0) - exit_program(1); + return ret; write_packet(of, output_streams[of->ost_index + ret], of->mux->sq_pkt); @@ -230,6 +230,8 @@ static void submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) else ost->finished |= MUXER_FINISHED; } + + return 0; } int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) @@ -237,7 +239,7 @@ int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) int ret; if (of->mux->header_written) { - submit_packet(of, ost, pkt); + return submit_packet(of, ost, pkt); } else { /* the muxer is not initialized yet, buffer the packet */ ret = queue_packet(of, ost, pkt); @@ -348,11 +350,13 @@ int of_check_init(OutputFile *of) ost->mux_timebase = ost->st->time_base; while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { - submit_packet(of, ost, pkt); + ret = submit_packet(of, ost, pkt); if (pkt) { ms->muxing_queue_data_size -= pkt->size; av_packet_free(&pkt); } + if (ret < 0) + return ret; } } -- 2.34.1
Do not call exit_program(), as that would conflict with moving this code into a separate thread. --- fftools/ffmpeg_mux.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 98d3adaeac..cf24ae7700 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -116,7 +116,7 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return 0; } -static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +static int write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; AVFormatContext *s = of->mux->fc; @@ -161,10 +161,8 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) av_log(s, loglevel, "Non-monotonous DTS in output stream " "%d:%d; previous: %"PRId64", current: %"PRId64"; ", ost->file_index, ost->st->index, ms->last_mux_dts, pkt->dts); - if (exit_on_error) { - av_log(NULL, AV_LOG_FATAL, "aborting.\n"); - exit_program(1); - } + if (exit_on_error) + return AVERROR(EINVAL); av_log(s, loglevel, "changing to %"PRId64". This may result " "in incorrect timestamps in the output file.\n", max); @@ -197,7 +195,10 @@ static void write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) print_error("av_interleaved_write_frame()", ret); main_return_code = 1; close_all_output_streams(ost, MUXER_FINISHED | ENCODER_FINISHED, ENCODER_FINISHED); + return ret; } + + return 0; } static int submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) @@ -221,14 +222,16 @@ static int submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) else if (ret < 0) return ret; - write_packet(of, output_streams[of->ost_index + ret], - of->mux->sq_pkt); + ret = write_packet(of, output_streams[of->ost_index + ret], + of->mux->sq_pkt); + if (ret < 0) + return ret; } } else { if (pkt) - write_packet(of, ost, pkt); - else - ost->finished |= MUXER_FINISHED; + return write_packet(of, ost, pkt); + + ost->finished |= MUXER_FINISHED; } return 0; -- 2.34.1
Return an error instead, as is already done in other places in this function. --- fftools/ffmpeg_mux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index cf24ae7700..2abadd3f9b 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -270,7 +270,7 @@ static int print_sdp(void) avc = av_malloc_array(nb_output_files, sizeof(*avc)); if (!avc) - exit_program(1); + return AVERROR(ENOMEM); for (i = 0, j = 0; i < nb_output_files; i++) { if (!strcmp(output_files[i]->format->name, "rtp")) { avc[j] = output_files[i]->mux->fc; -- 2.34.1
It retrieves the muxer's internal timestamp with under-defined semantics. Continuing to use this value would also require synchronization once the muxer is moved to a separate thread. Replace the value with last_mux_dts. --- fftools/ffmpeg.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 5df453c2d9..f57028b9b7 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1658,9 +1658,8 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti vid = 1; } /* compute min output value */ - if (av_stream_get_end_pts(ost->st) != AV_NOPTS_VALUE) { - pts = FFMAX(pts, av_rescale_q(av_stream_get_end_pts(ost->st), - ost->st->time_base, AV_TIME_BASE_Q)); + if (ost->last_mux_dts != AV_NOPTS_VALUE) { + pts = FFMAX(pts, ost->last_mux_dts); if (copy_ts) { if (copy_ts_first_pts == AV_NOPTS_VALUE && pts > 1) copy_ts_first_pts = pts; -- 2.34.1
ffmpeg will be switched to a fully threaded architecture, starting with muxers. --- configure | 2 +- fftools/ffmpeg.c | 20 -------------------- fftools/ffmpeg.h | 2 -- fftools/ffmpeg_opt.c | 2 -- 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/configure b/configure index 3dca1c4bd3..1c80c54baf 100755 --- a/configure +++ b/configure @@ -3808,7 +3808,7 @@ avfilter_extralibs="pthreads_extralibs" avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vdpau_x11_extralibs" # programs -ffmpeg_deps="avcodec avfilter avformat" +ffmpeg_deps="avcodec avfilter avformat threads" ffmpeg_select="aformat_filter anull_filter atrim_filter format_filter hflip_filter null_filter transpose_filter trim_filter vflip_filter" diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index f57028b9b7..aea7335c8f 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -161,9 +161,7 @@ static struct termios oldtty; static int restore_tty; #endif -#if HAVE_THREADS static void free_input_threads(void); -#endif /* sub2video hack: Convert subtitles to video with alpha to insert them in filter graphs. @@ -593,9 +591,7 @@ static void ffmpeg_cleanup(int ret) av_freep(&output_streams[i]); } -#if HAVE_THREADS free_input_threads(); -#endif for (i = 0; i < nb_input_files; i++) { avformat_close_input(&input_files[i]->ctx); av_packet_free(&input_files[i]->pkt); @@ -3660,7 +3656,6 @@ static int check_keyboard_interaction(int64_t cur_time) return 0; } -#if HAVE_THREADS static void *input_thread(void *arg) { InputFile *f = arg; @@ -3778,7 +3773,6 @@ static int get_input_packet_mt(InputFile *f, AVPacket **pkt) f->non_blocking ? AV_THREAD_MESSAGE_NONBLOCK : 0); } -#endif static int get_input_packet(InputFile *f, AVPacket **pkt) { @@ -3801,10 +3795,8 @@ static int get_input_packet(InputFile *f, AVPacket **pkt) } } -#if HAVE_THREADS if (f->thread_queue_size) return get_input_packet_mt(f, pkt); -#endif *pkt = f->pkt; return av_read_frame(f->ctx, *pkt); } @@ -3944,15 +3936,11 @@ static int process_input(int file_index) avcodec_flush_buffers(avctx); } } -#if HAVE_THREADS free_input_thread(file_index); -#endif ret = seek_to_start(ifile, is); -#if HAVE_THREADS thread_ret = init_input_thread(file_index); if (thread_ret < 0) return thread_ret; -#endif if (ret < 0) av_log(NULL, AV_LOG_WARNING, "Seek to start failed.\n"); else @@ -4194,11 +4182,9 @@ static int process_input(int file_index) process_input_packet(ist, pkt, 0); discard_packet: -#if HAVE_THREADS if (ifile->thread_queue_size) av_packet_free(&pkt); else -#endif av_packet_unref(pkt); return 0; @@ -4365,10 +4351,8 @@ static int transcode(void) timer_start = av_gettime_relative(); -#if HAVE_THREADS if ((ret = init_input_threads()) < 0) goto fail; -#endif while (!received_sigterm) { int64_t cur_time= av_gettime_relative(); @@ -4393,9 +4377,7 @@ static int transcode(void) /* dump report by using the output first video and audio streams */ print_report(0, timer_start, cur_time); } -#if HAVE_THREADS free_input_threads(); -#endif /* at the end of stream, we must flush the decoder buffers */ for (i = 0; i < nb_input_streams; i++) { @@ -4449,9 +4431,7 @@ static int transcode(void) ret = 0; fail: -#if HAVE_THREADS free_input_threads(); -#endif if (output_streams) { for (i = 0; i < nb_output_streams; i++) { diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 91f5b4024b..9baa701c67 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -426,13 +426,11 @@ typedef struct InputFile { AVPacket *pkt; -#if HAVE_THREADS AVThreadMessageQueue *in_thread_queue; pthread_t thread; /* thread reading from this file */ int non_blocking; /* reading packets from the thread should not block */ int joined; /* the thread has been joined */ int thread_queue_size; /* maximum number of queued packets */ -#endif } InputFile; enum forced_keyframes_const { diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 5ef4115cdb..7f46238534 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -1328,9 +1328,7 @@ static int open_input_file(OptionsContext *o, const char *filename) f->pkt = av_packet_alloc(); if (!f->pkt) exit_program(1); -#if HAVE_THREADS f->thread_queue_size = o->thread_queue_size; -#endif /* check if all codec options have been used */ unused_opts = strip_specifiers(o->g->codec_opts); -- 2.34.1
It is similar to AVThreadMessageQueue, but supports multiple streams, each with its own EOF state. --- fftools/Makefile | 1 + fftools/thread_queue.c | 245 +++++++++++++++++++++++++++++++++++++++++ fftools/thread_queue.h | 81 ++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 fftools/thread_queue.c create mode 100644 fftools/thread_queue.h diff --git a/fftools/Makefile b/fftools/Makefile index bc57ebe748..6285e6eacb 100644 --- a/fftools/Makefile +++ b/fftools/Makefile @@ -16,6 +16,7 @@ OBJS-ffmpeg += \ fftools/ffmpeg_opt.o \ fftools/objpool.o \ fftools/sync_queue.o \ + fftools/thread_queue.o \ define DOFFTOOL OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes) diff --git a/fftools/thread_queue.c b/fftools/thread_queue.c new file mode 100644 index 0000000000..a1ab4ce92e --- /dev/null +++ b/fftools/thread_queue.c @@ -0,0 +1,245 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/error.h" +#include "libavutil/fifo.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/mem.h" +#include "libavutil/thread.h" + +#include "objpool.h" +#include "thread_queue.h" + +enum { + FINISHED_SEND = (1 << 0), + FINISHED_RECV = (1 << 1), +}; + +typedef struct FifoElem { + void *obj; + unsigned int stream_idx; +} FifoElem; + +struct ThreadQueue { + int *finished; + unsigned int nb_streams; + + AVFifo *fifo; + + ObjPool *obj_pool; + void (*obj_move)(void *dst, void *src); + + pthread_mutex_t lock; + pthread_cond_t cond; +}; + +void tq_free(ThreadQueue **ptq) +{ + ThreadQueue *tq = *ptq; + + if (!tq) + return; + + if (tq->fifo) { + FifoElem elem; + while (av_fifo_read(tq->fifo, &elem, 1) >= 0) + objpool_release(tq->obj_pool, &elem.obj); + } + av_fifo_freep2(&tq->fifo); + + objpool_free(&tq->obj_pool); + + av_freep(&tq->finished); + + pthread_cond_destroy(&tq->cond); + pthread_mutex_destroy(&tq->lock); + + av_freep(ptq); +} + +ThreadQueue *tq_alloc(unsigned int nb_streams, size_t queue_size, + ObjPool *obj_pool, void (*obj_move)(void *dst, void *src)) +{ + ThreadQueue *tq; + int ret; + + tq = av_mallocz(sizeof(*tq)); + if (!tq) + return NULL; + + ret = pthread_cond_init(&tq->cond, NULL); + if (ret) { + av_freep(&tq); + return NULL; + } + + ret = pthread_mutex_init(&tq->lock, NULL); + if (ret) { + pthread_cond_destroy(&tq->cond); + av_freep(&tq); + return NULL; + } + + tq->finished = av_calloc(nb_streams, sizeof(*tq->finished)); + if (!tq->finished) + goto fail; + tq->nb_streams = nb_streams; + + tq->fifo = av_fifo_alloc2(queue_size, sizeof(FifoElem), 0); + if (!tq->fifo) + goto fail; + + tq->obj_pool = obj_pool; + tq->obj_move = obj_move; + + return tq; +fail: + tq_free(&tq); + return NULL; +} + +int tq_send(ThreadQueue *tq, unsigned int stream_idx, void *data) +{ + int *finished; + int ret; + + av_assert0(stream_idx < tq->nb_streams); + finished = &tq->finished[stream_idx]; + + pthread_mutex_lock(&tq->lock); + + if (*finished & FINISHED_SEND) { + ret = AVERROR(EINVAL); + goto finish; + } + + while (!(*finished & FINISHED_RECV) && !av_fifo_can_write(tq->fifo)) + pthread_cond_wait(&tq->cond, &tq->lock); + + if (*finished & FINISHED_RECV) { + ret = AVERROR_EOF; + *finished |= FINISHED_SEND; + } else { + FifoElem elem = { .stream_idx = stream_idx }; + + ret = objpool_get(tq->obj_pool, &elem.obj); + if (ret < 0) + goto finish; + + tq->obj_move(elem.obj, data); + + ret = av_fifo_write(tq->fifo, &elem, 1); + av_assert0(ret >= 0); + pthread_cond_broadcast(&tq->cond); + } + +finish: + pthread_mutex_unlock(&tq->lock); + + return ret; +} + +static int receive_locked(ThreadQueue *tq, int *stream_idx, + void *data) +{ + FifoElem elem; + unsigned int nb_finished = 0; + + if (av_fifo_read(tq->fifo, &elem, 1) >= 0) { + tq->obj_move(data, elem.obj); + objpool_release(tq->obj_pool, &elem.obj); + *stream_idx = elem.stream_idx; + return 0; + } + + for (unsigned int i = 0; i < tq->nb_streams; i++) { + if (!(tq->finished[i] & FINISHED_SEND)) + continue; + + /* return EOF to the consumer at most once for each stream */ + if (!(tq->finished[i] & FINISHED_RECV)) { + tq->finished[i] |= FINISHED_RECV; + *stream_idx = i; + return AVERROR_EOF; + } + + nb_finished++; + } + + return nb_finished == tq->nb_streams ? AVERROR_EOF : AVERROR(EAGAIN); +} + +int tq_receive(ThreadQueue *tq, int *stream_idx, void *data) +{ + int ret; + + *stream_idx = -1; + + pthread_mutex_lock(&tq->lock); + + while (1) { + ret = receive_locked(tq, stream_idx, data); + if (ret == AVERROR(EAGAIN)) { + pthread_cond_wait(&tq->cond, &tq->lock); + continue; + } + + break; + } + + if (ret == 0) + pthread_cond_broadcast(&tq->cond); + + pthread_mutex_unlock(&tq->lock); + + return ret; +} + +void tq_send_finish(ThreadQueue *tq, unsigned int stream_idx) +{ + av_assert0(stream_idx < tq->nb_streams); + + pthread_mutex_lock(&tq->lock); + + /* mark the stream as send-finished; + * next time the consumer thread tries to read this stream it will get + * an EOF and recv-finished flag will be set */ + tq->finished[stream_idx] |= FINISHED_SEND; + pthread_cond_broadcast(&tq->cond); + + pthread_mutex_unlock(&tq->lock); +} + +void tq_receive_finish(ThreadQueue *tq, unsigned int stream_idx) +{ + av_assert0(stream_idx < tq->nb_streams); + + pthread_mutex_lock(&tq->lock); + + /* mark the stream as recv-finished; + * next time the producer thread tries to send for this stream, it will + * get an EOF and send-finished flag will be set */ + tq->finished[stream_idx] |= FINISHED_RECV; + pthread_cond_broadcast(&tq->cond); + + pthread_mutex_unlock(&tq->lock); +} diff --git a/fftools/thread_queue.h b/fftools/thread_queue.h new file mode 100644 index 0000000000..0cc8c71ebd --- /dev/null +++ b/fftools/thread_queue.h @@ -0,0 +1,81 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_THREAD_QUEUE_H +#define FFTOOLS_THREAD_QUEUE_H + +#include <string.h> + +#include "objpool.h" + +typedef struct ThreadQueue ThreadQueue; + +/** + * Allocate a queue for sending data between threads. + * + * @param nb_streams number of streams for which a distinct EOF state is + * maintained + * @param queue_size number of items that can be stored in the queue without + * blocking + * @param obj_pool object pool that will be used to allocate items stored in the + * queue; the pool becomes owned by the queue + * @param callback that moves the contents between two data pointers + */ +ThreadQueue *tq_alloc(unsigned int nb_streams, size_t queue_size, + ObjPool *obj_pool, void (*obj_move)(void *dst, void *src)); +void tq_free(ThreadQueue **tq); + +/** + * Send an item for the given stream to the queue. + * + * @param data the item to send, its contents will be moved using the callback + * provided to tq_alloc(); on failure the item will be left + * untouched + * @return + * - 0 the item was successfully sent + * - AVERROR(ENOMEM) could not allocate an item for writing to the FIFO + * - AVERROR(EINVAL) the sending side has previously been marked as finished + * - AVERROR_EOF the receiving side has marked the given stream as finished + */ +int tq_send(ThreadQueue *tq, unsigned int stream_idx, void *data); +/** + * Mark the given stream finished from the sending side. + */ +void tq_send_finish(ThreadQueue *tq, unsigned int stream_idx); + +/** + * Read the next item from the queue. + * + * @param stream_idx the index of the stream that was processed or -1 will be + * written here + * @param data the data item will be written here on success using the + * callback provided to tq_alloc() + * @return + * - 0 a data item was successfully read; *stream_idx contains a non-negative + * stream index + * - AVERROR_EOF When *stream_idx is non-negative, this signals that the sending + * side has marked the given stream as finished. This will happen at most once + * for each stream. When *stream_idx is -1, all streams are done. + */ +int tq_receive(ThreadQueue *tq, int *stream_idx, void *data); +/** + * Mark the given stream finished from the receiving side. + */ +void tq_receive_finish(ThreadQueue *tq, unsigned int stream_idx); + +#endif // FFTOOLS_THREAD_QUEUE_H -- 2.34.1
--- doc/ffmpeg.texi | 11 +- fftools/ffmpeg.c | 18 ++- fftools/ffmpeg.h | 7 +- fftools/ffmpeg_mux.c | 299 +++++++++++++++++++++++++++++++------------ fftools/ffmpeg_opt.c | 4 +- 5 files changed, 236 insertions(+), 103 deletions(-) diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi index 7542832eb3..e4e2d6ddac 100644 --- a/doc/ffmpeg.texi +++ b/doc/ffmpeg.texi @@ -1900,13 +1900,16 @@ to the @option{-ss} option is considered an actual timestamp, and is not offset by the start time of the file. This matters only for files which do not start from timestamp 0, such as transport streams. -@item -thread_queue_size @var{size} (@emph{input}) -This option sets the maximum number of queued packets when reading from the -file or device. With low latency / high rate live streams, packets may be -discarded if they are not read in a timely manner; setting this value can +@item -thread_queue_size @var{size} (@emph{input/output}) +For input, this option sets the maximum number of queued packets when reading +from the file or device. With low latency / high rate live streams, packets may +be discarded if they are not read in a timely manner; setting this value can force ffmpeg to use a separate input thread and read packets as soon as they arrive. By default ffmpeg only does this if multiple inputs are specified. +For output, this option specified the maximum number of packets that may be +queued to each muxing thread. + @item -sdp_file @var{file} (@emph{global}) Print sdp information for an output stream to @var{file}. This allows dumping sdp information when at least one output isn't an diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index aea7335c8f..2a7ff16b74 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -746,9 +746,6 @@ static void output_packet(OutputFile *of, AVPacket *pkt, goto mux_fail; } - if (eof) - ost->finished |= MUXER_FINISHED; - return; mux_fail: @@ -1521,7 +1518,7 @@ static void print_final_stats(int64_t total_size) enum AVMediaType type = ost->enc_ctx->codec_type; total_size += ost->data_size; - total_packets += ost->packets_written; + total_packets += atomic_load(&ost->packets_written); av_log(NULL, AV_LOG_VERBOSE, " Output stream #%d:%d (%s): ", i, j, av_get_media_type_string(type)); @@ -1534,7 +1531,7 @@ static void print_final_stats(int64_t total_size) } av_log(NULL, AV_LOG_VERBOSE, "%"PRIu64" packets muxed (%"PRIu64" bytes); ", - ost->packets_written, ost->data_size); + atomic_load(&ost->packets_written), ost->data_size); av_log(NULL, AV_LOG_VERBOSE, "\n"); } @@ -1603,7 +1600,7 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti if (!vid && enc->codec_type == AVMEDIA_TYPE_VIDEO) { float fps; - frame_number = ost->packets_written; + frame_number = atomic_load(&ost->packets_written); fps = t > 1 ? frame_number / t : 0; av_bprintf(&buf, "frame=%5d fps=%3.*f q=%3.1f ", frame_number, fps < 9.95, fps, q); @@ -3480,9 +3477,8 @@ static int need_output(void) for (i = 0; i < nb_output_streams; i++) { OutputStream *ost = output_streams[i]; - OutputFile *of = output_files[ost->file_index]; - if (ost->finished || of_finished(of)) + if (ost->finished) continue; return 1; @@ -4402,9 +4398,11 @@ static int transcode(void) /* close each encoder */ for (i = 0; i < nb_output_streams; i++) { + uint64_t packets_written; ost = output_streams[i]; - total_packets_written += ost->packets_written; - if (!ost->packets_written && (abort_on_flags & ABORT_ON_FLAG_EMPTY_OUTPUT_STREAM)) { + packets_written = atomic_load(&ost->packets_written); + total_packets_written += packets_written; + if (!packets_written && (abort_on_flags & ABORT_ON_FLAG_EMPTY_OUTPUT_STREAM)) { av_log(NULL, AV_LOG_FATAL, "Empty output on stream %d.\n", i); exit_program(1); } diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 9baa701c67..8940f719b0 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -21,6 +21,7 @@ #include "config.h" +#include <stdatomic.h> #include <stdint.h> #include <stdio.h> #include <signal.h> @@ -555,7 +556,7 @@ typedef struct OutputStream { // combined size of all the packets written uint64_t data_size; // number of packets send to the muxer - uint64_t packets_written; + atomic_uint_least64_t packets_written; // number of frames/samples sent to the encoder uint64_t frames_encoded; uint64_t samples_encoded; @@ -697,14 +698,14 @@ int hw_device_setup_for_filter(FilterGraph *fg); int hwaccel_decode_init(AVCodecContext *avctx); int of_muxer_init(OutputFile *of, AVFormatContext *fc, - AVDictionary *opts, int64_t limit_filesize); + AVDictionary *opts, int64_t limit_filesize, + int thread_queue_size); /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of); int of_write_trailer(OutputFile *of); void of_close(OutputFile **pof); int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost); -int of_finished(OutputFile *of); int64_t of_filesize(OutputFile *of); AVChapter * const * of_get_chapters(OutputFile *of, unsigned int *nb_chapters); diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c index 2abadd3f9b..67b875d41d 100644 --- a/fftools/ffmpeg_mux.c +++ b/fftools/ffmpeg_mux.c @@ -16,17 +16,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <stdatomic.h> #include <stdio.h> #include <string.h> #include "ffmpeg.h" +#include "objpool.h" #include "sync_queue.h" +#include "thread_queue.h" #include "libavutil/fifo.h" #include "libavutil/intreadwrite.h" #include "libavutil/log.h" #include "libavutil/mem.h" #include "libavutil/timestamp.h" +#include "libavutil/thread.h" #include "libavcodec/packet.h" @@ -51,13 +55,18 @@ typedef struct MuxStream { struct Muxer { AVFormatContext *fc; + pthread_t thread; + ThreadQueue *tq; + MuxStream *streams; AVDictionary *opts; + int thread_queue_size; + /* filesize limit expressed in bytes */ int64_t limit_filesize; - int64_t final_filesize; + atomic_int_least64_t last_filesize; int header_written; AVPacket *sq_pkt; @@ -65,15 +74,6 @@ struct Muxer { static int want_sdp = 1; -static void close_all_output_streams(OutputStream *ost, OSTFinished this_stream, OSTFinished others) -{ - int i; - for (i = 0; i < nb_output_streams; i++) { - OutputStream *ost2 = output_streams[i]; - ost2->finished |= ost == ost2 ? this_stream : others; - } -} - static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; @@ -116,13 +116,32 @@ static int queue_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) return 0; } +static int64_t filesize(AVIOContext *pb) +{ + int64_t ret = -1; + + if (pb) { + ret = avio_size(pb); + if (ret <= 0) // FIXME improve avio_size() so it works with non seekable output too + ret = avio_tell(pb); + } + + return ret; +} + static int write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) { MuxStream *ms = &of->mux->streams[ost->index]; AVFormatContext *s = of->mux->fc; AVStream *st = ost->st; + int64_t fs; int ret; + fs = filesize(s->pb); + atomic_store(&of->mux->last_filesize, fs); + if (fs >= of->mux->limit_filesize) + return AVERROR_EOF; + if ((st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->vsync_method == VSYNC_DROP) || (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_sync_method < 0)) pkt->pts = pkt->dts = AV_NOPTS_VALUE; @@ -175,7 +194,7 @@ static int write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) ms->last_mux_dts = pkt->dts; ost->data_size += pkt->size; - ost->packets_written++; + atomic_fetch_add(&ost->packets_written, 1); pkt->stream_index = ost->index; @@ -193,66 +212,81 @@ static int write_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) ret = av_interleaved_write_frame(s, pkt); if (ret < 0) { print_error("av_interleaved_write_frame()", ret); - main_return_code = 1; - close_all_output_streams(ost, MUXER_FINISHED | ENCODER_FINISHED, ENCODER_FINISHED); return ret; } return 0; } -static int submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +static int sync_queue_process(OutputFile *of, OutputStream *ost, AVPacket *pkt) { if (ost->sq_idx_mux >= 0) { int ret = sq_send(of->sq_mux, ost->sq_idx_mux, SQPKT(pkt)); - if (ret < 0) { - if (pkt) - av_packet_unref(pkt); - if (ret == AVERROR_EOF) { - ost->finished |= MUXER_FINISHED; - return 0; - } else - return ret; - } + if (ret < 0) + return ret; while (1) { ret = sq_receive(of->sq_mux, -1, SQPKT(of->mux->sq_pkt)); - if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) - return 0; - else if (ret < 0) - return ret; + if (ret < 0) + return (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) ? 0 : ret; ret = write_packet(of, output_streams[of->ost_index + ret], of->mux->sq_pkt); if (ret < 0) return ret; } - } else { - if (pkt) - return write_packet(of, ost, pkt); - - ost->finished |= MUXER_FINISHED; - } + } else if (pkt) + return write_packet(of, ost, pkt); return 0; } -int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) +static void *muxer_thread(void *arg) { - int ret; + OutputFile *of = arg; + Muxer *mux = of->mux; + AVPacket *pkt = NULL; + int ret = 0; + + pkt = av_packet_alloc(); + if (!pkt) { + ret = AVERROR(ENOMEM); + goto finish; + } - if (of->mux->header_written) { - return submit_packet(of, ost, pkt); - } else { - /* the muxer is not initialized yet, buffer the packet */ - ret = queue_packet(of, ost, pkt); - if (ret < 0) { - av_packet_unref(pkt); - return ret; + while (1) { + OutputStream *ost; + int stream_idx; + + ret = tq_receive(mux->tq, &stream_idx, pkt); + if (stream_idx < 0) { + av_log(NULL, AV_LOG_DEBUG, + "All streams finished for output file #%d\n", of->index); + ret = 0; + break; + } + + ost = output_streams[of->ost_index + stream_idx]; + ret = sync_queue_process(of, ost, ret < 0 ? NULL : pkt); + av_packet_unref(pkt); + if (ret == AVERROR_EOF) + tq_receive_finish(mux->tq, stream_idx); + else if (ret < 0) { + av_log(NULL, AV_LOG_ERROR, + "Error muxing a packet for output file #%d\n", of->index); + break; } } - return 0; +finish: + av_packet_free(&pkt); + + for (unsigned int i = 0; i < mux->fc->nb_streams; i++) + tq_receive_finish(mux->tq, i); + + av_log(NULL, AV_LOG_DEBUG, "Terminating muxer thread %d\n", of->index); + + return (void*)(intptr_t)ret; } static int print_sdp(void) @@ -303,11 +337,125 @@ static int print_sdp(void) av_freep(&sdp_filename); } + // SDP successfully written, allow muxer threads to start + ret = 1; + fail: av_freep(&avc); return ret; } +static int submit_packet(OutputFile *of, OutputStream *ost, AVPacket *pkt) +{ + Muxer *mux = of->mux; + int ret = 0; + + if (!pkt || ost->finished & MUXER_FINISHED) + goto finish; + + ret = tq_send(mux->tq, ost->index, pkt); + if (ret < 0) + goto finish; + + return 0; + +finish: + if (pkt) + av_packet_unref(pkt); + + ost->finished |= MUXER_FINISHED; + tq_send_finish(mux->tq, ost->index); + return ret == AVERROR_EOF ? 0 : ret; +} + +int of_submit_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost) +{ + int ret; + + if (of->mux->tq) { + return submit_packet(of, ost, pkt); + } else { + /* the muxer is not initialized yet, buffer the packet */ + ret = queue_packet(of, ost, pkt); + if (ret < 0) { + av_packet_unref(pkt); + return ret; + } + } + + return 0; +} + +static int thread_stop(OutputFile *of) +{ + Muxer *mux = of->mux; + void *ret; + + if (!mux || !mux->tq) + return 0; + + for (unsigned int i = 0; i < mux->fc->nb_streams; i++) + tq_send_finish(mux->tq, i); + + pthread_join(mux->thread, &ret); + + tq_free(&mux->tq); + + return (int)(intptr_t)ret; +} + +static void pkt_move(void *dst, void *src) +{ + av_packet_move_ref(dst, src); +} + +static int thread_start(OutputFile *of) +{ + Muxer *mux = of->mux; + AVFormatContext *fc = mux->fc; + ObjPool *op; + int ret; + + op = objpool_alloc_packets(); + if (!op) + return AVERROR(ENOMEM); + + mux->tq = tq_alloc(fc->nb_streams, mux->thread_queue_size, op, pkt_move); + if (!mux->tq) { + objpool_free(&op); + return AVERROR(ENOMEM); + } + + ret = pthread_create(&mux->thread, NULL, muxer_thread, (void*)of); + if (ret) { + tq_free(&mux->tq); + return AVERROR(ret); + } + + /* flush the muxing queues */ + for (int i = 0; i < fc->nb_streams; i++) { + MuxStream *ms = &of->mux->streams[i]; + OutputStream *ost = output_streams[of->ost_index + i]; + AVPacket *pkt; + + /* try to improve muxing time_base (only possible if nothing has been written yet) */ + if (!av_fifo_can_read(ms->muxing_queue)) + ost->mux_timebase = ost->st->time_base; + + while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { + ret = submit_packet(of, ost, pkt); + if (pkt) { + ms->muxing_queue_data_size -= pkt->size; + av_packet_free(&pkt); + } + if (ret < 0) + return ret; + } + } + + return 0; +} + /* open the muxer when all the streams are initialized */ int of_check_init(OutputFile *of) { @@ -339,28 +487,19 @@ int of_check_init(OutputFile *of) if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Error writing the SDP.\n"); return ret; - } - } - - /* flush the muxing queues */ - for (i = 0; i < fc->nb_streams; i++) { - MuxStream *ms = &of->mux->streams[i]; - OutputStream *ost = output_streams[of->ost_index + i]; - AVPacket *pkt; - - /* try to improve muxing time_base (only possible if nothing has been written yet) */ - if (!av_fifo_can_read(ms->muxing_queue)) - ost->mux_timebase = ost->st->time_base; - - while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) { - ret = submit_packet(of, ost, pkt); - if (pkt) { - ms->muxing_queue_data_size -= pkt->size; - av_packet_free(&pkt); + } else if (ret == 1) { + /* SDP is written only after all the muxers are ready, so now we + * start ALL the threads */ + for (i = 0; i < nb_output_files; i++) { + ret = thread_start(output_files[i]); + if (ret < 0) + return ret; } - if (ret < 0) - return ret; } + } else { + ret = thread_start(of); + if (ret < 0) + return ret; } return 0; @@ -371,7 +510,7 @@ int of_write_trailer(OutputFile *of) AVFormatContext *fc = of->mux->fc; int ret; - if (!of->mux->header_written) { + if (!of->mux->tq) { av_log(NULL, AV_LOG_ERROR, "Nothing was written into output file %d (%s), because " "at least one of its streams received no packets.\n", @@ -379,13 +518,17 @@ int of_write_trailer(OutputFile *of) return AVERROR(EINVAL); } + ret = thread_stop(of); + if (ret < 0) + main_return_code = ret; + ret = av_write_trailer(fc); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Error writing trailer of %s: %s\n", fc->url, av_err2str(ret)); return ret; } - of->mux->final_filesize = of_filesize(of); + of->mux->last_filesize = filesize(fc->pb); if (!(of->format->flags & AVFMT_NOFILE)) { ret = avio_closep(&fc->pb); @@ -448,6 +591,8 @@ void of_close(OutputFile **pof) if (!of) return; + thread_stop(of); + sq_free(&of->sq_encode); sq_free(&of->sq_mux); @@ -457,7 +602,8 @@ void of_close(OutputFile **pof) } int of_muxer_init(OutputFile *of, AVFormatContext *fc, - AVDictionary *opts, int64_t limit_filesize) + AVDictionary *opts, int64_t limit_filesize, + int thread_queue_size) { Muxer *mux = av_mallocz(sizeof(*mux)); int ret = 0; @@ -487,6 +633,7 @@ int of_muxer_init(OutputFile *of, AVFormatContext *fc, ms->last_mux_dts = AV_NOPTS_VALUE; } + mux->thread_queue_size = thread_queue_size > 0 ? thread_queue_size : 8; mux->limit_filesize = limit_filesize; mux->opts = opts; @@ -515,25 +662,9 @@ fail: return ret; } -int of_finished(OutputFile *of) -{ - return of_filesize(of) >= of->mux->limit_filesize; -} - int64_t of_filesize(OutputFile *of) { - AVIOContext *pb = of->mux->fc->pb; - int64_t ret = -1; - - if (of->mux->final_filesize) - ret = of->mux->final_filesize; - else if (pb) { - ret = avio_size(pb); - if (ret <= 0) // FIXME improve avio_size() so it works with non seekable output too - ret = avio_tell(pb); - } - - return ret; + return atomic_load(&of->mux->last_filesize); } AVChapter * const * diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 7f46238534..807c0263b9 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -3055,7 +3055,7 @@ loop_end: of->nb_streams = oc->nb_streams; of->url = filename; - err = of_muxer_init(of, oc, format_opts, o->limit_filesize); + err = of_muxer_init(of, oc, format_opts, o->limit_filesize, o->thread_queue_size); if (err < 0) { av_log(NULL, AV_LOG_FATAL, "Error initializing internal muxing state\n"); exit_program(1); @@ -3841,7 +3841,7 @@ const OptionDef options[] = { { "disposition", OPT_STRING | HAS_ARG | OPT_SPEC | OPT_OUTPUT, { .off = OFFSET(disposition) }, "disposition", "" }, - { "thread_queue_size", HAS_ARG | OPT_INT | OPT_OFFSET | OPT_EXPERT | OPT_INPUT, + { "thread_queue_size", HAS_ARG | OPT_INT | OPT_OFFSET | OPT_EXPERT | OPT_INPUT | OPT_OUTPUT, { .off = OFFSET(thread_queue_size) }, "set the maximum number of queued packets from the demuxer" }, { "find_stream_info", OPT_BOOL | OPT_PERFILE | OPT_INPUT | OPT_EXPERT, { &find_stream_info }, -- 2.34.1
The current version of this set can also be found in my tree git://git.khirnov.net/libav branch ffmpeg_mt/mux -- Anton Khirnov
On Fri, Jun 17, 2022 at 12:27:18PM +0200, Anton Khirnov wrote:
The current version of this set can also be found in my tree git://git.khirnov.net/libav branch ffmpeg_mt/mux
There are really many files changing, its hard to say for sure that all are the same issue, but basically it all seems more or less frames in some streams including cases where there are hugely more or 0 Here are some examples: ffmpeg -i matrixbench_mpeg2.mpg -vcodec rawvideo -pix_fmt rgb555 -allow_raw_vfw 1 -vframes 1 -bitexact file-rgb555.mkv the new file is much bigger (due to the audio track) -rw-r----- 1 michael michael 2765813 Jul 8 16:57 file-rgb555.mkv -rw-r----- 1 michael michael 834643 Jul 8 17:02 file-rgb555-ref.mkv another one: ./ffmpeg -y -i vlcticket/8344/DVR_NVR_IP\ Camera01_20130321162325_20130321162358_576877.mp4 -vframes 1 -aframes 1 -bitexact -f framecrc - This appears to loose the video stream #channel_layout_name 1: mono -0, 0, 0, 1, 2880000, 0x4136bc92 1, 112, 112, 320, 640, 0x2cd73b36 sample in https://samples.ffmpeg.org/camera-dvr/hikvision/ This one fails a bit worse than before (ffmpeg succeeds before besides producing errors as well) my notes say this worked better only before 04aa09c4bcf2d5a634a35da3a3ae3fc1abe30ef8 the file is a little big and i havnt found it anywhere online, i will try to send it privately to you ffmpeg -i 2014-10-17\ 11.31\ i95Dev\ -\ Carlo\ Pazolini\ _\ KWI\ -\ Meeting.g2m -bitexact -max_muxing_queue_size 8000 -vframes 2 file-g2m5.avi Metadata: DeviceConformanceTemplate: L2 WMFSDKNeeded : 0.0.0.0000 WMFSDKVersion : 12.0.7601.17514 IsVBR : 1 WM/ToolVersion : 6.4.3 Build 1767 WM/ToolName : GoToMeeting BitRateFrom the writer: 97087 Audio samples : 34341 Video samples : 3740 recording time : Fri, 17 Oct 2014 12:28:16 Eastern Daylight Time Duration: 00:57:13.86, start: 0.000000, bitrate: 100 kb/s Stream #0:0: Audio: wmav2 (a[1][0][0] / 0x0161), 44100 Hz, mono, fltp, 48 kb/s Stream #0:1: Data: none, 2 kb/s Stream #0:2: Video: g2m (G2M5 / 0x354D3247), rgb24, 1440x900, 49 kb/s, 1k tbr, 1k tbn Stream mapping: Stream #0:2 -> #0:0 (g2m (native) -> mpeg4 (native)) Stream #0:0 -> #0:1 (wmav2 (native) -> mp3 (libmp3lame)) Press [q] to stop, [?] for help [libmp3lame @ 0x55c17cf03140] Queue input is backward in time Output #0, avi, to 'file-g2m5.avi': Metadata: DeviceConformanceTemplate: L2 WMFSDKNeeded : 0.0.0.0000 WMFSDKVersion : 12.0.7601.17514 IsVBR : 1 WM/ToolVersion : 6.4.3 Build 1767 WM/ToolName : GoToMeeting BitRateFrom the writer: 97087 Audio samples : 34341 Video samples : 3740 recording time : Fri, 17 Oct 2014 12:28:16 Eastern Daylight Time Stream #0:0: Video: mpeg4 (FMP4 / 0x34504D46), yuv420p(tv, progressive), 1440x900, q=2-31, 200 kb/s, 1k fps, 1k tbn Metadata: encoder : Lavc mpeg4 Side data: cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A Stream #0:1: Audio: mp3 (U[0][0][0] / 0x0055), 44100 Hz, mono, fltp Metadata: encoder : Lavc libmp3lame [avi @ 0x55c17cf31340] Too large number of skipped frames 194184 > 600000kbits/s speed= 140x av_interleaved_write_frame(): Invalid argument Error muxing a packet for output file #0 [avi @ 0x55c17cf31340] Too large number of skipped frames 194085 > 60000 frame= 2 fps=1.3 q=2.0 Lsize= 1855kB time=00:03:14.18 bitrate= 78.3kbits/s speed= 129x video:149kB audio:1517kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 11.381945% Conversion failed! ./ffmpeg -i tickets/1666/avc-intra-panasonic-AG-HPX301E.mov -vframes 3 -aframes 2 -bitexact -f framecrc - duplicate behavior of the a/vframe issue above, one stream disappears sample in https://samples.ffmpeg.org/ffmpeg-bugs/trac/ticket1666/ [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB Any man who breaks a law that conscience tells him is unjust and willingly accepts the penalty by staying in jail in order to arouse the conscience of the community on the injustice of the law is at that moment expressing the very highest respect for law. - Martin Luther King Jr
Quoting Michael Niedermayer (2022-07-08 18:58:11)
On Fri, Jun 17, 2022 at 12:27:18PM +0200, Anton Khirnov wrote:
The current version of this set can also be found in my tree git://git.khirnov.net/libav branch ffmpeg_mt/mux
There are really many files changing, its hard to say for sure that all are the same issue, but basically it all seems more or less frames in some streams including cases where there are hugely more or 0
Here are some examples:
ffmpeg -i matrixbench_mpeg2.mpg -vcodec rawvideo -pix_fmt rgb555 -allow_raw_vfw 1 -vframes 1 -bitexact file-rgb555.mkv
the new file is much bigger (due to the audio track)
-rw-r----- 1 michael michael 2765813 Jul 8 16:57 file-rgb555.mkv -rw-r----- 1 michael michael 834643 Jul 8 17:02 file-rgb555-ref.mkv
Where can I find this file?
another one: ./ffmpeg -y -i vlcticket/8344/DVR_NVR_IP\ Camera01_20130321162325_20130321162358_576877.mp4 -vframes 1 -aframes 1 -bitexact -f framecrc -
This appears to loose the video stream
#channel_layout_name 1: mono -0, 0, 0, 1, 2880000, 0x4136bc92 1, 112, 112, 320, 640, 0x2cd73b36
The sync queue got confused by stale frame durations, which are now overwritten in the updated version of 22/35. But do note that there may be valid situations where a stream will "disappear" when you specify -frames constraints on multiple streams, because -frames is supposed to cut the whole file once the constraint is reached, not just the stream it applies to. If you want a specific number of frames in each stream, you should rather use the trim/atrim filters. See also my discussion with Andreas under 24/35 in this thread.
This one fails a bit worse than before (ffmpeg succeeds before besides producing errors as well) my notes say this worked better only before 04aa09c4bcf2d5a634a35da3a3ae3fc1abe30ef8 the file is a little big and i havnt found it anywhere online, i will try to send it privately to you
ffmpeg -i 2014-10-17\ 11.31\ i95Dev\ -\ Carlo\ Pazolini\ _\ KWI\ -\ Meeting.g2m -bitexact -max_muxing_queue_size 8000 -vframes 2 file-g2m5.avi
Metadata: DeviceConformanceTemplate: L2 WMFSDKNeeded : 0.0.0.0000 WMFSDKVersion : 12.0.7601.17514 IsVBR : 1 WM/ToolVersion : 6.4.3 Build 1767 WM/ToolName : GoToMeeting BitRateFrom the writer: 97087 Audio samples : 34341 Video samples : 3740 recording time : Fri, 17 Oct 2014 12:28:16 Eastern Daylight Time Duration: 00:57:13.86, start: 0.000000, bitrate: 100 kb/s Stream #0:0: Audio: wmav2 (a[1][0][0] / 0x0161), 44100 Hz, mono, fltp, 48 kb/s Stream #0:1: Data: none, 2 kb/s Stream #0:2: Video: g2m (G2M5 / 0x354D3247), rgb24, 1440x900, 49 kb/s, 1k tbr, 1k tbn Stream mapping: Stream #0:2 -> #0:0 (g2m (native) -> mpeg4 (native)) Stream #0:0 -> #0:1 (wmav2 (native) -> mp3 (libmp3lame)) Press [q] to stop, [?] for help [libmp3lame @ 0x55c17cf03140] Queue input is backward in time Output #0, avi, to 'file-g2m5.avi': Metadata: DeviceConformanceTemplate: L2 WMFSDKNeeded : 0.0.0.0000 WMFSDKVersion : 12.0.7601.17514 IsVBR : 1 WM/ToolVersion : 6.4.3 Build 1767 WM/ToolName : GoToMeeting BitRateFrom the writer: 97087 Audio samples : 34341 Video samples : 3740 recording time : Fri, 17 Oct 2014 12:28:16 Eastern Daylight Time Stream #0:0: Video: mpeg4 (FMP4 / 0x34504D46), yuv420p(tv, progressive), 1440x900, q=2-31, 200 kb/s, 1k fps, 1k tbn Metadata: encoder : Lavc mpeg4 Side data: cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A Stream #0:1: Audio: mp3 (U[0][0][0] / 0x0055), 44100 Hz, mono, fltp Metadata: encoder : Lavc libmp3lame [avi @ 0x55c17cf31340] Too large number of skipped frames 194184 > 600000kbits/s speed= 140x av_interleaved_write_frame(): Invalid argument Error muxing a packet for output file #0 [avi @ 0x55c17cf31340] Too large number of skipped frames 194085 > 60000 frame= 2 fps=1.3 q=2.0 Lsize= 1855kB time=00:03:14.18 bitrate= 78.3kbits/s speed= 129x video:149kB audio:1517kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 11.381945% Conversion failed!
The new code ends up sending a few more audio packets to the muxer, so av_interleaved_write_frame() returns an error, which results in a non-zero exit status. In the current code, the same error appears during flushing the muxing queue in av_write_trailer() and is actually returned from av_write_trailer(), but ffmpeg for some reason does not treat av_write_trailer() errors the same way. I do not know why that is, seems like a bug to me. In any case, conversion is quite broken both before and after my patches, so I would say it is not a problem with my code.
./ffmpeg -i tickets/1666/avc-intra-panasonic-AG-HPX301E.mov -vframes 3 -aframes 2 -bitexact -f framecrc - duplicate behavior of the a/vframe issue above, one stream disappears
sample in https://samples.ffmpeg.org/ffmpeg-bugs/trac/ticket1666/
Again, should be resolved by the updated patch. -- Anton Khirnov
On Wed, Jul 13, 2022 at 12:58:54PM +0200, Anton Khirnov wrote:
Quoting Michael Niedermayer (2022-07-08 18:58:11)
On Fri, Jun 17, 2022 at 12:27:18PM +0200, Anton Khirnov wrote:
The current version of this set can also be found in my tree git://git.khirnov.net/libav branch ffmpeg_mt/mux
There are really many files changing, its hard to say for sure that all are the same issue, but basically it all seems more or less frames in some streams including cases where there are hugely more or 0
Here are some examples:
ffmpeg -i matrixbench_mpeg2.mpg -vcodec rawvideo -pix_fmt rgb555 -allow_raw_vfw 1 -vframes 1 -bitexact file-rgb555.mkv
the new file is much bigger (due to the audio track)
-rw-r----- 1 michael michael 2765813 Jul 8 16:57 file-rgb555.mkv -rw-r----- 1 michael michael 834643 Jul 8 17:02 file-rgb555-ref.mkv
Where can I find this file?
the 2 file-rgb555.mkv files are the outputs from ffmpeg from the matrixbench_mpeg2 file and teh command line above. From before and after the change IIRC matrixbench_mpeg2.mpg should be here: https://samples.ffmpeg.org/benchmark/testsuite1/ thx [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB The smallest minority on earth is the individual. Those who deny individual rights cannot claim to be defenders of minorities. - Ayn Rand
Quoting Michael Niedermayer (2022-07-14 00:12:59)
On Wed, Jul 13, 2022 at 12:58:54PM +0200, Anton Khirnov wrote:
Quoting Michael Niedermayer (2022-07-08 18:58:11)
On Fri, Jun 17, 2022 at 12:27:18PM +0200, Anton Khirnov wrote:
The current version of this set can also be found in my tree git://git.khirnov.net/libav branch ffmpeg_mt/mux
There are really many files changing, its hard to say for sure that all are the same issue, but basically it all seems more or less frames in some streams including cases where there are hugely more or 0
Here are some examples:
ffmpeg -i matrixbench_mpeg2.mpg -vcodec rawvideo -pix_fmt rgb555 -allow_raw_vfw 1 -vframes 1 -bitexact file-rgb555.mkv
the new file is much bigger (due to the audio track)
-rw-r----- 1 michael michael 2765813 Jul 8 16:57 file-rgb555.mkv -rw-r----- 1 michael michael 834643 Jul 8 17:02 file-rgb555-ref.mkv
Where can I find this file?
the 2 file-rgb555.mkv files are the outputs from ffmpeg from the matrixbench_mpeg2 file and teh command line above. From before and after the change IIRC
matrixbench_mpeg2.mpg should be here: https://samples.ffmpeg.org/benchmark/testsuite1/
The output is ~815kB in the current branch, same as before. So I assume the problem got fixed by the changes I did for one of the other samples. Thanks for testing and let me know if you find any other issues. -- Anton Khirnov
participants (3)
-
Andreas Rheinhardt -
Anton Khirnov -
Michael Niedermayer