January 2026 - ffmpeg-devel - lists.ffmpeg.org

[PR] avfilter/formats,vsrc_testsrc: use type-safe formats functions (PR #21606)
by mkver 30 Jan '26

30 Jan '26

PR #21606 opened by mkver URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21606 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21606.patch From 53a0e6e90bd5c54cf0a9423d9272a69dc9bcfef9 Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt <andreas.rheinhardt(a)outlook.com> Date: Fri, 30 Jan 2026 00:56:49 +0100 Subject: [PATCH 1/2] avfilter/formats,vsrc_testsrc: use type-safe formats functions Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt(a)outlook.com> --- libavfilter/formats.c | 4 ++-- libavfilter/vsrc_testsrc.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libavfilter/formats.c b/libavfilter/formats.c index 2186eb1619..6fbdeb2d26 100644 --- a/libavfilter/formats.c +++ b/libavfilter/formats.c @@ -1179,11 +1179,11 @@ int ff_default_query_formats(AVFilterContext *ctx) switch (f->formats_state) { case FF_FILTER_FORMATS_PIXFMT_LIST: type = AVMEDIA_TYPE_VIDEO; - formats = ff_make_format_list(f->formats.pixels_list); + formats = ff_make_pixel_format_list(f->formats.pixels_list); break; case FF_FILTER_FORMATS_SAMPLEFMTS_LIST: type = AVMEDIA_TYPE_AUDIO; - formats = ff_make_format_list(f->formats.samples_list); + formats = ff_make_sample_format_list(f->formats.samples_list); break; case FF_FILTER_FORMATS_SINGLE_PIXFMT: type = AVMEDIA_TYPE_VIDEO; diff --git a/libavfilter/vsrc_testsrc.c b/libavfilter/vsrc_testsrc.c index 4d7288f18a..bfc3491596 100644 --- a/libavfilter/vsrc_testsrc.c +++ b/libavfilter/vsrc_testsrc.c @@ -1493,7 +1493,7 @@ static int smptebars_query_formats(const AVFilterContext *ctx, if ((ret = ff_set_common_color_ranges2(ctx, cfg_in, cfg_out, ff_make_formats_list_singleton(AVCOL_RANGE_MPEG)))) return ret; - return ff_set_common_formats_from_list2(ctx, cfg_in, cfg_out, smptebars_pix_fmts); + return ff_set_pixel_formats_from_list2(ctx, cfg_in, cfg_out, smptebars_pix_fmts); } AVFILTER_DEFINE_CLASS_EXT(palbars, "pal(75|100)bars", options); @@ -2260,7 +2260,7 @@ static int zoneplate_query_formats(const AVFilterContext *ctx, if ((ret = ff_set_common_color_ranges2(ctx, cfg_in, cfg_out, ff_make_formats_list_singleton(AVCOL_RANGE_JPEG)))) return ret; - return ff_set_common_formats_from_list2(ctx, cfg_in, cfg_out, zoneplate_pix_fmts); + return ff_set_pixel_formats_from_list2(ctx, cfg_in, cfg_out, zoneplate_pix_fmts); } static const AVFilterPad avfilter_vsrc_zoneplate_outputs[] = { -- 2.52.0 From 32793f755fb605bbf7f4970adc1566fb4ba761ca Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt <andreas.rheinhardt(a)outlook.com> Date: Fri, 30 Jan 2026 01:01:45 +0100 Subject: [PATCH 2/2] fftools/ffmpeg_{filter,mux_init}: Improve type-safety Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt(a)outlook.com> --- fftools/ffmpeg.h | 5 ++++- fftools/ffmpeg_filter.c | 15 +++++++++------ fftools/ffmpeg_mux_init.c | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 7720dd9c59..3e66534d48 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -347,7 +347,10 @@ typedef struct OutputFilterOptions { int sample_rate; AVChannelLayout ch_layout; - const int *formats; + union { + const enum AVPixelFormat *pix_fmts; + const enum AVSampleFormat *sample_fmts; + }; const int *sample_rates; const AVChannelLayout *ch_layouts; const AVRational *frame_rates; diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c index 4387c992da..765b65d0ec 100644 --- a/fftools/ffmpeg_filter.c +++ b/fftools/ffmpeg_filter.c @@ -223,7 +223,10 @@ typedef struct OutputFilterPriv { // those are only set if no format is specified and the encoder gives us multiple options // They point directly to the relevant lists of the encoder. - const int *formats; + union { + const enum AVPixelFormat *pix_fmts; + const enum AVSampleFormat *sample_fmts; + }; const AVChannelLayout *ch_layouts; const int *sample_rates; const enum AVColorSpace *color_spaces; @@ -399,10 +402,10 @@ static void choose_ ## name (OutputFilterPriv *ofp, AVBPrint *bprint) \ av_bprint_chars(bprint, ':', 1); \ } -DEF_CHOOSE_FORMAT(pix_fmts, enum AVPixelFormat, format, formats, +DEF_CHOOSE_FORMAT(pix_fmts, enum AVPixelFormat, format, pix_fmts, AV_PIX_FMT_NONE, "%s", av_get_pix_fmt_name) -DEF_CHOOSE_FORMAT(sample_fmts, enum AVSampleFormat, format, formats, +DEF_CHOOSE_FORMAT(sample_fmts, enum AVSampleFormat, format, sample_fmts, AV_SAMPLE_FMT_NONE, "%s", av_get_sample_fmt_name) DEF_CHOOSE_FORMAT(sample_rates, int, sample_rate, sample_rates, 0, @@ -859,7 +862,7 @@ int ofilter_bind_enc(OutputFilter *ofilter, unsigned sched_idx_enc, if (opts->format != AV_PIX_FMT_NONE) { ofp->format = opts->format; } else - ofp->formats = opts->formats; + ofp->pix_fmts = opts->pix_fmts; if (opts->color_space != AVCOL_SPC_UNSPECIFIED) ofp->color_space = opts->color_space; @@ -898,7 +901,7 @@ int ofilter_bind_enc(OutputFilter *ofilter, unsigned sched_idx_enc, if (opts->format != AV_SAMPLE_FMT_NONE) { ofp->format = opts->format; } else { - ofp->formats = opts->formats; + ofp->sample_fmts = opts->sample_fmts; } if (opts->sample_rate) { ofp->sample_rate = opts->sample_rate; @@ -1702,7 +1705,7 @@ static int configure_output_video_filter(FilterGraphPriv *fgp, AVFilterGraph *gr } av_assert0(!(ofp->flags & OFILTER_FLAG_DISABLE_CONVERT) || - ofp->format != AV_PIX_FMT_NONE || !ofp->formats); + ofp->format != AV_PIX_FMT_NONE || !ofp->pix_fmts); av_bprint_init(&bprint, 0, AV_BPRINT_SIZE_UNLIMITED); choose_pix_fmts(ofp, &bprint); choose_color_spaces(ofp, &bprint); diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c index 0569f62836..4cb8f91d6e 100644 --- a/fftools/ffmpeg_mux_init.c +++ b/fftools/ffmpeg_mux_init.c @@ -949,7 +949,7 @@ ost_bind_filter(const Muxer *mux, MuxStream *ms, OutputFilter *ofilter, if (!keep_pix_fmt) { ret = avcodec_get_supported_config(enc_ctx, NULL, AV_CODEC_CONFIG_PIX_FORMAT, 0, - (const void **) &opts.formats, NULL); + (const void **) &opts.pix_fmts, NULL); if (ret < 0) return ret; } @@ -978,7 +978,7 @@ ost_bind_filter(const Muxer *mux, MuxStream *ms, OutputFilter *ofilter, } else { ret = avcodec_get_supported_config(enc_ctx, NULL, AV_CODEC_CONFIG_SAMPLE_FORMAT, 0, - (const void **) &opts.formats, NULL); + (const void **) &opts.sample_fmts, NULL); if (ret < 0) return ret; ret = avcodec_get_supported_config(enc_ctx, NULL, -- 2.52.0

1 0

FOSDEM 26
by Michael Niedermayer 29 Jan '26

29 Jan '26

Hi Everyone I think FOSDEM 2026 is soon, no sadly iam not physically there but iam interrested in hearing any FFmpeg related talks and anything else related to FFmpeg. Will there be a way to remotely participate? Will there be a recording of talks and discussions ? thx -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB The real ebay dictionary, page 1 "Used only once" - "Some unspecified defect prevented a second use" "In good condition" - "Can be repaird by experienced expert" "As is" - "You wouldnt want it even if you were payed for it, if you knew ..."

3 2

[PR] fftools/ffprobe: keep decoder buffers unflushed for show_streams() (PR #21605)
by ngaullier 29 Jan '26

29 Jan '26

PR #21605 opened by ngaullier URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21605 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21605.patch When a decoder buffer is flushed, parts of the private context is reset, which may affect show_streams(). Example: ffprobe -of flat fate-suite/ac3/mp3ac325-4864-small.ts \ -analyze_frames -show_entries stream=ltrt_cmixlev Before: ltrt_cmixlev="0.000000" After: ltrt_cmixlev="0.707107" Currently, it seems that only ac3 downmix info is concerned. (ac3 downmix options are exported since 376bb8481ada994ffe16d0a02f6c8767bac21016). Fix regression since 045a8b15b19ec7f872fb01cfb986faeaa26cb8bb. Signed-off-by: Nicolas Gaullier <nicolas.gaullier(a)cji.paris> From 0204f391722f8c82d49bb94220ef7c8429d2c6e5 Mon Sep 17 00:00:00 2001 From: Nicolas Gaullier <nicolas.gaullier(a)cji.paris> Date: Tue, 27 Jan 2026 17:29:11 +0100 Subject: [PATCH] fftools/ffprobe: keep decoder buffers unflushed for show_streams() When a decoder buffer is flushed, parts of the private context is reset, which may affect show_streams(). Example: ffprobe -of flat fate-suite/ac3/mp3ac325-4864-small.ts \ -analyze_frames -show_entries stream=ltrt_cmixlev Before: ltrt_cmixlev="0.000000" After: ltrt_cmixlev="0.707107" Currently, it seems that only ac3 downmix info is concerned. (ac3 downmix options are exported since 376bb8481ada994ffe16d0a02f6c8767bac21016). Fix regression since 045a8b15b19ec7f872fb01cfb986faeaa26cb8bb. Signed-off-by: Nicolas Gaullier <nicolas.gaullier(a)cji.paris> --- fftools/ffprobe.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/fftools/ffprobe.c b/fftools/ffprobe.c index 67111bc31f..f256d781d0 100644 --- a/fftools/ffprobe.c +++ b/fftools/ffprobe.c @@ -1695,8 +1695,6 @@ static int read_interval_packets(AVTextFormatContext *tfc, InputFile *ifile, pkt->stream_index = i; if (do_read_frames) { while (process_frame(tfc, ifile, frame, pkt, &(int){1}) > 0); - if (ifile->streams[i].dec_ctx) - avcodec_flush_buffers(ifile->streams[i].dec_ctx); } } @@ -1710,6 +1708,18 @@ end: return ret; } +static void flush_buffers(InputFile *ifile) +{ + int i; + + if (!do_read_frames) + return; + for (i = 0; i < ifile->nb_streams; i++) { + if (ifile->streams[i].dec_ctx) + avcodec_flush_buffers(ifile->streams[i].dec_ctx); + } +} + static int read_packets(AVTextFormatContext *tfc, InputFile *ifile) { AVFormatContext *fmt_ctx = ifile->fmt_ctx; @@ -1721,6 +1731,10 @@ static int read_packets(AVTextFormatContext *tfc, InputFile *ifile) ret = read_interval_packets(tfc, ifile, &interval, &cur_ts); } else { for (i = 0; i < read_intervals_nb; i++) { + /* flushing buffers can reset parts of the private context which may be + * readen by show_streams(), so only flush between each read_interval */ + if (i) + flush_buffers(ifile); ret = read_interval_packets(tfc, ifile, &read_intervals[i], &cur_ts); if (ret < 0) break; -- 2.52.0

1 0

[PR] avfilter: version bump for filter mestimate_d3d12 (PR #21604)
by Steven Xiao 29 Jan '26

29 Jan '26

PR #21604 opened by Steven Xiao (younengxiao) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21604 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21604.patch Bump library version for filter "mestimate_d3d12". This is the follow-up for commit https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21217/commits Signed-off-by: younengxiao <steven.xiao(a)amd.com> From 2fbb66614114a1274857e84a035dec9d15f13c63 Mon Sep 17 00:00:00 2001 From: stevxiao <steven.xiao(a)amd.com> Date: Thu, 29 Jan 2026 09:06:20 -0500 Subject: [PATCH] avfilter: version bump for filter mestimate_d3d12 Bump library version for filter "mestimate_d3d12". This is the follow-up for commit https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21217/commits Signed-off-by: younengxiao <steven.xiao(a)amd.com> --- libavfilter/version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavfilter/version.h b/libavfilter/version.h index 44dacc29be..64cd692ab6 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -31,8 +31,8 @@ #include "version_major.h" -#define LIBAVFILTER_VERSION_MINOR 11 -#define LIBAVFILTER_VERSION_MICRO 101 +#define LIBAVFILTER_VERSION_MINOR 12 +#define LIBAVFILTER_VERSION_MICRO 100 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \ -- 2.52.0

1 0

Re: FOSDEM 26
by Michael Niedermayer 29 Jan '26

29 Jan '26

Hi Kieran it seems you forgot to CC the list On Thu, Jan 29, 2026 at 02:09:50PM +0100, Kieran Kunhya wrote: > I will be having discussions with other FFmpeg developers in a bar at > FOSDEM and I don't plan to record these conversations. Thats of course your right. And not what i meant. thx > > Kieran > > On Thu, 29 Jan 2026, 14:01 Michael Niedermayer via ffmpeg-devel, < > ffmpeg-devel(a)ffmpeg.org> wrote: > > > Hi Everyone > > > > I think FOSDEM 2026 is soon, no sadly iam not physically there > > but iam interrested in hearing any FFmpeg related talks and anything > > else related to FFmpeg. > > > > Will there be a way to remotely participate? > > Will there be a recording of talks and discussions ? > > > > thx > > > > -- > > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > > > The real ebay dictionary, page 1 > > "Used only once" - "Some unspecified defect prevented a second use" > > "In good condition" - "Can be repaird by experienced expert" > > "As is" - "You wouldnt want it even if you were payed for it, if you knew > > ..." > > _______________________________________________ > > ffmpeg-devel mailing list -- ffmpeg-devel(a)ffmpeg.org > > To unsubscribe send an email to ffmpeg-devel-leave(a)ffmpeg.org > > -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB The greatest way to live with honor in this world is to be what we pretend to be. -- Socrates

1 0

Parsing of loudness mp4 boxes ludt, tlou and alou FFmpeg patch
by Tomasz Rosinski 29 Jan '26

29 Jan '26

Hi, please review attached patch containing parsing of loudness mp4 boxes implementation. Best regards, Tomasz Rosinski

1 0

[PR] avformat/whep: add WHEP demuxer support (PR #21603)
by Artem Smorodin 29 Jan '26

29 Jan '26

PR #21603 opened by Artem Smorodin (artem.smorodin.dacast) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21603 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21603.patch This PR contains the implementation of WHEP (WebRTC-HTTP Egress Protocol). To implement it, I had to split `whip.c` and separate the RTC/ICE/DTLS/SRTP part (`rtc.c`/`rtc.h`) from it. WHIP remained almost unchanged (except for the name of the `whip_flags` option). I changed the names of some variables and structures from whip to rtc for consistency, but did not change the logic. The only important change for WHIP is renaming `whip_flags` to `rtc_flags`. The old name remains in place but is marked as deprecated. Let me know if you find this unacceptable. As for WHEP: This demuxer was implemented using the same RTC codebase as whip. During implementation, I also looked at how it was implemented for rtsp. This demuxer also supports media timeout and an option to set the reordering queue size. Here are the WHEP-related features that have been implemented: 1. Generation and parsing of SDP offer 2. Full NACK support (NACK PLI implemented, but will not work for H.264 as `need_keyframe` is not implemented for `rtpdec_h264`) 3. RTX support 4. Reorder queue (including drain by max_delay) 5. RTCP feedback TODO: 1. Only H.264 and OPUS supported right now (to extend it, mainly need to add codecs to SDP generation/parsing) 2. Full ICE is not implemented (same as for WHIP) 3. TWCC and REMB is not implemented 4. Probably many other minor WebRTC features I tested this using: 1. ossrs/srs (supports NACK, but not RTX) 2. Cloudflare Streams (supports NACK + RTX)| @JackLau @lq This may be presumptuous of me, but I would like to ask you, if possible, to prioritize reviewing this PR over working on WHIP, as each merge will be a nightmare for me. Thank you for your understanding. From 86ac37cdb4b7b21e0b60d62321a85312e99f9251 Mon Sep 17 00:00:00 2001 From: Artem Smorodin <artem.smorodin(a)dacast.com> Date: Thu, 29 Jan 2026 16:04:51 +0300 Subject: [PATCH] avformat/whep: add WHEP demuxer support avformat/whip: split whip.c to whip and rtc code --- configure | 2 + doc/demuxers.texi | 61 ++ doc/muxers.texi | 4 +- libavformat/Makefile | 3 +- libavformat/allformats.c | 1 + libavformat/rtc.c | 1105 +++++++++++++++++++++++++ libavformat/rtc.h | 274 +++++++ libavformat/whep.c | 1119 +++++++++++++++++++++++++ libavformat/whip.c | 1660 +++++--------------------------------- 9 files changed, 2761 insertions(+), 1468 deletions(-) create mode 100644 libavformat/rtc.c create mode 100644 libavformat/rtc.h create mode 100644 libavformat/whep.c diff --git a/configure b/configure index 6d6e6673dc..ed8fcdf36d 100755 --- a/configure +++ b/configure @@ -3902,6 +3902,8 @@ wav_demuxer_select="riffdec" wav_muxer_select="riffenc" webm_chunk_muxer_select="webm_muxer" webm_dash_manifest_demuxer_select="matroska_demuxer" +whep_demuxer_deps_any="dtls_protocol" +whep_demuxer_select="rtpdec http_protocol" whip_muxer_deps_any="dtls_protocol" whip_muxer_select="rtp_muxer http_protocol" wtv_demuxer_select="mpegts_demuxer riffdec" diff --git a/doc/demuxers.texi b/doc/demuxers.texi index c1dda7f1eb..a4e8beb0ee 100644 --- a/doc/demuxers.texi +++ b/doc/demuxers.texi @@ -1179,4 +1179,65 @@ this is set to 0, which means that a sensible value is chosen based on the input format. @end table +@anchor{whep} +@section whep + +WebRTC (Real-Time Communication) demuxer that supports sub-second latency playback according to +the WHEP (WebRTC-HTTP Egress Protocol) specification. + +This is an experimental feature. + +It uses HTTP as a signaling protocol to exchange SDP capabilities and ICE lite candidates. Then, +it uses STUN binding requests and responses to establish a session over UDP. Subsequently, it +initiates a DTLS handshake to exchange the SRTP encryption keys. Lastly, it receives RTP packets +containing audio and video and decrypt them using SRTP. + +Depending on the RTT to the peer, you may need to set the maximum possible delay (max_delay). +@example +ffplay -max_delay 500000 -f whep -i http://localhost:1985/rtc/v1/whep/?stream=livestream +@end example + +@subsection Options + +This demuxer supports the following options: + +@table @option + +@item handshake_timeout @var{integer} +Set the timeout in milliseconds for ICE and DTLS handshake. +Default value is 5000. + +@item media_timeout @var{integer} +Set the timeout in milliseconds for receiving media packets. +Default value is 0(disabled). + +@item pkt_size @var{integer} +Set the maximum size, in bytes, of RTP packets that send out. +Default value is 1200. + +@item buffer_size, ts_buffer_size @var{integer} +Set the buffer size, in bytes, of underlying protocol. +Default value is -1(auto). The UDP auto selects a reasonable value. + +Using the buffer_size option name is deprecated and should not be used. + +@item rtc_flags @var{flags} +Possible values: + +@table @samp +@item dtls_active +The demuxer will try to set dtls active role and send the first client hello. +@end table + +@item authorization @var{string} +The optional Bearer token for WHEP Authorization. + +@item cert_file @var{string} +The optional certificate file path for DTLS. + +@item key_file @var{string} +The optional private key file path for DTLS. + +@end table + @c man end DEMUXERS diff --git a/doc/muxers.texi b/doc/muxers.texi index e1f737b1d9..b0af0494f0 100644 --- a/doc/muxers.texi +++ b/doc/muxers.texi @@ -3959,7 +3959,7 @@ Default value is 1200. Set the buffer size, in bytes, of underlying protocol. Default value is -1(auto). The UDP auto selects a reasonable value. -@item whip_flags @var{flags} +@item whip_flags, rtc_flags @var{flags} Possible values: @table @samp @@ -3967,6 +3967,8 @@ Possible values: The muxer will try to set dtls active role and send the first client hello. @end table +Using the whip_flags option name is deprecated and should not be used. + @item authorization @var{string} The optional Bearer token for WHIP Authorization. diff --git a/libavformat/Makefile b/libavformat/Makefile index 5fd3f7252a..4ef2fae93e 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -645,7 +645,8 @@ OBJS-$(CONFIG_WEBM_CHUNK_MUXER) += webm_chunk.o OBJS-$(CONFIG_WEBP_MUXER) += webpenc.o OBJS-$(CONFIG_WEBVTT_DEMUXER) += webvttdec.o subtitles.o OBJS-$(CONFIG_WEBVTT_MUXER) += webvttenc.o -OBJS-$(CONFIG_WHIP_MUXER) += whip.o avc.o http.o srtp.o +OBJS-$(CONFIG_WHEP_DEMUXER) += whep.o rtc.o http.o srtp.o +OBJS-$(CONFIG_WHIP_MUXER) += whip.o rtc.o avc.o http.o srtp.o OBJS-$(CONFIG_WSAUD_DEMUXER) += westwood_aud.o OBJS-$(CONFIG_WSAUD_MUXER) += westwood_audenc.o OBJS-$(CONFIG_WSD_DEMUXER) += wsddec.o rawdec.o diff --git a/libavformat/allformats.c b/libavformat/allformats.c index 6ec361fb7b..e2a366cea1 100644 --- a/libavformat/allformats.c +++ b/libavformat/allformats.c @@ -518,6 +518,7 @@ extern const FFOutputFormat ff_webp_muxer; extern const FFInputFormat ff_webvtt_demuxer; extern const FFOutputFormat ff_webvtt_muxer; extern const FFInputFormat ff_wsaud_demuxer; +extern const FFInputFormat ff_whep_demuxer; extern const FFOutputFormat ff_whip_muxer; extern const FFOutputFormat ff_wsaud_muxer; extern const FFInputFormat ff_wsd_demuxer; diff --git a/libavformat/rtc.c b/libavformat/rtc.c new file mode 100644 index 0000000000..40e44d43ca --- /dev/null +++ b/libavformat/rtc.c @@ -0,0 +1,1105 @@ +/* + * RTC input/output format + * Copyright (c) 2023 The FFmpeg Project + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "avio_internal.h" +#include "libavutil/base64.h" +#include "libavutil/bprint.h" +#include "libavutil/crc.h" +#include "libavutil/hmac.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/mem.h" +#include "libavutil/random_seed.h" +#include "libavutil/time.h" +#include "http.h" +#include "internal.h" +#include "mux.h" +#include "network.h" +#include "rtp.h" +#include "rtc.h" + +/** + * If we try to read from UDP and get EAGAIN, we sleep for 5ms and retry up to 10 times. + * This will limit the total duration (in milliseconds, 50ms) + */ +#define ICE_DTLS_READ_MAX_RETRY 10 +#define ICE_DTLS_READ_SLEEP_DURATION 5 + +/* The magic cookie for Session Traversal Utilities for NAT (STUN) messages. */ +#define STUN_MAGIC_COOKIE 0x2112A442 + +/** + * Refer to RFC 8445 5.1.2 + * priority = (2^24)*(type preference) + (2^8)*(local preference) + (2^0)*(256 - component ID) + * host candidate priority is 126 << 24 | 65535 << 8 | 255 + */ +#define STUN_HOST_CANDIDATE_PRIORITY 126 << 24 | 65535 << 8 | 255 + +/** + * The DTLS content type. + * See https://tools.ietf.org/html/rfc2246#section-6.2.1 + * change_cipher_spec(20), alert(21), handshake(22), application_data(23) + */ +#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20 + +/** + * The DTLS record layer header has a total size of 13 bytes, consisting of + * ContentType (1 byte), ProtocolVersion (2 bytes), Epoch (2 bytes), + * SequenceNumber (6 bytes), and Length (2 bytes). + * See https://datatracker.ietf.org/doc/html/rfc9147#section-4 + */ +#define DTLS_RECORD_LAYER_HEADER_LEN 13 + +/** + * The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2. + * See https://datatracker.ietf.org/doc/html/rfc9147#name-the-dtls-record-layer + */ +#define DTLS_VERSION_10 0xfeff +#define DTLS_VERSION_12 0xfefd + +/** + * The STUN message header, which is 20 bytes long, comprises the + * STUNMessageType (1B), MessageLength (2B), MagicCookie (4B), + * and TransactionID (12B). + * See https://datatracker.ietf.org/doc/html/rfc5389#section-6 + */ +#define ICE_STUN_HEADER_SIZE 20 + +/** + * The RTP header is 12 bytes long, comprising the Version(1B), PT(1B), + * SequenceNumber(2B), Timestamp(4B), and SSRC(4B). + * See https://www.rfc-editor.org/rfc/rfc3550#section-5.1 + */ +#define RTC_RTP_HEADER_SIZE 12 + +/** + * For RTCP, PT is [128, 223] (or without marker [0, 95]). Literally, RTCP starts + * from 64 not 0, so PT is [192, 223] (or without marker [64, 95]), see "RTCP Control + * Packet Types (PT)" at + * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-pa… + * + * For RTP, the PT is [96, 127], or [224, 255] with marker. See "RTP Payload Types (PT) + * for standard audio and video encodings" at + * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-pa… + */ +#define RTC_RTCP_PT_START 192 +#define RTC_RTCP_PT_END 223 + +/* STUN Attribute, comprehension-required range (0x0000-0x7FFF) */ +enum STUNAttr { + STUN_ATTR_USERNAME = 0x0006, /// shared secret response/bind request + STUN_ATTR_PRIORITY = 0x0024, /// must be included in a Binding request + STUN_ATTR_USE_CANDIDATE = 0x0025, /// bind request + STUN_ATTR_MESSAGE_INTEGRITY = 0x0008, /// bind request/response + STUN_ATTR_FINGERPRINT = 0x8028, /// rfc5389 + STUN_ATTR_ICE_CONTROLLING = 0x802A, /// ICE controlling role +}; + +/** + * Whether the packet is a DTLS packet. + */ +int ff_rtc_is_dtls_packet(uint8_t *b, int size) +{ + int ret = 0; + if (size > DTLS_RECORD_LAYER_HEADER_LEN) { + uint16_t version = AV_RB16(&b[1]); + ret = b[0] >= DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC && + (version == DTLS_VERSION_10 || version == DTLS_VERSION_12); + } + return ret; +} + + +/** + * Get or Generate a self-signed certificate and private key for DTLS, + * fingerprint for SDP + */ +static av_cold int certificate_key_init(AVFormatContext *s) +{ + int ret = 0; + RTCContext *rtc = s->priv_data; + + if (rtc->cert_file && rtc->key_file) { + /* Read the private key and certificate from the file. */ + if ((ret = ff_ssl_read_key_cert(rtc->key_file, rtc->cert_file, + rtc->key_buf, sizeof(rtc->key_buf), + rtc->cert_buf, sizeof(rtc->cert_buf), + &rtc->dtls_fingerprint)) < 0) { + av_log(s, AV_LOG_ERROR, "Failed to read DTLS certificate from cert=%s, key=%s\n", + rtc->cert_file, rtc->key_file); + return ret; + } + } else { + /* Generate a private key to ctx->dtls_pkey and self-signed certificate. */ + if ((ret = ff_ssl_gen_key_cert(rtc->key_buf, sizeof(rtc->key_buf), + rtc->cert_buf, sizeof(rtc->cert_buf), + &rtc->dtls_fingerprint)) < 0) { + av_log(s, AV_LOG_ERROR, "Failed to generate DTLS private key and certificate\n"); + return ret; + } + } + + return ret; +} + +static av_cold int dtls_initialize(AVFormatContext *s) +{ + int ret = 0; + RTCContext *rtc = s->priv_data; + int is_dtls_active = rtc->flags & RTC_DTLS_ACTIVE; + AVDictionary *opts = NULL; + char buf[256]; + + ff_url_join(buf, sizeof(buf), "dtls", NULL, rtc->ice_host, rtc->ice_port, NULL); + av_dict_set_int(&opts, "mtu", rtc->pkt_size, 0); + if (rtc->cert_file) { + av_dict_set(&opts, "cert_file", rtc->cert_file, 0); + } else + av_dict_set(&opts, "cert_pem", rtc->cert_buf, 0); + + if (rtc->key_file) { + av_dict_set(&opts, "key_file", rtc->key_file, 0); + } else + av_dict_set(&opts, "key_pem", rtc->key_buf, 0); + av_dict_set_int(&opts, "external_sock", 1, 0); + av_dict_set_int(&opts, "use_srtp", 1, 0); + av_dict_set_int(&opts, "listen", is_dtls_active ? 0 : 1, 0); + ret = ffurl_open_whitelist(&rtc->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, + &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); + av_dict_free(&opts); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to open DTLS url:%s\n", buf); + goto end; + } + /* reuse the udp created by rtc */ + ff_tls_set_external_socket(rtc->dtls_uc, rtc->udp); +end: + return ret; +} + +/** + * Initialize and check the options for the WebRTC muxer. + */ +av_cold int ff_rtc_initialize(AVFormatContext *s) +{ + int ret, ideal_pkt_size = 532; + RTCContext *rtc = s->priv_data; + uint32_t seed; + + rtc->rtc_starttime = av_gettime_relative(); + + ret = certificate_key_init(s); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to init certificate and key\n"); + return ret; + } + + /* Initialize the random number generator. */ + seed = av_get_random_seed(); + av_lfg_init(&rtc->rnd, seed); + + /* 64 bit tie breaker for ICE-CONTROLLING (RFC 8445 16.1) */ + ret = av_random_bytes((uint8_t *)&rtc->ice_tie_breaker, sizeof(rtc->ice_tie_breaker)); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Couldn't generate random bytes for ICE tie breaker\n"); + return ret; + } + + rtc->audio_first_seq = av_lfg_get(&rtc->rnd) & 0x0fff; + rtc->video_first_seq = rtc->audio_first_seq + 1; + + if (rtc->pkt_size < ideal_pkt_size) + av_log(rtc, AV_LOG_WARNING, "pkt_size=%d(<%d) is too small, may cause packet loss\n", + rtc->pkt_size, ideal_pkt_size); + + if (rtc->state < RTC_STATE_INIT) + rtc->state = RTC_STATE_INIT; + rtc->rtc_init_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "Init state=%d, handshake_timeout=%dms, pkt_size=%d, seed=%d, elapsed=%.2fms\n", + rtc->state, rtc->handshake_timeout, rtc->pkt_size, seed, RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative())); + + return 0; +} + +/** + * Exchange SDP offer with WebRTC peer to get the answer. + * + * @return 0 if OK, AVERROR_xxx on error + */ +static int exchange_sdp(AVFormatContext *s) +{ + int ret; + char buf[MAX_URL_SIZE]; + AVBPrint bp; + RTCContext *rtc = s->priv_data; + /* The URL context is an HTTP transport layer for the WebRTC-HTTP protocol. */ + URLContext *rtc_uc = NULL; + AVDictionary *opts = NULL; + char *hex_data = NULL; + const char *proto_name = avio_find_protocol_name(s->url); + + /* To prevent a crash during cleanup, always initialize it. */ + av_bprint_init(&bp, 1, RTC_MAX_SDP_SIZE); + + if (!av_strstart(proto_name, "http", NULL)) { + av_log(rtc, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose http, url is %s\n", + proto_name, s->url); + ret = AVERROR(EINVAL); + goto end; + } + + if (!rtc->sdp_offer || !strlen(rtc->sdp_offer)) { + av_log(rtc, AV_LOG_ERROR, "No offer to exchange\n"); + ret = AVERROR(EINVAL); + goto end; + } + + ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\nContent-Type: application/sdp\r\n"); + if (rtc->authorization) + ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", rtc->authorization); + if (ret <= 0 || ret >= sizeof(buf)) { + av_log(rtc, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); + ret = AVERROR(EINVAL); + goto end; + } + + av_dict_set(&opts, "headers", buf, 0); + av_dict_set_int(&opts, "chunked_post", 0, 0); + + hex_data = av_mallocz(2 * strlen(rtc->sdp_offer) + 1); + if (!hex_data) { + ret = AVERROR(ENOMEM); + goto end; + } + ff_data_to_hex(hex_data, rtc->sdp_offer, strlen(rtc->sdp_offer), 0); + av_dict_set(&opts, "post_data", hex_data, 0); + + ret = ffurl_open_whitelist(&rtc_uc, s->url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, + &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to request url=%s, offer: %s\n", s->url, rtc->sdp_offer); + goto end; + } + + if (ff_http_get_new_location(rtc_uc)) { + rtc->rtc_resource_url = av_strdup(ff_http_get_new_location(rtc_uc)); + if (!rtc->rtc_resource_url) { + ret = AVERROR(ENOMEM); + goto end; + } + } + + while (1) { + ret = ffurl_read(rtc_uc, buf, sizeof(buf)); + if (ret == AVERROR_EOF) { + /* Reset the error because we read all response as answer util EOF. */ + ret = 0; + break; + } + if (ret <= 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to read response from url=%s, offer is %s, answer is %s\n", + s->url, rtc->sdp_offer, rtc->sdp_answer); + goto end; + } + + av_bprintf(&bp, "%.*s", ret, buf); + if (!av_bprint_is_complete(&bp)) { + av_log(rtc, AV_LOG_ERROR, "Answer exceed max size %d, %.*s, %s\n", RTC_MAX_SDP_SIZE, ret, buf, bp.str); + ret = AVERROR(EIO); + goto end; + } + } + + if (!av_strstart(bp.str, "v=", NULL)) { + av_log(rtc, AV_LOG_ERROR, "Invalid answer: %s\n", bp.str); + ret = AVERROR(EINVAL); + goto end; + } + + rtc->sdp_answer = av_strdup(bp.str); + if (!rtc->sdp_answer) { + ret = AVERROR(ENOMEM); + goto end; + } + + if (rtc->state < RTC_STATE_ANSWER) + rtc->state = RTC_STATE_ANSWER; + av_log(rtc, AV_LOG_VERBOSE, "Got state=%d, answer: %s\n", rtc->state, rtc->sdp_answer); + +end: + ffurl_closep(&rtc_uc); + av_bprint_finalize(&bp, NULL); + av_dict_free(&opts); + av_freep(&hex_data); + return ret; +} + +/** + * Parses the ICE ufrag, pwd, and candidates from the SDP answer. + * + * This function is used to extract the ICE ufrag, pwd, and candidates from the SDP answer. + * It returns an error if any of these fields is NULL. The function only uses the first + * candidate if there are multiple candidates. However, support for multiple candidates + * will be added in the future. + * + * @param s Pointer to the AVFormatContext + * @returns Returns 0 if successful or AVERROR_xxx if an error occurs. + */ +static int parse_answer(AVFormatContext *s) +{ + int ret = 0; + AVIOContext *pb; + char line[MAX_URL_SIZE]; + const char *ptr; + int i; + RTCContext *rtc = s->priv_data; + + if (!rtc->sdp_answer || !strlen(rtc->sdp_answer)) { + av_log(rtc, AV_LOG_ERROR, "No answer to parse\n"); + ret = AVERROR(EINVAL); + goto end; + } + + pb = avio_alloc_context(rtc->sdp_answer, strlen(rtc->sdp_answer), 0, NULL, NULL, NULL, NULL); + if (!pb) + return AVERROR(ENOMEM); + + for (i = 0; !avio_feof(pb); i++) { + ff_get_chomp_line(pb, line, sizeof(line)); + if (av_strstart(line, "a=ice-lite", &ptr)) + rtc->is_peer_ice_lite = 1; + if (av_strstart(line, "a=ice-ufrag:", &ptr) && !rtc->ice_ufrag_remote) { + rtc->ice_ufrag_remote = av_strdup(ptr); + if (!rtc->ice_ufrag_remote) { + ret = AVERROR(ENOMEM); + goto end; + } + } else if (av_strstart(line, "a=ice-pwd:", &ptr) && !rtc->ice_pwd_remote) { + rtc->ice_pwd_remote = av_strdup(ptr); + if (!rtc->ice_pwd_remote) { + ret = AVERROR(ENOMEM); + goto end; + } + } else if (av_strstart(line, "a=candidate:", &ptr) && !rtc->ice_protocol) { + if (ptr && av_stristr(ptr, "host")) { + /* Refer to RFC 5245 15.1 */ + char foundation[33], protocol[17], host[129]; + int component_id, priority, port; + ret = sscanf(ptr, "%32s %d %16s %d %128s %d typ host", foundation, &component_id, protocol, &priority, host, &port); + if (ret != 6) { + av_log(rtc, AV_LOG_ERROR, "Failed %d to parse line %d %s from %s\n", + ret, i, line, rtc->sdp_answer); + ret = AVERROR(EIO); + goto end; + } + + if (av_strcasecmp(protocol, "udp")) { + av_log(rtc, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n", + protocol, i, line, rtc->sdp_answer); + ret = AVERROR(EIO); + goto end; + } + + rtc->ice_protocol = av_strdup(protocol); + rtc->ice_host = av_strdup(host); + rtc->ice_port = port; + if (!rtc->ice_protocol || !rtc->ice_host) { + ret = AVERROR(ENOMEM); + goto end; + } + } + } + } + + if (!rtc->ice_pwd_remote || !strlen(rtc->ice_pwd_remote)) { + av_log(rtc, AV_LOG_ERROR, "No remote ice pwd parsed from %s\n", rtc->sdp_answer); + ret = AVERROR(EINVAL); + goto end; + } + + if (!rtc->ice_ufrag_remote || !strlen(rtc->ice_ufrag_remote)) { + av_log(rtc, AV_LOG_ERROR, "No remote ice ufrag parsed from %s\n", rtc->sdp_answer); + ret = AVERROR(EINVAL); + goto end; + } + + if (!rtc->ice_protocol || !rtc->ice_host || !rtc->ice_port) { + av_log(rtc, AV_LOG_ERROR, "No ice candidate parsed from %s\n", rtc->sdp_answer); + ret = AVERROR(EINVAL); + goto end; + } + + if (rtc->state < RTC_STATE_NEGOTIATED) + rtc->state = RTC_STATE_NEGOTIATED; + rtc->rtc_answer_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "SDP state=%d, offer=%zuB, answer=%zuB, ufrag=%s, pwd=%zuB, transport=%s://%s:%d, elapsed=%.2fms\n", + rtc->state, strlen(rtc->sdp_offer), strlen(rtc->sdp_answer), rtc->ice_ufrag_remote, strlen(rtc->ice_pwd_remote), + rtc->ice_protocol, rtc->ice_host, rtc->ice_port, RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative())); + +end: + avio_context_free(&pb); + return ret; +} + +/** + * Creates and marshals an ICE binding request packet. + * + * This function creates and marshals an ICE binding request packet. The function only + * generates the username attribute and does not include goog-network-info, + * use-candidate. However, some of these attributes may be added in the future. + * + * @param s Pointer to the AVFormatContext + * @param buf Pointer to memory buffer to store the request packet + * @param buf_size Size of the memory buffer + * @param request_size Pointer to an integer that receives the size of the request packet + * @return Returns 0 if successful or AVERROR_xxx if an error occurs. + */ +int ff_rtc_ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, int *request_size) +{ + int ret, size, crc32; + char username[128]; + AVIOContext *pb = NULL; + AVHMAC *hmac = NULL; + RTCContext *rtc = s->priv_data; + + pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL); + if (!pb) + return AVERROR(ENOMEM); + + hmac = av_hmac_alloc(AV_HMAC_SHA1); + if (!hmac) { + ret = AVERROR(ENOMEM); + goto end; + } + + /* Write 20 bytes header */ + avio_wb16(pb, 0x0001); /* STUN binding request */ + avio_wb16(pb, 0); /* length */ + avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */ + avio_wb32(pb, av_lfg_get(&rtc->rnd)); /* transaction ID */ + avio_wb32(pb, av_lfg_get(&rtc->rnd)); /* transaction ID */ + avio_wb32(pb, av_lfg_get(&rtc->rnd)); /* transaction ID */ + + /* The username is the concatenation of the two ICE ufrag */ + ret = snprintf(username, sizeof(username), "%s:%s", rtc->ice_ufrag_remote, rtc->ice_ufrag_local); + if (ret <= 0 || ret >= sizeof(username)) { + av_log(rtc, AV_LOG_ERROR, "Failed to build username %s:%s, max=%zu, ret=%d\n", + rtc->ice_ufrag_remote, rtc->ice_ufrag_local, sizeof(username), ret); + ret = AVERROR(EIO); + goto end; + } + + /* Write the username attribute */ + avio_wb16(pb, STUN_ATTR_USERNAME); /* attribute type username */ + avio_wb16(pb, ret); /* size of username */ + avio_write(pb, username, ret); /* bytes of username */ + ffio_fill(pb, 0, (4 - (ret % 4)) % 4); /* padding */ + + /* Write the use-candidate attribute */ + avio_wb16(pb, STUN_ATTR_USE_CANDIDATE); /* attribute type use-candidate */ + avio_wb16(pb, 0); /* size of use-candidate */ + + avio_wb16(pb, STUN_ATTR_PRIORITY); + avio_wb16(pb, 4); + avio_wb32(pb, STUN_HOST_CANDIDATE_PRIORITY); + + avio_wb16(pb, STUN_ATTR_ICE_CONTROLLING); + avio_wb16(pb, 8); + avio_wb64(pb, rtc->ice_tie_breaker); + + /* Build and update message integrity */ + avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */ + avio_wb16(pb, 20); /* size of message integrity */ + ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */ + size = avio_tell(pb); + buf[2] = (size - 20) >> 8; + buf[3] = (size - 20) & 0xFF; + av_hmac_init(hmac, rtc->ice_pwd_remote, strlen(rtc->ice_pwd_remote)); + av_hmac_update(hmac, buf, size - 24); + av_hmac_final(hmac, buf + size - 20, 20); + + /* Write the fingerprint attribute */ + avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */ + avio_wb16(pb, 4); /* size of fingerprint */ + ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */ + size = avio_tell(pb); + buf[2] = (size - 20) >> 8; + buf[3] = (size - 20) & 0xFF; + /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */ + crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF; + avio_skip(pb, -4); + avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */ + + *request_size = size; + +end: + avio_context_free(&pb); + av_hmac_free(hmac); + return ret; +} + +/** + * Create an ICE binding response. + * + * This function generates an ICE binding response and writes it to the provided + * buffer. The response is signed using the local password for message integrity. + * + * @param s Pointer to the AVFormatContext structure. + * @param tid Pointer to the transaction ID of the binding request. The tid_size should be 12. + * @param tid_size The size of the transaction ID, should be 12. + * @param buf Pointer to the buffer where the response will be written. + * @param buf_size The size of the buffer provided for the response. + * @param response_size Pointer to an integer that will store the size of the generated response. + * @return Returns 0 if successful or AVERROR_xxx if an error occurs. + */ +static int ice_create_response(AVFormatContext *s, char *tid, int tid_size, uint8_t *buf, int buf_size, int *response_size) +{ + int ret = 0, size, crc32; + AVIOContext *pb = NULL; + AVHMAC *hmac = NULL; + RTCContext *rtc = s->priv_data; + + if (tid_size != 12) { + av_log(rtc, AV_LOG_ERROR, "Invalid transaction ID size. Expected 12, got %d\n", tid_size); + return AVERROR(EINVAL); + } + + pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL); + if (!pb) + return AVERROR(ENOMEM); + + hmac = av_hmac_alloc(AV_HMAC_SHA1); + if (!hmac) { + ret = AVERROR(ENOMEM); + goto end; + } + + /* Write 20 bytes header */ + avio_wb16(pb, 0x0101); /* STUN binding response */ + avio_wb16(pb, 0); /* length */ + avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */ + avio_write(pb, tid, tid_size); /* transaction ID */ + + /* Build and update message integrity */ + avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */ + avio_wb16(pb, 20); /* size of message integrity */ + ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */ + size = avio_tell(pb); + buf[2] = (size - 20) >> 8; + buf[3] = (size - 20) & 0xFF; + av_hmac_init(hmac, rtc->ice_pwd_local, strlen(rtc->ice_pwd_local)); + av_hmac_update(hmac, buf, size - 24); + av_hmac_final(hmac, buf + size - 20, 20); + + /* Write the fingerprint attribute */ + avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */ + avio_wb16(pb, 4); /* size of fingerprint */ + ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */ + size = avio_tell(pb); + buf[2] = (size - 20) >> 8; + buf[3] = (size - 20) & 0xFF; + /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */ + crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF; + avio_skip(pb, -4); + avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */ + + *response_size = size; + +end: + avio_context_free(&pb); + av_hmac_free(hmac); + return ret; +} + +/** + * A Binding request has class=0b00 (request) and method=0b000000000001 (Binding) + * and is encoded into the first 16 bits as 0x0001. + * See https://datatracker.ietf.org/doc/html/rfc5389#section-6 + */ +static int ice_is_binding_request(uint8_t *b, int size) +{ + return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0001; +} + +/** + * A Binding response has class=0b10 (success response) and method=0b000000000001, + * and is encoded into the first 16 bits as 0x0101. + */ +int ff_rtc_ice_is_binding_response(uint8_t *b, int size) +{ + return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0101; +} + +/** + * In RTP packets, the first byte is represented as 0b10xxxxxx, where the initial + * two bits (0b10) indicate the RTP version, + * see https://www.rfc-editor.org/rfc/rfc3550#section-5.1 + * The RTCP packet header is similar to RTP, + * see https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1 + */ +int ff_rtc_media_is_rtp_rtcp(const uint8_t *b, int size) +{ + return size >= RTC_RTP_HEADER_SIZE && (b[0] & 0xC0) == 0x80; +} + +/* Whether the packet is RTCP. */ +int ff_rtc_media_is_rtcp(const uint8_t *b, int size) +{ + return size >= RTC_RTP_HEADER_SIZE && b[1] >= RTC_RTCP_PT_START && b[1] <= RTC_RTCP_PT_END; +} + +/** + * This function handles incoming binding request messages by responding to them. + * If the message is not a binding request, it will be ignored. + */ +int ff_rtc_ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_size) +{ + int ret = 0, size; + char tid[12]; + RTCContext *rtc = s->priv_data; + + /* Ignore if not a binding request. */ + if (!ice_is_binding_request(buf, buf_size)) + return ret; + + if (buf_size < ICE_STUN_HEADER_SIZE) { + av_log(rtc, AV_LOG_ERROR, "Invalid STUN message, expected at least %d, got %d\n", + ICE_STUN_HEADER_SIZE, buf_size); + return AVERROR(EINVAL); + } + + /* Parse transaction id from binding request in buf. */ + memcpy(tid, buf + 8, 12); + + /* Build the STUN binding response. */ + ret = ice_create_response(s, tid, sizeof(tid), rtc->buf, sizeof(rtc->buf), &size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to create STUN binding response, size=%d\n", size); + return ret; + } + + ret = ffurl_write(rtc->udp, rtc->buf, size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to send STUN binding response, size=%d\n", size); + return ret; + } + + return 0; +} + +/** + * To establish a connection with the UDP server, we utilize ICE-LITE in a Client-Server + * mode. In this setup, FFmpeg acts as the UDP client, while the peer functions as the + * UDP server. + */ +static int udp_connect(AVFormatContext *s) +{ + int ret = 0; + char url[256]; + AVDictionary *opts = NULL; + RTCContext *rtc = s->priv_data; + + /* Build UDP URL and create the UDP context as transport. */ + ff_url_join(url, sizeof(url), "udp", NULL, rtc->ice_host, rtc->ice_port, NULL); + + av_dict_set_int(&opts, "connect", 1, 0); + av_dict_set_int(&opts, "fifo_size", 0, 0); + /* Pass through the pkt_size and buffer_size to underling protocol */ + av_dict_set_int(&opts, "pkt_size", rtc->pkt_size, 0); + av_dict_set_int(&opts, "buffer_size", rtc->ts_buffer_size, 0); + + ret = ffurl_open_whitelist(&rtc->udp, url, AVIO_FLAG_WRITE, &s->interrupt_callback, + &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to connect udp://%s:%d\n", rtc->ice_host, rtc->ice_port); + goto end; + } + + /* Make the socket non-blocking, set to READ and WRITE mode after connected */ + ff_socket_nonblock(ffurl_get_file_handle(rtc->udp), 1); + rtc->udp->flags |= AVIO_FLAG_READ | AVIO_FLAG_NONBLOCK; + + if (rtc->state < RTC_STATE_UDP_CONNECTED) + rtc->state = RTC_STATE_UDP_CONNECTED; + rtc->rtc_udp_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "UDP state=%d, elapsed=%.2fms, connected to udp://%s:%d\n", + rtc->state, RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative()), rtc->ice_host, rtc->ice_port); + +end: + av_dict_free(&opts); + return ret; +} + +static int ice_dtls_handshake(AVFormatContext *s) +{ + int ret = 0, size, i; + int64_t starttime = av_gettime_relative(), now; + RTCContext *rtc = s->priv_data; + int is_dtls_active = rtc->flags & RTC_DTLS_ACTIVE; + + if (rtc->state < RTC_STATE_UDP_CONNECTED || !rtc->udp) { + av_log(rtc, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", rtc->state, rtc->udp); + return AVERROR(EINVAL); + } + + while (1) { + if (rtc->state <= RTC_STATE_ICE_CONNECTING) { + /* Build the STUN binding request. */ + ret = ff_rtc_ice_create_request(s, rtc->buf, sizeof(rtc->buf), &size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); + goto end; + } + + ret = ffurl_write(rtc->udp, rtc->buf, size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); + goto end; + } + + if (rtc->state < RTC_STATE_ICE_CONNECTING) + rtc->state = RTC_STATE_ICE_CONNECTING; + } + +next_packet: + if (rtc->state >= RTC_STATE_DTLS_FINISHED) + /* DTLS handshake is done, exit the loop. */ + break; + + now = av_gettime_relative(); + if (now - starttime >= rtc->handshake_timeout * RTC_US_PER_MS) { + av_log(rtc, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%.2fms, elapsed=%.2fms, state=%d\n", + rtc->handshake_timeout, RTC_ELAPSED(starttime, now), RTC_ELAPSED(rtc->rtc_starttime, now), rtc->state); + ret = AVERROR(ETIMEDOUT); + goto end; + } + + /* Read the STUN or DTLS messages from peer. */ + for (i = 0; i < ICE_DTLS_READ_MAX_RETRY; i++) { + if (rtc->state > RTC_STATE_ICE_CONNECTED) + break; + ret = ffurl_read(rtc->udp, rtc->buf, sizeof(rtc->buf)); + if (ret > 0) + break; + if (ret == AVERROR(EAGAIN)) { + av_usleep(ICE_DTLS_READ_SLEEP_DURATION * RTC_US_PER_MS); + continue; + } + if (is_dtls_active) + break; + av_log(rtc, AV_LOG_ERROR, "Failed to read message\n"); + goto end; + } + + /* Handle the ICE binding response. */ + if (ff_rtc_ice_is_binding_response(rtc->buf, ret)) { + if (rtc->state < RTC_STATE_ICE_CONNECTED) { + if (rtc->is_peer_ice_lite) + rtc->state = RTC_STATE_ICE_CONNECTED; + } + goto next_packet; + } + + /* When a binding request is received, it is necessary to respond immediately. */ + if (ice_is_binding_request(rtc->buf, ret)) { + if ((ret = ff_rtc_ice_handle_binding_request(s, rtc->buf, ret)) < 0) + goto end; + goto next_packet; + } + + /* Handle DTLS handshake */ + if (ff_rtc_is_dtls_packet(rtc->buf, ret) || is_dtls_active) { + rtc->rtc_ice_time = av_gettime_relative(); + /* Start consent timer when ICE selected */ + rtc->rtc_last_consent_tx_time = rtc->rtc_last_consent_rx_time = rtc->rtc_ice_time; + rtc->state = RTC_STATE_ICE_CONNECTED; + av_log(rtc, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%.2fms\n", + rtc->state, rtc->ice_host, rtc->ice_port, rtc->rtc_resource_url ? rtc->rtc_resource_url : "", + rtc->ice_ufrag_remote, rtc->ice_ufrag_local, ret, RTC_ELAPSED(rtc->rtc_starttime, rtc->rtc_ice_time)); + + ret = dtls_initialize(s); + if (ret < 0) + goto end; + ret = ffurl_handshake(rtc->dtls_uc); + if (ret < 0) { + rtc->state = RTC_STATE_FAILED; + av_log(rtc, AV_LOG_ERROR, "DTLS session failed\n"); + goto end; + } + if (!ret) { + rtc->state = RTC_STATE_DTLS_FINISHED; + rtc->rtc_dtls_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%.2fms\n", + RTC_ELAPSED(rtc->rtc_starttime, rtc->rtc_dtls_time)); + } + goto next_packet; + } + } + +end: + return ret; +} + +/** + * Establish the SRTP context using the keying material exported from DTLS. + * + * Create separate SRTP contexts for sending video and audio, as their sequences differ + * and should not share a single context. Generate a single SRTP context for receiving + * RTCP only. + * + * @return 0 if OK, AVERROR_xxx on error + */ +static int setup_srtp(AVFormatContext *s) +{ + int ret; + char recv_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN]; + char send_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN]; + char buf[AV_BASE64_SIZE(DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN)]; + /** + * The profile for OpenSSL's SRTP is SRTP_AES128_CM_SHA1_80, see ssl/d1_srtp.c. + * The profile for FFmpeg's SRTP is SRTP_AES128_CM_HMAC_SHA1_80, see libavformat/srtp.c. + */ + const char* suite = "SRTP_AES128_CM_HMAC_SHA1_80"; + RTCContext *rtc = s->priv_data; + int is_dtls_active = rtc->flags & RTC_DTLS_ACTIVE; + char *cp = is_dtls_active ? send_key : recv_key; + char *sp = is_dtls_active ? recv_key : send_key; + + ret = ff_dtls_export_materials(rtc->dtls_uc, rtc->dtls_srtp_materials, sizeof(rtc->dtls_srtp_materials)); + if (ret < 0) + goto end; + /** + * This represents the material used to build the SRTP master key. It is + * generated by DTLS and has the following layout: + * 16B 16B 14B 14B + * client_key | server_key | client_salt | server_salt + */ + char *client_key = rtc->dtls_srtp_materials; + char *server_key = rtc->dtls_srtp_materials + DTLS_SRTP_KEY_LEN; + char *client_salt = server_key + DTLS_SRTP_KEY_LEN; + char *server_salt = client_salt + DTLS_SRTP_SALT_LEN; + + memcpy(cp, client_key, DTLS_SRTP_KEY_LEN); + memcpy(cp + DTLS_SRTP_KEY_LEN, client_salt, DTLS_SRTP_SALT_LEN); + + memcpy(sp, server_key, DTLS_SRTP_KEY_LEN); + memcpy(sp + DTLS_SRTP_KEY_LEN, server_salt, DTLS_SRTP_SALT_LEN); + + /* Setup SRTP context for outgoing packets */ + if (!av_base64_encode(buf, sizeof(buf), send_key, sizeof(send_key))) { + av_log(rtc, AV_LOG_ERROR, "Failed to encode send key\n"); + ret = AVERROR(EIO); + goto end; + } + + ret = ff_srtp_set_crypto(&rtc->srtp_audio_send, suite, buf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for audio send\n"); + goto end; + } + + ret = ff_srtp_set_crypto(&rtc->srtp_video_send, suite, buf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for video send\n"); + goto end; + } + + ret = ff_srtp_set_crypto(&rtc->srtp_video_rtx_send, suite, buf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for video rtx send\n"); + goto end; + } + + ret = ff_srtp_set_crypto(&rtc->srtp_rtcp_send, suite, buf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for rtcp send\n"); + goto end; + } + + /* Setup SRTP context for incoming packets */ + if (!av_base64_encode(buf, sizeof(buf), recv_key, sizeof(recv_key))) { + av_log(rtc, AV_LOG_ERROR, "Failed to encode recv key\n"); + ret = AVERROR(EIO); + goto end; + } + + ret = ff_srtp_set_crypto(&rtc->srtp_recv, suite, buf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for recv\n"); + goto end; + } + memcpy(rtc->srtp_recv_key, recv_key, sizeof(recv_key)); + + if (rtc->state < RTC_STATE_SRTP_FINISHED) + rtc->state = RTC_STATE_SRTP_FINISHED; + rtc->rtc_srtp_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "SRTP setup done, state=%d, suite=%s, key=%zuB, elapsed=%.2fms\n", + rtc->state, suite, sizeof(send_key), RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative())); + +end: + return ret; +} + +/** + * RTC is connectionless, for it's based on UDP, so it check whether sesison is + * timeout. In such case, publishers can't republish the stream util the session + * is timeout. + * This function is called to notify the server that the stream is ended, server + * should expire and close the session immediately, so that publishers can republish + * the stream quickly. + */ +static int dispose_session(AVFormatContext *s) +{ + int ret; + char buf[MAX_URL_SIZE]; + URLContext *rtc_uc = NULL; + AVDictionary *opts = NULL; + RTCContext *rtc = s->priv_data; + + if (!rtc->rtc_resource_url) + return 0; + + ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\n"); + if (rtc->authorization) + ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", rtc->authorization); + if (ret <= 0 || ret >= sizeof(buf)) { + av_log(rtc, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); + ret = AVERROR(EINVAL); + goto end; + } + + av_dict_set(&opts, "headers", buf, 0); + av_dict_set_int(&opts, "chunked_post", 0, 0); + av_dict_set(&opts, "method", "DELETE", 0); + ret = ffurl_open_whitelist(&rtc_uc, rtc->rtc_resource_url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, + &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to DELETE url=%s\n", rtc->rtc_resource_url); + goto end; + } + + while (1) { + ret = ffurl_read(rtc_uc, buf, sizeof(buf)); + if (ret == AVERROR_EOF) { + ret = 0; + break; + } + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to read response from DELETE url=%s\n", rtc->rtc_resource_url); + goto end; + } + } + + av_log(rtc, AV_LOG_INFO, "Dispose resource %s ok\n", rtc->rtc_resource_url); + +end: + ffurl_closep(&rtc_uc); + av_dict_free(&opts); + return ret; +} + +av_cold int ff_rtc_connect(AVFormatContext *s) +{ + int ret; + + if ((ret = exchange_sdp(s)) < 0) + goto end; + + if ((ret = parse_answer(s)) < 0) + goto end; + + if ((ret = udp_connect(s)) < 0) + goto end; + + if ((ret = ice_dtls_handshake(s)) < 0) + goto end; + + if ((ret = setup_srtp(s)) < 0) + goto end; + +end: + return ret; +} + +av_cold void ff_rtc_close(AVFormatContext *s) +{ + int i, ret; + RTCContext *rtc = s->priv_data; + + ret = dispose_session(s); + if (ret < 0) + av_log(rtc, AV_LOG_WARNING, "Failed to dispose resource, ret=%d\n", ret); + + for (i = 0; i < s->nb_streams; i++) { + AVFormatContext* rtp_ctx = s->streams[i]->priv_data; + if (!rtp_ctx) + continue; + + av_write_trailer(rtp_ctx); + /** + * Keep in mind that it is necessary to free the buffer of pb since we allocate + * it and pass it to pb using avio_alloc_context, while avio_context_free does + * not perform this action. + */ + av_freep(&rtp_ctx->pb->buffer); + avio_context_free(&rtp_ctx->pb); + avformat_free_context(rtp_ctx); + s->streams[i]->priv_data = NULL; + } + + av_freep(&rtc->sdp_offer); + av_freep(&rtc->sdp_answer); + av_freep(&rtc->rtc_resource_url); + av_freep(&rtc->ice_ufrag_remote); + av_freep(&rtc->ice_pwd_remote); + av_freep(&rtc->ice_protocol); + av_freep(&rtc->ice_host); + av_freep(&rtc->authorization); + av_freep(&rtc->cert_file); + av_freep(&rtc->key_file); + ff_srtp_free(&rtc->srtp_audio_send); + ff_srtp_free(&rtc->srtp_video_send); + ff_srtp_free(&rtc->srtp_video_rtx_send); + ff_srtp_free(&rtc->srtp_rtcp_send); + ff_srtp_free(&rtc->srtp_recv); + ffurl_close(rtc->dtls_uc); + ffurl_closep(&rtc->udp); + av_freep(&rtc->dtls_fingerprint); +} + +#define OFFSET(x) offsetof(RTCContext, x) +#define ENC AV_OPT_FLAG_ENCODING_PARAM +#define DEC AV_OPT_FLAG_DECODING_PARAM +#define DEP AV_OPT_FLAG_DEPRECATED +const AVOption ff_rtc_options[] = { + { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC | DEC }, + { "media_timeout", "Timeout in milliseconds for receiving media packets.", OFFSET(media_timeout), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, DEC }, + { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC | DEC }, + { "buffer_size", "The buffer size, in bytes, of underlying protocol", OFFSET(ts_buffer_size), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, ENC | DEC | DEP }, + { "ts_buffer_size", "The buffer size, in bytes, of underlying protocol", OFFSET(ts_buffer_size), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, ENC | DEC }, + { "reorder_queue_size", "Set number of packets to buffer for handling of reordered packets", OFFSET(reordering_queue_size), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, DEC }, + { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0}, 0, UINT_MAX, ENC | DEP, .unit = "flags" }, + { "rtc_flags", "Set flags affecting RTC connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0}, 0, UINT_MAX, ENC | DEC, .unit = "flags" }, + { "dtls_active", "Set dtls role as active", 0, AV_OPT_TYPE_CONST, { .i64 = RTC_DTLS_ACTIVE}, 0, UINT_MAX, ENC | DEC, .unit = "flags" }, + { "authorization", "The optional Bearer token for Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC | DEC }, + { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC | DEC }, + { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC | DEC }, + { NULL }, +}; diff --git a/libavformat/rtc.h b/libavformat/rtc.h new file mode 100644 index 0000000000..8abdc1ec99 --- /dev/null +++ b/libavformat/rtc.h @@ -0,0 +1,274 @@ +/* + * RTC definations + * Copyright (c) 2023 The FFmpeg Project + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVFORMAT_RTC_H +#define AVFORMAT_RTC_H + +#include "libavutil/opt.h" +#include "libavutil/lfg.h" + +#include "avformat.h" +#include "srtp.h" +#include "tls.h" +#include "url.h" + +/** + * Maximum size limit of a Session Description Protocol (SDP), + * be it an offer or answer. + */ +#define RTC_MAX_SDP_SIZE 8192 + +/** + * The maximum size of the Secure Real-time Transport Protocol (SRTP) HMAC checksum + * and padding that is appended to the end of the packet. To calculate the maximum + * size of the User Datagram Protocol (UDP) packet that can be sent out, subtract + * this size from the `pkt_size`. + */ +#define DTLS_SRTP_CHECKSUM_LEN 16 + +/** + * The size of the Secure Real-time Transport Protocol (SRTP) master key material + * that is exported by Secure Sockets Layer (SSL) after a successful Datagram + * Transport Layer Security (DTLS) handshake. This material consists of a key + * of 16 bytes and a salt of 14 bytes. + */ +#define DTLS_SRTP_KEY_LEN 16 +#define DTLS_SRTP_SALT_LEN 14 + +/** + * Maximum size of the buffer for sending and receiving UDP packets. + * Please note that this size does not limit the size of the UDP packet that can be sent. + * To set the limit for packet size, modify the `pkt_size` parameter. + * For instance, it is possible to set the UDP buffer to 4096 to send or receive packets, + * but please keep in mind that the `pkt_size` option limits the packet size to 1400. + */ +#define RTC_MAX_UDP_BUFFER_SIZE 4096 + +/* Referring to Chrome's definition of RTP payload types. */ +#define RTC_RTP_PAYLOAD_TYPE_H264 106 +#define RTC_RTP_PAYLOAD_TYPE_OPUS 111 +#define RTC_RTP_PAYLOAD_TYPE_VIDEO_RTX 105 + +#define RTC_US_PER_MS 1000 + +/** + * Refer to RFC 7675 5.1, + * + * To prevent expiry of consent, a STUN binding request can be sent periodically. + * Implementations SHOULD set a default interval of 5 seconds(5000ms). + * + * Consent expires after 30 seconds(30000ms). + */ +#define RTC_ICE_CONSENT_CHECK_INTERVAL 5000 +#define RTC_ICE_CONSENT_EXPIRED_TIMER 30000 + +/** + * In the case of ICE-LITE, these fields are not used; instead, they are defined + * as constant values. + */ +#define RTC_SDP_SESSION_ID "4489045141692799359" +#define RTC_SDP_CREATOR_IP "127.0.0.1" + +/* Calculate the elapsed time from starttime to endtime in milliseconds. */ +#define RTC_ELAPSED(starttime, endtime) ((float)(endtime - starttime) / 1000) + +typedef enum RTCFlags { + RTC_DTLS_ACTIVE = (1 << 0), +} RTCFlags; + +enum RTCState { + RTC_STATE_NONE, + + /* The initial state. */ + RTC_STATE_INIT, + /* The muxer has sent the offer to the peer. */ + RTC_STATE_OFFER, + /* The muxer has received the answer from the peer. */ + RTC_STATE_ANSWER, + /** + * After parsing the answer received from the peer, the muxer negotiates the abilities + * in the offer that it generated. + */ + RTC_STATE_NEGOTIATED, + /* The muxer has connected to the peer via UDP. */ + RTC_STATE_UDP_CONNECTED, + /* The muxer has sent the ICE request to the peer. */ + RTC_STATE_ICE_CONNECTING, + /* The muxer has received the ICE response from the peer. */ + RTC_STATE_ICE_CONNECTED, + /* The muxer has finished the DTLS handshake with the peer. */ + RTC_STATE_DTLS_FINISHED, + /* The muxer has finished the SRTP setup. */ + RTC_STATE_SRTP_FINISHED, + /* The muxer is ready to send/receive media frames. */ + RTC_STATE_READY, + /* The muxer is failed. */ + RTC_STATE_FAILED, +}; + +typedef struct RTCContext { + AVClass *av_class; + + uint32_t flags; + /* The state of the RTC connection. */ + enum RTCState state; + + /* Parameters for the input audio and video codecs. */ + AVCodecParameters *audio_par; + AVCodecParameters *video_par; + + /** + * The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet; + * therefore, it is essential to insert the SPS and PPS before each IDR frame + * in such cases. + */ + int h264_annexb_insert_sps_pps; + + /* The random number generator. */ + AVLFG rnd; + + /* The ICE username and pwd fragment generated by the muxer. */ + char ice_ufrag_local[9]; + char ice_pwd_local[33]; + /* The SSRC of the audio and video stream, generated by the muxer. */ + uint32_t audio_ssrc; + uint32_t video_ssrc; + uint32_t video_rtx_ssrc; + + uint16_t audio_first_seq; + uint16_t video_first_seq; + /* The PT(Payload Type) of stream, generated by the muxer. */ + uint8_t audio_payload_type; + uint8_t video_payload_type; + uint8_t video_rtx_payload_type; + /** + * This is the SDP offer generated by the muxer based on the codec parameters, + * DTLS, and ICE information. + */ + char *sdp_offer; + + int is_peer_ice_lite; + uint64_t ice_tie_breaker; // random 64 bit, for ICE-CONTROLLING + /* The ICE username and pwd from remote server. */ + char *ice_ufrag_remote; + char *ice_pwd_remote; + /** + * This represents the ICE candidate protocol, priority, host and port. + * Currently, we only support one candidate and choose the first UDP candidate. + * However, we plan to support multiple candidates in the future. + */ + char *ice_protocol; + char *ice_host; + int ice_port; + + /* The SDP answer received from the WebRTC server. */ + char *sdp_answer; + /* The resource URL returned in the Location header of WHIP/WHEP HTTP response. */ + char *rtc_resource_url; + + /* These variables represent timestamps used for calculating and tracking the cost. */ + int64_t rtc_starttime; + int64_t rtc_init_time; + int64_t rtc_offer_time; + int64_t rtc_answer_time; + int64_t rtc_udp_time; + int64_t rtc_ice_time; + int64_t rtc_dtls_time; + int64_t rtc_srtp_time; + int64_t rtc_last_consent_tx_time; + int64_t rtc_last_consent_rx_time; + int64_t rtc_last_media_rx_time; + + /* The certificate and private key content used for DTLS handshake */ + char cert_buf[MAX_CERTIFICATE_SIZE]; + char key_buf[MAX_CERTIFICATE_SIZE]; + /* The fingerprint of certificate, used in SDP offer. */ + char *dtls_fingerprint; + /** + * This represents the material used to build the SRTP master key. It is + * generated by DTLS and has the following layout: + * 16B 16B 14B 14B + * client_key | server_key | client_salt | server_salt + */ + uint8_t dtls_srtp_materials[(DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN) * 2]; + + /* TODO: Use AVIOContext instead of URLContext */ + URLContext *dtls_uc; + + /* The SRTP send context, to encrypt outgoing packets. */ + SRTPContext srtp_audio_send; + SRTPContext srtp_video_send; + SRTPContext srtp_video_rtx_send; + SRTPContext srtp_rtcp_send; + /* The SRTP receive context, to decrypt incoming packets. */ + SRTPContext srtp_recv; + + uint8_t srtp_recv_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN]; + + /* The UDP transport is used for delivering ICE, DTLS and SRTP packets. */ + URLContext *udp; + /* The buffer for UDP transmission. */ + char buf[RTC_MAX_UDP_BUFFER_SIZE]; + + /* The timeout in milliseconds for ICE and DTLS handshake. */ + int handshake_timeout; + /* The timeout in milliseconds for receiving media packets (RTP). */ + int media_timeout; + /** + * The size of RTP packet, should generally be set to MTU. + * Note that pion requires a smaller value, for example, 1200. + */ + int pkt_size; + int ts_buffer_size;/* Underlying protocol send/receive buffer size */ + + int reordering_queue_size; /* RTP reordering queue size */ + /** + * The optional Bearer token for WHIP/WHEP Authorization. + * See https://www.ietf.org/archive/id/draft-ietf-wish-whip-08.html#name-authentic… + */ + char* authorization; + /* The certificate and private key used for DTLS handshake. */ + char* cert_file; + char* key_file; +} RTCContext; + +int ff_rtc_initialize(AVFormatContext *s); + +int ff_rtc_connect(AVFormatContext *s); + +void ff_rtc_close(AVFormatContext *s); + +int ff_rtc_ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, int *request_size); + +int ff_rtc_ice_is_binding_response(uint8_t *b, int size); + +int ff_rtc_ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_size); + +int ff_rtc_is_dtls_packet(uint8_t *b, int size); + +int ff_rtc_media_is_rtcp(const uint8_t *b, int size); + +int ff_rtc_media_is_rtp_rtcp(const uint8_t *b, int size); + + +extern const AVOption ff_rtc_options[]; + +#endif /* AVFORMAT_RTC_H */ diff --git a/libavformat/whep.c b/libavformat/whep.c new file mode 100644 index 0000000000..ee253004a1 --- /dev/null +++ b/libavformat/whep.c @@ -0,0 +1,1119 @@ +/* + * WebRTC-HTTP egress protocol (WHEP) demuxer + * Copyright (c) 2026 Artem Smorodin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/avstring.h" +#include "libavutil/base64.h" +#include "libavutil/bprint.h" +#include "libavutil/channel_layout.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/mem.h" +#include "libavutil/time.h" + +#include "libavcodec/codec_desc.h" + +#include "avio_internal.h" +#include "internal.h" +#include "demux.h" +#include "network.h" +#include "rtp.h" +#include "rtpdec.h" +#include "rtc.h" + +/* If we try to read from UDP and get EAGAIN, we sleep for 5ms and retry up to 10 times. */ +#define WHEP_READ_MAX_RETRY 10 +#define WHEP_READ_SLEEP_DURATION 5 + +#define DEFAULT_REORDERING_DELAY 100000 +#define WHEP_RTP_REORDER_QUEUE_DEFAULT_SIZE 5000 + +#define WHEP_DEFAULT_AUDIO_SAMPLERATE 48000 + +typedef struct WHEPStream { + int stream_index; + int payload_type; + uint32_t ssrc; + uint32_t rtx_ssrc; + int rtx_payload_type; + int rtx_apt; + + RTPDemuxContext *rtp_ctx; + const RTPDynamicProtocolHandler *dynamic_handler; + PayloadContext *dynamic_protocol_context; + + char delayed_fmtp[2048]; + + SRTPContext srtp; + int srtp_inited; + + SRTPContext srtp_rtx; + int srtp_rtx_inited; +} WHEPStream; + +typedef struct WHEPContext { + RTCContext rtc; + WHEPStream **streams; + int nb_streams; + WHEPStream *cur_ws; +} WHEPContext; + + +static void get_word_until_chars(char *buf, int buf_size, + const char *sep, const char **pp) +{ + const char *p; + char *q; + + p = *pp; + p += strspn(p, SPACE_CHARS); + q = buf; + while (!strchr(sep, *p) && *p != '\0') { + if ((q - buf) < buf_size - 1) + *q++ = *p; + p++; + } + if (buf_size > 0) + *q = '\0'; + *pp = p; +} + +static void get_word_sep(char *buf, int buf_size, const char *sep, + const char **pp) +{ + if (**pp == '/') (*pp)++; + get_word_until_chars(buf, buf_size, sep, pp); +} + +static void get_word(char *buf, int buf_size, const char **pp) +{ + get_word_until_chars(buf, buf_size, SPACE_CHARS, pp); +} + +static WHEPStream *find_stream_by_payload_type(WHEPContext *whep, int payload_type) +{ + int i; + for (i = 0; i < whep->nb_streams; i++) { + if (whep->streams[i]->payload_type == payload_type) + return whep->streams[i]; + } + return NULL; +} + +static WHEPStream *find_stream_by_rtx_payload_type(WHEPContext *whep, int payload_type) +{ + int i; + for (i = 0; i < whep->nb_streams; i++) { + if (whep->streams[i]->rtx_payload_type == payload_type) + return whep->streams[i]; + } + return NULL; +} + +static WHEPStream *find_stream_by_ssrc(WHEPContext *whep, uint32_t ssrc) +{ + int i; + for (i = 0; i < whep->nb_streams; i++) { + if (whep->streams[i]->ssrc == ssrc) + return whep->streams[i]; + } + return NULL; +} + +static WHEPStream *find_stream_by_rtx_ssrc(WHEPContext *whep, uint32_t ssrc) +{ + int i; + for (i = 0; i < whep->nb_streams; i++) { + if (whep->streams[i]->rtx_ssrc == ssrc) + return whep->streams[i]; + } + return NULL; +} + +static void init_rtp_handler(const RTPDynamicProtocolHandler *handler, + WHEPStream *ws, AVStream *st) +{ + AVCodecParameters *par = st ? st->codecpar : NULL; + if (!handler) + return; + + if (par) + par->codec_id = handler->codec_id; + + ws->dynamic_handler = handler; + + if (st) + ffstream(st)->need_parsing = handler->need_parsing; + + if (handler->priv_data_size) { + ws->dynamic_protocol_context = av_mallocz(handler->priv_data_size); + if (!ws->dynamic_protocol_context) + ws->dynamic_handler = NULL; + } +} + +static void finalize_rtp_handler_init(AVFormatContext *s, WHEPStream *ws, + AVStream *st) +{ + if (ws->dynamic_handler && ws->dynamic_handler->init) { + int ret = ws->dynamic_handler->init(s, st ? st->index : -1, + ws->dynamic_protocol_context); + if (ret < 0) { + if (ws->dynamic_protocol_context) { + if (ws->dynamic_handler->close) + ws->dynamic_handler->close(ws->dynamic_protocol_context); + av_free(ws->dynamic_protocol_context); + } + ws->dynamic_protocol_context = NULL; + ws->dynamic_handler = NULL; + } + } +} + +static int parse_rtpmap(AVFormatContext *s, WHEPStream *ws, + int payload_type, const char *p) +{ + AVStream *st = s->streams[ws->stream_index]; + AVCodecParameters *par = st->codecpar; + char buf[256]; + int i; + const AVCodecDescriptor *desc; + const char *c_name; + + if (ws->payload_type != payload_type) { + ws->payload_type = payload_type; + if (ws->rtp_ctx) + ws->rtp_ctx->payload_type = payload_type; + } + + /* See if we can handle this kind of payload */ + get_word_sep(buf, sizeof(buf), "/ ", &p); + if (payload_type < RTP_PT_PRIVATE) + par->codec_id = ff_rtp_codec_id(buf, par->codec_type); + + if (par->codec_id == AV_CODEC_ID_NONE) { + const RTPDynamicProtocolHandler *handler = + ff_rtp_handler_find_by_name(buf, par->codec_type); + init_rtp_handler(handler, ws, st); + if (!ws->dynamic_handler) + par->codec_id = ff_rtp_codec_id(buf, par->codec_type); + } + + desc = avcodec_descriptor_get(par->codec_id); + c_name = desc && desc->name ? desc->name : "(null)"; + + get_word_sep(buf, sizeof(buf), "/", &p); + i = atoi(buf); + switch (par->codec_type) { + case AVMEDIA_TYPE_AUDIO: + av_log(s, AV_LOG_DEBUG, "audio codec set to: %s\n", c_name); + par->sample_rate = WHEP_DEFAULT_AUDIO_SAMPLERATE; + par->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_MONO; + if (i > 0) { + par->sample_rate = i; + avpriv_set_pts_info(st, 32, 1, par->sample_rate); + get_word_sep(buf, sizeof(buf), "/", &p); + i = atoi(buf); + if (i > 0) + av_channel_layout_default(&par->ch_layout, i); + } + av_log(s, AV_LOG_DEBUG, "audio samplerate set to: %i\n", par->sample_rate); + av_log(s, AV_LOG_DEBUG, "audio channels set to: %i\n", par->ch_layout.nb_channels); + break; + case AVMEDIA_TYPE_VIDEO: + av_log(s, AV_LOG_DEBUG, "video codec set to: %s\n", c_name); + if (i > 0) + avpriv_set_pts_info(st, 32, 1, i); + break; + default: + break; + } + + finalize_rtp_handler_init(s, ws, st); + if (ws->dynamic_handler) + ff_rtp_parse_set_dynamic_protocol(ws->rtp_ctx, + ws->dynamic_protocol_context, + ws->dynamic_handler); + + if (ws->delayed_fmtp[0] && ws->dynamic_handler && + ws->dynamic_handler->parse_sdp_a_line) { + ws->dynamic_handler->parse_sdp_a_line(s, st->index, + ws->dynamic_protocol_context, + ws->delayed_fmtp); + ws->delayed_fmtp[0] = '\0'; + } + + return 0; +} + +static int add_stream(AVFormatContext *s, enum AVMediaType codec_type, + int payload_type, WHEPStream **out) +{ + int ret = 0; + int queue_size; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + WHEPStream *ws = NULL; + AVStream *st = NULL; + + if (s->nb_streams >= s->max_streams) { + av_log(s, AV_LOG_ERROR, "Too many streams, max=%d\n", s->max_streams); + return AVERROR(ENOSYS); + } + + st = avformat_new_stream(s, NULL); + if (!st) + return AVERROR(ENOMEM); + + st->id = whep->nb_streams; + st->codecpar->codec_type = codec_type; + + ws = av_mallocz(sizeof(*ws)); + if (!ws) { + ret = AVERROR(ENOMEM); + goto end; + } + ws->payload_type = payload_type; + ws->rtx_payload_type = -1; + ws->rtx_apt = -1; + ws->stream_index = st->index; + + queue_size = s->max_delay ? rtc->reordering_queue_size : 0; + ws->rtp_ctx = ff_rtp_parse_open(s, st, payload_type, queue_size); + if (!ws->rtp_ctx) { + ret = AVERROR(ENOMEM); + goto end; + } + + dynarray_add(&whep->streams, &whep->nb_streams, ws); + if (out) + *out = ws; + ws = NULL; + +end: + if (ws) { + ff_rtp_parse_close(ws->rtp_ctx); + av_free(ws); + } + return ret; +} + +static int parse_sdp_answer(AVFormatContext *s) +{ + int ret = 0, i; + AVIOContext *pb = NULL; + char line[MAX_URL_SIZE]; + const char *p; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + WHEPStream *cur = NULL; + + if (!rtc->sdp_answer || !strlen(rtc->sdp_answer)) { + av_log(rtc, AV_LOG_ERROR, "No answer to parse\n"); + return AVERROR(EINVAL); + } + + pb = avio_alloc_context(rtc->sdp_answer, strlen(rtc->sdp_answer), 0, + NULL, NULL, NULL, NULL); + if (!pb) + return AVERROR(ENOMEM); + + while (!avio_feof(pb)) { + ff_get_chomp_line(pb, line, sizeof(line)); + if (!line[0]) + continue; + + if (av_strstart(line, "m=", &p)) { + char media[16], port[16], proto[64], fmt[16]; + enum AVMediaType codec_type = AVMEDIA_TYPE_UNKNOWN; + int payload_type = -1; + int expected_payload_type = -1; + + get_word(media, sizeof(media), &p); + get_word(port, sizeof(port), &p); + get_word(proto, sizeof(proto), &p); + if (!media[0]) + continue; + + if (!strcmp(media, "audio")) { + codec_type = AVMEDIA_TYPE_AUDIO; + expected_payload_type = rtc->audio_payload_type; + } else if (!strcmp(media, "video")) { + codec_type = AVMEDIA_TYPE_VIDEO; + expected_payload_type = rtc->video_payload_type; + } else { + cur = NULL; + + continue; + } + + while (1) { + int pt; + get_word(fmt, sizeof(fmt), &p); + if (!fmt[0]) + break; + pt = atoi(fmt); + if (payload_type < 0) + payload_type = pt; + if (expected_payload_type >= 0 && pt == expected_payload_type) + payload_type = pt; + } + if (payload_type < 0) + continue; + + ret = add_stream(s, codec_type, payload_type, &cur); + if (ret < 0) + goto end; + continue; + } + + if (av_strstart(line, "a=rtpmap:", &p)) { + char pt_buf[16]; + char codec_name[32]; + int payload_type; + WHEPStream *ws; + + get_word(pt_buf, sizeof(pt_buf), &p); + payload_type = atoi(pt_buf); + p += strspn(p, SPACE_CHARS); + { + const char *p2 = p; + get_word_sep(codec_name, sizeof(codec_name), "/ ", &p2); + if (!av_strcasecmp(codec_name, "rtx")) + continue; + } + ws = find_stream_by_payload_type(whep, payload_type); + if (!ws) + continue; + ret = parse_rtpmap(s, ws, payload_type, p); + if (ret < 0) + goto end; + continue; + } + + if (av_strstart(line, "a=fmtp:", &p)) { + char pt_buf[16]; + int payload_type; + int apt_payload_type; + WHEPStream *ws; + const char *fmtp_attr = line + 2; + const char *apt; + + get_word(pt_buf, sizeof(pt_buf), &p); + payload_type = atoi(pt_buf); + apt = strstr(fmtp_attr, "apt="); + if (apt) { + apt_payload_type = atoi(apt + 4); + ws = find_stream_by_payload_type(whep, apt_payload_type); + if (!ws && cur && cur->payload_type == apt_payload_type) + ws = cur; + if (ws) { + ws->rtx_payload_type = payload_type; + ws->rtx_apt = apt_payload_type; + } + continue; + } + + ws = find_stream_by_payload_type(whep, payload_type); + if (!ws) + continue; + if (ws->dynamic_handler && ws->dynamic_handler->parse_sdp_a_line) { + ws->dynamic_handler->parse_sdp_a_line(s, ws->stream_index, + ws->dynamic_protocol_context, fmtp_attr); + } else if (!ws->delayed_fmtp[0]) { + av_strlcpy(ws->delayed_fmtp, fmtp_attr, sizeof(ws->delayed_fmtp)); + } + continue; + } + + if (av_strstart(line, "a=ssrc-group:", &p)) { + char semantics[16], ssrc1_buf[32], ssrc2_buf[32]; + uint32_t ssrc1, ssrc2; + + get_word(semantics, sizeof(semantics), &p); + if (av_strcasecmp(semantics, "FID")) + continue; + + get_word(ssrc1_buf, sizeof(ssrc1_buf), &p); + get_word(ssrc2_buf, sizeof(ssrc2_buf), &p); + if (!ssrc1_buf[0] || !ssrc2_buf[0]) + continue; + + ssrc1 = strtoul(ssrc1_buf, NULL, 10); + ssrc2 = strtoul(ssrc2_buf, NULL, 10); + + if (cur) { + if (!cur->ssrc || cur->ssrc == ssrc2) + cur->ssrc = ssrc1; + cur->rtx_ssrc = ssrc2; + } + continue; + } + + if (av_strstart(line, "a=ssrc:", &p)) { + uint32_t ssrc = strtoul(p, NULL, 10); + if (cur && !cur->ssrc) + cur->ssrc = ssrc; + continue; + } + + if (av_strstart(line, "a=", &p)) { + if (cur && cur->dynamic_handler && cur->dynamic_handler->parse_sdp_a_line) { + cur->dynamic_handler->parse_sdp_a_line(s, cur->stream_index, + cur->dynamic_protocol_context, p); + } + } + + if (av_strstart(line, "s=", &p)) { + av_dict_set(&s->metadata, "title", p, 0); + } + } + + for (i = 0; i < whep->nb_streams; i++) { + WHEPStream *ws = whep->streams[i]; + if (!ws) + continue; + if (ws->rtx_payload_type >= 0 || ws->rtx_ssrc) { + av_log(rtc, AV_LOG_VERBOSE, + "RTX negotiated stream=%d pt=%d ssrc=%u rtx_pt=%d rtx_ssrc=%u\n", + ws->stream_index, ws->payload_type, ws->ssrc, + ws->rtx_payload_type, ws->rtx_ssrc); + } + } + + if (!whep->nb_streams) { + av_log(rtc, AV_LOG_ERROR, "No media streams parsed from answer\n"); + ret = AVERROR(EINVAL); + goto end; + } + +end: + avio_context_free(&pb); + return ret; +} + +static int init_srtp_streams(AVFormatContext *s) +{ + int i, ret; + char keybuf[AV_BASE64_SIZE(DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN)]; + const char *suite = "SRTP_AES128_CM_HMAC_SHA1_80"; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + + if (!av_base64_encode(keybuf, sizeof(keybuf), + rtc->srtp_recv_key, sizeof(rtc->srtp_recv_key))) { + av_log(rtc, AV_LOG_ERROR, "Failed to encode recv key\n"); + return AVERROR(EIO); + } + + for (i = 0; i < whep->nb_streams; i++) { + WHEPStream *ws = whep->streams[i]; + if (!ws) + continue; + ret = ff_srtp_set_crypto(&ws->srtp, suite, keybuf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for stream %d\n", i); + return ret; + } + ws->srtp_inited = 1; + + ret = ff_srtp_set_crypto(&ws->srtp_rtx, suite, keybuf); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to set crypto for rtx stream %d\n", i); + return ret; + } + ws->srtp_rtx_inited = 1; + } + + return 0; +} + +static int generate_sdp_offer(AVFormatContext *s) +{ + int ret = 0; + AVBPrint bp; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + int is_dtls_active = rtc->flags & RTC_DTLS_ACTIVE; + + /* To prevent a crash during cleanup, always initialize it. */ + av_bprint_init(&bp, 1, RTC_MAX_SDP_SIZE); + + if (rtc->sdp_offer) { + av_log(rtc, AV_LOG_ERROR, "SDP offer is already set\n"); + ret = AVERROR(EINVAL); + goto end; + } + + snprintf(rtc->ice_ufrag_local, sizeof(rtc->ice_ufrag_local), "%08x", + av_lfg_get(&rtc->rnd)); + snprintf(rtc->ice_pwd_local, sizeof(rtc->ice_pwd_local), "%08x%08x%08x%08x", + av_lfg_get(&rtc->rnd), av_lfg_get(&rtc->rnd), av_lfg_get(&rtc->rnd), + av_lfg_get(&rtc->rnd)); + + rtc->audio_payload_type = RTC_RTP_PAYLOAD_TYPE_OPUS; + rtc->video_payload_type = RTC_RTP_PAYLOAD_TYPE_H264; + rtc->video_rtx_payload_type = RTC_RTP_PAYLOAD_TYPE_VIDEO_RTX; + + av_bprintf(&bp, "" + "v=0\r\n" + "o=FFmpeg %s 2 IN IP4 %s\r\n" + "s=FFmpegPlaySession\r\n" + "t=0 0\r\n" + "a=group:BUNDLE 0 1\r\n" + "a=extmap-allow-mixed\r\n" + "a=msid-semantic: WMS\r\n", + RTC_SDP_SESSION_ID, + RTC_SDP_CREATOR_IP); + + av_bprintf(&bp, "" + "m=audio 9 UDP/TLS/RTP/SAVPF %u\r\n" + "c=IN IP4 0.0.0.0\r\n" + "a=ice-ufrag:%s\r\n" + "a=ice-pwd:%s\r\n" + "a=fingerprint:sha-256 %s\r\n" + "a=setup:%s\r\n" + "a=mid:0\r\n" + "a=recvonly\r\n" + "a=rtcp-mux\r\n" + "a=rtpmap:%u opus/48000/2\r\n", + rtc->audio_payload_type, + rtc->ice_ufrag_local, + rtc->ice_pwd_local, + rtc->dtls_fingerprint, + is_dtls_active ? "active" : "passive", + rtc->audio_payload_type); + + av_bprintf(&bp, "" + "m=video 9 UDP/TLS/RTP/SAVPF %u %u\r\n" + "c=IN IP4 0.0.0.0\r\n" + "a=ice-ufrag:%s\r\n" + "a=ice-pwd:%s\r\n" + "a=fingerprint:sha-256 %s\r\n" + "a=setup:%s\r\n" + "a=mid:1\r\n" + "a=recvonly\r\n" + "a=rtcp-mux\r\n" + "a=rtcp-rsize\r\n" + "a=rtpmap:%u H264/90000\r\n" + "a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\n" + "a=rtcp-fb:%u nack\r\n" + "a=rtcp-fb:%u nack pli\r\n" + "a=rtpmap:%u rtx/90000\r\n" + "a=fmtp:%u apt=%u\r\n", + rtc->video_payload_type, + rtc->video_rtx_payload_type, + rtc->ice_ufrag_local, + rtc->ice_pwd_local, + rtc->dtls_fingerprint, + is_dtls_active ? "active" : "passive", + rtc->video_payload_type, + rtc->video_payload_type, + rtc->video_payload_type, + rtc->video_payload_type, + rtc->video_rtx_payload_type, + rtc->video_rtx_payload_type, + rtc->video_payload_type); + + if (!av_bprint_is_complete(&bp)) { + av_log(rtc, AV_LOG_ERROR, "Offer exceed max %d, %s\n", RTC_MAX_SDP_SIZE, bp.str); + ret = AVERROR(EIO); + goto end; + } + + rtc->sdp_offer = av_strdup(bp.str); + if (!rtc->sdp_offer) { + ret = AVERROR(ENOMEM); + goto end; + } + + if (rtc->state < RTC_STATE_OFFER) + rtc->state = RTC_STATE_OFFER; + rtc->rtc_offer_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "Generated state=%d, offer: %s\n", + rtc->state, rtc->sdp_offer); + +end: + av_bprint_finalize(&bp, NULL); + return ret; +} + +static int send_rtcp_rr(AVFormatContext *s, WHEPStream *ws, int count) +{ + int ret; + int len; + int cipher_len; + uint8_t *buf = NULL; + AVIOContext *pb = NULL; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + + if ((ret = avio_open_dyn_buf(&pb)) < 0) + return ret; + + ret = ff_rtp_check_and_send_back_rr(ws->rtp_ctx, NULL, pb, count); + len = avio_close_dyn_buf(pb, &buf); + if (ret < 0 || len <= 0 || !buf) { + av_free(buf); + return 0; + } + + cipher_len = ff_srtp_encrypt(&rtc->srtp_rtcp_send, buf, len, rtc->buf, sizeof(rtc->buf)); + if (cipher_len <= 0 || cipher_len < len) { + av_log(rtc, AV_LOG_WARNING, "Failed to encrypt rtcp packet=%dB, cipher=%dB\n", + len, cipher_len); + av_free(buf); + return 0; + } + + av_log(rtc, AV_LOG_TRACE, "Sending SRTCP RR stream=%d plain=%dB cipher=%dB\n", + ws->stream_index, len, cipher_len); + ret = ffurl_write(rtc->udp, rtc->buf, cipher_len); + if (ret < 0) + av_log(rtc, AV_LOG_ERROR, "Failed to send rtcp packet=%dB, ret=%d\n", cipher_len, ret); + + av_free(buf); + return ret < 0 ? ret : 0; +} + +static int send_rtcp_feedback(AVFormatContext *s, WHEPStream *ws) +{ + int ret; + int len; + int cipher_len; + uint8_t *buf = NULL; + AVIOContext *pb = NULL; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + + if ((ret = avio_open_dyn_buf(&pb)) < 0) + return ret; + + ret = ff_rtp_send_rtcp_feedback(ws->rtp_ctx, NULL, pb); + len = avio_close_dyn_buf(pb, &buf); + if (ret < 0 || len <= 0 || !buf) { + av_free(buf); + return 0; + } + + cipher_len = ff_srtp_encrypt(&rtc->srtp_rtcp_send, buf, len, + rtc->buf, sizeof(rtc->buf)); + if (cipher_len <= 0 || cipher_len < len) { + av_log(rtc, AV_LOG_WARNING, + "Failed to encrypt rtcp feedback packet=%dB, cipher=%dB\n", + len, cipher_len); + av_free(buf); + return 0; + } + + av_log(rtc, AV_LOG_TRACE, + "Sending SRTCP feedback stream=%d plain=%dB cipher=%dB\n", + ws->stream_index, len, cipher_len); + ret = ffurl_write(rtc->udp, rtc->buf, cipher_len); + if (ret < 0) + av_log(rtc, AV_LOG_ERROR, + "Failed to send rtcp feedback packet=%dB, ret=%d\n", + cipher_len, ret); + + av_free(buf); + return ret < 0 ? ret : 0; +} + +static av_cold int whep_read_header(AVFormatContext *s) +{ + int ret; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + + if (s->max_delay < 0) /* Not set by the caller */ + s->max_delay = DEFAULT_REORDERING_DELAY; + + if (rtc->reordering_queue_size <= 0) + rtc->reordering_queue_size = WHEP_RTP_REORDER_QUEUE_DEFAULT_SIZE; + + if ((ret = ff_rtc_initialize(s)) < 0) + goto end; + + if ((ret = generate_sdp_offer(s)) < 0) + goto end; + + if ((ret = ff_rtc_connect(s)) < 0) + goto end; + + if ((ret = parse_sdp_answer(s)) < 0) + goto end; + + if ((ret = init_srtp_streams(s)) < 0) + goto end; + + if (rtc->state < RTC_STATE_READY) + rtc->state = RTC_STATE_READY; + + rtc->rtc_last_media_rx_time = av_gettime_relative(); + + av_log(rtc, AV_LOG_INFO, "Demuxer state=%d, elapsed=%.2fms(init:%.2f,offer:%.2f,answer:%.2f,udp:%.2f,ice:%.2f,dtls:%.2f,srtp:%.2f)\n", + rtc->state, RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative()), + RTC_ELAPSED(rtc->rtc_starttime, rtc->rtc_init_time), + RTC_ELAPSED(rtc->rtc_init_time, rtc->rtc_offer_time), + RTC_ELAPSED(rtc->rtc_offer_time, rtc->rtc_answer_time), + RTC_ELAPSED(rtc->rtc_answer_time, rtc->rtc_udp_time), + RTC_ELAPSED(rtc->rtc_udp_time, rtc->rtc_ice_time), + RTC_ELAPSED(rtc->rtc_ice_time, rtc->rtc_dtls_time), + RTC_ELAPSED(rtc->rtc_dtls_time, rtc->rtc_srtp_time)); + +end: + if (ret < 0) + rtc->state = RTC_STATE_FAILED; + return ret; +} + +static int get_rtp_payload_offset(const uint8_t *buf, int len, int *offset) +{ + int csrc, ext; + int header_len; + + if (len < 12) + return AVERROR_INVALIDDATA; + if ((buf[0] >> 6) != RTP_VERSION) + return AVERROR_INVALIDDATA; + + csrc = buf[0] & 0x0f; + ext = buf[0] & 0x10; + + header_len = 12 + 4 * csrc; + if (len < header_len) + return AVERROR_INVALIDDATA; + + if (ext) { + int ext_len; + + if (len < header_len + 4) + return AVERROR_INVALIDDATA; + /* See RFC 3550 Section 5.3.1 RTP Header Extension handling */ + ext_len = (AV_RB16(buf + header_len + 2) + 1) * 4; + if (len < header_len + ext_len) + return AVERROR_INVALIDDATA; + header_len += ext_len; + } + + *offset = header_len; + return 0; +} + +static int whep_unwrap_rtx_packet(AVFormatContext *s, WHEPStream *ws, + uint8_t *buf, int *size) +{ + int ret, payload_offset; + uint16_t osn; + uint32_t media_ssrc; + int apt = ws->rtx_apt >= 0 ? ws->rtx_apt : ws->payload_type; + + ret = get_rtp_payload_offset(buf, *size, &payload_offset); + if (ret < 0) + return ret; + + if (*size - payload_offset < 2) + return AVERROR_INVALIDDATA; + + osn = AV_RB16(buf + payload_offset); + + memmove(buf + payload_offset, buf + payload_offset + 2, *size - payload_offset - 2); + *size -= 2; + + AV_WB16(buf + 2, osn); + buf[1] = (buf[1] & 0x80) | (apt & 0x7f); + + media_ssrc = ws->ssrc; + if (!media_ssrc && ws->rtp_ctx && ws->rtp_ctx->ssrc) + media_ssrc = ws->ssrc = ws->rtp_ctx->ssrc; + if (!media_ssrc) { + av_log(s, AV_LOG_DEBUG, "Cannot unwrap RTX before media SSRC is known\n"); + return AVERROR(EAGAIN); + } + + AV_WB32(buf + 8, media_ssrc); + return 0; +} + +static int whep_read_packet(AVFormatContext *s, AVPacket *pkt) +{ + int ret, i; + int64_t now; + int64_t first_queue_time; + int size; + uint8_t *buf; + WHEPContext *whep = s->priv_data; + RTCContext *rtc = &whep->rtc; + WHEPStream *ws; + WHEPStream *first_queue_ws; + uint32_t ssrc; + int payload_type; + int is_rtcp; + int is_rtx; + SRTPContext *srtp_ctx; + int64_t wait_end; + + while (1) { + if (ff_check_interrupt(&s->interrupt_callback)) + return AVERROR_EXIT; + + ws = whep->cur_ws; + if (ws) { + ret = ff_rtp_parse_packet(ws->rtp_ctx, pkt, NULL, 0); + if (ret >= 0) { + if (!ws->ssrc && ws->rtp_ctx->ssrc) + ws->ssrc = ws->rtp_ctx->ssrc; + send_rtcp_feedback(s, ws); + if (ret == 0) + whep->cur_ws = NULL; + return 0; + } + whep->cur_ws = NULL; + } + + now = av_gettime_relative(); + if (rtc->media_timeout > 0 && rtc->rtc_last_media_rx_time && + now - rtc->rtc_last_media_rx_time > rtc->media_timeout * RTC_US_PER_MS) { + av_log(rtc, AV_LOG_ERROR, + "Media timeout after %.2fms (limited %dms), terminate session\n", + RTC_ELAPSED(rtc->rtc_last_media_rx_time, now), rtc->media_timeout); + rtc->state = RTC_STATE_FAILED; + return AVERROR(ETIMEDOUT); + } + first_queue_time = 0; + first_queue_ws = NULL; + if (s->max_delay > 0) { + for (i = 0; i < whep->nb_streams; i++) { + int64_t queue_time = ff_rtp_queued_packet_time(whep->streams[i]->rtp_ctx); + if (queue_time && (!first_queue_time || queue_time < first_queue_time)) { + first_queue_time = queue_time; + first_queue_ws = whep->streams[i]; + } + } + } + wait_end = first_queue_time ? first_queue_time + s->max_delay : 0; + if (wait_end && now > wait_end && first_queue_ws) { + ret = ff_rtp_parse_packet(first_queue_ws->rtp_ctx, pkt, NULL, 0); + if (ret >= 0) { + if (!first_queue_ws->ssrc && first_queue_ws->rtp_ctx->ssrc) + first_queue_ws->ssrc = first_queue_ws->rtp_ctx->ssrc; + send_rtcp_feedback(s, first_queue_ws); + whep->cur_ws = ret ? first_queue_ws : NULL; + return 0; + } + } + if (now - rtc->rtc_last_consent_rx_time > RTC_ICE_CONSENT_EXPIRED_TIMER * RTC_US_PER_MS) { + av_log(rtc, AV_LOG_ERROR, + "Consent Freshness expired after %.2fms (limited %dms), terminate session\n", + RTC_ELAPSED(rtc->rtc_last_consent_rx_time, now), RTC_ICE_CONSENT_EXPIRED_TIMER); + rtc->state = RTC_STATE_FAILED; + return AVERROR(ETIMEDOUT); + } + if (now - rtc->rtc_last_consent_tx_time > RTC_ICE_CONSENT_CHECK_INTERVAL * RTC_US_PER_MS) { + ret = ff_rtc_ice_create_request(s, rtc->buf, sizeof(rtc->buf), &size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); + return ret; + } + ret = ffurl_write(rtc->udp, rtc->buf, size); + if (ret < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); + return ret; + } + rtc->rtc_last_consent_tx_time = now; + av_log(rtc, AV_LOG_DEBUG, "Consent Freshness check sent\n"); + } + + for (i = 0; i < WHEP_READ_MAX_RETRY; i++) { + ret = ffurl_read(rtc->udp, rtc->buf, sizeof(rtc->buf)); + if (ret > 0) + break; + if (ret == AVERROR(EAGAIN)) { + if (s->flags & AVFMT_FLAG_NONBLOCK) + return AVERROR(EAGAIN); + if (wait_end) { + now = av_gettime_relative(); + if (now > wait_end && first_queue_ws) { + ret = ff_rtp_parse_packet(first_queue_ws->rtp_ctx, pkt, NULL, 0); + if (ret >= 0) { + if (!first_queue_ws->ssrc && first_queue_ws->rtp_ctx->ssrc) + first_queue_ws->ssrc = first_queue_ws->rtp_ctx->ssrc; + send_rtcp_feedback(s, first_queue_ws); + whep->cur_ws = ret ? first_queue_ws : NULL; + return 0; + } + } + if (now < wait_end) { + int64_t sleep_us = FFMIN((int64_t)WHEP_READ_SLEEP_DURATION * RTC_US_PER_MS, + wait_end - now); + av_usleep(sleep_us); + continue; + } + } + av_usleep(WHEP_READ_SLEEP_DURATION * RTC_US_PER_MS); + continue; + } + if (ret < 0) + return ret; + if (!ret) + return AVERROR_EOF; + } + if (ret <= 0) + continue; + + if (ff_rtc_ice_is_binding_response(rtc->buf, ret)) { + rtc->rtc_last_consent_rx_time = av_gettime_relative(); + av_log(rtc, AV_LOG_DEBUG, "Consent Freshness check received\n"); + continue; + } + + if (ff_rtc_is_dtls_packet(rtc->buf, ret)) { + if ((ret = ffurl_write(rtc->dtls_uc, rtc->buf, ret)) < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to handle DTLS message\n"); + return ret; + } + continue; + } + + if (!ff_rtc_media_is_rtp_rtcp(rtc->buf, ret)) { + ff_rtc_ice_handle_binding_request(s, rtc->buf, ret); + continue; + } + + is_rtcp = ff_rtc_media_is_rtcp(rtc->buf, ret); + if (is_rtcp) { + ssrc = ret >= 8 ? AV_RB32(&rtc->buf[4]) : 0; + ws = find_stream_by_ssrc(whep, ssrc); + if (!ws) + ws = find_stream_by_rtx_ssrc(whep, ssrc); + } else { + payload_type = rtc->buf[1] & 0x7f; + ssrc = ret >= 12 ? AV_RB32(&rtc->buf[8]) : 0; + ws = find_stream_by_ssrc(whep, ssrc); + if (!ws) + ws = find_stream_by_rtx_ssrc(whep, ssrc); + if (!ws) + ws = find_stream_by_payload_type(whep, payload_type); + if (!ws) + ws = find_stream_by_rtx_payload_type(whep, payload_type); + } + if (!ws && whep->nb_streams == 1) + ws = whep->streams[0]; + if (!ws) + continue; + if (!is_rtcp) + rtc->rtc_last_media_rx_time = av_gettime_relative(); + + is_rtx = !is_rtcp && + ((ws->rtx_ssrc && ssrc == ws->rtx_ssrc) || + (ws->rtx_payload_type >= 0 && payload_type == ws->rtx_payload_type)); + srtp_ctx = is_rtx ? &ws->srtp_rtx : &ws->srtp; + + if (!ws->srtp_inited || (is_rtx && !ws->srtp_rtx_inited)) + continue; + + size = ret; + if ((ret = ff_srtp_decrypt(srtp_ctx, rtc->buf, &size)) < 0) { + av_log(rtc, AV_LOG_WARNING, "Failed to decrypt srtp packet, ret=%d\n", ret); + continue; + } + + if (ff_rtc_media_is_rtcp(rtc->buf, size)) { + uint8_t *pkt_buf = av_memdup(rtc->buf, size); + if (!pkt_buf) + return AVERROR(ENOMEM); + buf = pkt_buf; + ff_rtp_parse_packet(ws->rtp_ctx, pkt, &buf, size); + if (buf) + av_freep(&pkt_buf); + continue; + } + + if (size > 0) { + int pkt_size = size; + uint8_t *pkt_buf = av_memdup(rtc->buf, pkt_size); + if (!pkt_buf) + return AVERROR(ENOMEM); + buf = pkt_buf; + + if (is_rtx) { + ret = whep_unwrap_rtx_packet(s, ws, buf, &pkt_size); + if (ret < 0) { + av_free(pkt_buf); + continue; + } + } + + ret = ff_rtp_parse_packet(ws->rtp_ctx, pkt, &buf, pkt_size); + if (buf) + av_freep(&pkt_buf); + send_rtcp_feedback(s, ws); + send_rtcp_rr(s, ws, pkt_size); + if (ret >= 0) { + if (!ws->ssrc && ws->rtp_ctx->ssrc) + ws->ssrc = ws->rtp_ctx->ssrc; + whep->cur_ws = ret ? ws : NULL; + return 0; + } + } + } +} + +static av_cold int whep_read_close(AVFormatContext *s) +{ + int i; + WHEPContext *whep = s->priv_data; + + for (i = 0; i < whep->nb_streams; i++) { + WHEPStream *ws = whep->streams[i]; + if (!ws) + continue; + if (ws->dynamic_handler && ws->dynamic_protocol_context) { + if (ws->dynamic_handler->close) + ws->dynamic_handler->close(ws->dynamic_protocol_context); + av_free(ws->dynamic_protocol_context); + } + ff_srtp_free(&ws->srtp); + ff_srtp_free(&ws->srtp_rtx); + ff_rtp_parse_close(ws->rtp_ctx); + av_free(ws); + } + av_freep(&whep->streams); + + ff_rtc_close(s); + return 0; +} + +static const AVClass whep_demuxer_class = { + .class_name = "WHEP demuxer", + .item_name = av_default_item_name, + .option = ff_rtc_options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const FFInputFormat ff_whep_demuxer = { + .p.name = "whep", + .p.long_name = NULL_IF_CONFIG_SMALL("WHEP(WebRTC-HTTP egress protocol) demuxer"), + .p.flags = AVFMT_NOFILE | AVFMT_EXPERIMENTAL, + .p.priv_class = &whep_demuxer_class, + .priv_data_size = sizeof(WHEPContext), + .read_header = whep_read_header, + .read_packet = whep_read_packet, + .read_close = whep_read_close, +}; diff --git a/libavformat/whip.c b/libavformat/whip.c index 8aed0c31e5..a0fc07d5cd 100644 --- a/libavformat/whip.c +++ b/libavformat/whip.c @@ -21,443 +21,19 @@ #include "libavcodec/h264.h" #include "libavcodec/startcode.h" -#include "libavutil/avassert.h" -#include "libavutil/base64.h" #include "libavutil/bprint.h" -#include "libavutil/crc.h" -#include "libavutil/hmac.h" -#include "libavutil/intreadwrite.h" -#include "libavutil/lfg.h" -#include "libavutil/opt.h" +#include "libavutil/avassert.h" #include "libavutil/mem.h" -#include "libavutil/random_seed.h" #include "libavutil/time.h" +#include "libavutil/intreadwrite.h" + +#include "internal.h" +#include "avio_internal.h" #include "avc.h" #include "nal.h" -#include "avio_internal.h" -#include "http.h" -#include "internal.h" +#include "rtc.h" #include "mux.h" -#include "network.h" #include "rtp.h" -#include "srtp.h" -#include "tls.h" - -/** - * Maximum size limit of a Session Description Protocol (SDP), - * be it an offer or answer. - */ -#define MAX_SDP_SIZE 8192 - -/** - * The size of the Secure Real-time Transport Protocol (SRTP) master key material - * that is exported by Secure Sockets Layer (SSL) after a successful Datagram - * Transport Layer Security (DTLS) handshake. This material consists of a key - * of 16 bytes and a salt of 14 bytes. - */ -#define DTLS_SRTP_KEY_LEN 16 -#define DTLS_SRTP_SALT_LEN 14 - -/** - * The maximum size of the Secure Real-time Transport Protocol (SRTP) HMAC checksum - * and padding that is appended to the end of the packet. To calculate the maximum - * size of the User Datagram Protocol (UDP) packet that can be sent out, subtract - * this size from the `pkt_size`. - */ -#define DTLS_SRTP_CHECKSUM_LEN 16 - -#define WHIP_US_PER_MS 1000 - -/** - * If we try to read from UDP and get EAGAIN, we sleep for 5ms and retry up to 10 times. - * This will limit the total duration (in milliseconds, 50ms) - */ -#define ICE_DTLS_READ_MAX_RETRY 10 -#define ICE_DTLS_READ_SLEEP_DURATION 5 - -/* The magic cookie for Session Traversal Utilities for NAT (STUN) messages. */ -#define STUN_MAGIC_COOKIE 0x2112A442 - -/** - * Refer to RFC 8445 5.1.2 - * priority = (2^24)*(type preference) + (2^8)*(local preference) + (2^0)*(256 - component ID) - * host candidate priority is 126 << 24 | 65535 << 8 | 255 - */ -#define STUN_HOST_CANDIDATE_PRIORITY 126 << 24 | 65535 << 8 | 255 - -/** - * The DTLS content type. - * See https://tools.ietf.org/html/rfc2246#section-6.2.1 - * change_cipher_spec(20), alert(21), handshake(22), application_data(23) - */ -#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20 - -/** - * The DTLS record layer header has a total size of 13 bytes, consisting of - * ContentType (1 byte), ProtocolVersion (2 bytes), Epoch (2 bytes), - * SequenceNumber (6 bytes), and Length (2 bytes). - * See https://datatracker.ietf.org/doc/html/rfc9147#section-4 - */ -#define DTLS_RECORD_LAYER_HEADER_LEN 13 - -/** - * The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2. - * See https://datatracker.ietf.org/doc/html/rfc9147#name-the-dtls-record-layer - */ -#define DTLS_VERSION_10 0xfeff -#define DTLS_VERSION_12 0xfefd - -/** - * Maximum size of the buffer for sending and receiving UDP packets. - * Please note that this size does not limit the size of the UDP packet that can be sent. - * To set the limit for packet size, modify the `pkt_size` parameter. - * For instance, it is possible to set the UDP buffer to 4096 to send or receive packets, - * but please keep in mind that the `pkt_size` option limits the packet size to 1400. - */ -#define MAX_UDP_BUFFER_SIZE 4096 - -/* Referring to Chrome's definition of RTP payload types. */ -#define WHIP_RTP_PAYLOAD_TYPE_H264 106 -#define WHIP_RTP_PAYLOAD_TYPE_OPUS 111 -#define WHIP_RTP_PAYLOAD_TYPE_VIDEO_RTX 105 - -/** - * The STUN message header, which is 20 bytes long, comprises the - * STUNMessageType (1B), MessageLength (2B), MagicCookie (4B), - * and TransactionID (12B). - * See https://datatracker.ietf.org/doc/html/rfc5389#section-6 - */ -#define ICE_STUN_HEADER_SIZE 20 - -/** - * The RTP header is 12 bytes long, comprising the Version(1B), PT(1B), - * SequenceNumber(2B), Timestamp(4B), and SSRC(4B). - * See https://www.rfc-editor.org/rfc/rfc3550#section-5.1 - */ -#define WHIP_RTP_HEADER_SIZE 12 - -/** - * For RTCP, PT is [128, 223] (or without marker [0, 95]). Literally, RTCP starts - * from 64 not 0, so PT is [192, 223] (or without marker [64, 95]), see "RTCP Control - * Packet Types (PT)" at - * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-pa… - * - * For RTP, the PT is [96, 127], or [224, 255] with marker. See "RTP Payload Types (PT) - * for standard audio and video encodings" at - * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-pa… - */ -#define WHIP_RTCP_PT_START 192 -#define WHIP_RTCP_PT_END 223 - -/** - * In the case of ICE-LITE, these fields are not used; instead, they are defined - * as constant values. - */ -#define WHIP_SDP_SESSION_ID "4489045141692799359" -#define WHIP_SDP_CREATOR_IP "127.0.0.1" - -/** - * Refer to RFC 7675 5.1, - * - * To prevent expiry of consent, a STUN binding request can be sent periodically. - * Implementations SHOULD set a default interval of 5 seconds(5000ms). - * - * Consent expires after 30 seconds(30000ms). - */ -#define WHIP_ICE_CONSENT_CHECK_INTERVAL 5000 -#define WHIP_ICE_CONSENT_EXPIRED_TIMER 30000 - -/* Calculate the elapsed time from starttime to endtime in milliseconds. */ -#define ELAPSED(starttime, endtime) ((float)(endtime - starttime) / 1000) - -/* STUN Attribute, comprehension-required range (0x0000-0x7FFF) */ -enum STUNAttr { - STUN_ATTR_USERNAME = 0x0006, /// shared secret response/bind request - STUN_ATTR_PRIORITY = 0x0024, /// must be included in a Binding request - STUN_ATTR_USE_CANDIDATE = 0x0025, /// bind request - STUN_ATTR_MESSAGE_INTEGRITY = 0x0008, /// bind request/response - STUN_ATTR_FINGERPRINT = 0x8028, /// rfc5389 - STUN_ATTR_ICE_CONTROLLING = 0x802A, /// ICE controlling role -}; - -enum WHIPState { - WHIP_STATE_NONE, - - /* The initial state. */ - WHIP_STATE_INIT, - /* The muxer has sent the offer to the peer. */ - WHIP_STATE_OFFER, - /* The muxer has received the answer from the peer. */ - WHIP_STATE_ANSWER, - /** - * After parsing the answer received from the peer, the muxer negotiates the abilities - * in the offer that it generated. - */ - WHIP_STATE_NEGOTIATED, - /* The muxer has connected to the peer via UDP. */ - WHIP_STATE_UDP_CONNECTED, - /* The muxer has sent the ICE request to the peer. */ - WHIP_STATE_ICE_CONNECTING, - /* The muxer has received the ICE response from the peer. */ - WHIP_STATE_ICE_CONNECTED, - /* The muxer has finished the DTLS handshake with the peer. */ - WHIP_STATE_DTLS_FINISHED, - /* The muxer has finished the SRTP setup. */ - WHIP_STATE_SRTP_FINISHED, - /* The muxer is ready to send/receive media frames. */ - WHIP_STATE_READY, - /* The muxer is failed. */ - WHIP_STATE_FAILED, -}; - -typedef enum WHIPFlags { - WHIP_DTLS_ACTIVE = (1 << 0), -} WHIPFlags; - -typedef struct WHIPContext { - AVClass *av_class; - - uint32_t flags; - /* The state of the RTC connection. */ - enum WHIPState state; - - /* Parameters for the input audio and video codecs. */ - AVCodecParameters *audio_par; - AVCodecParameters *video_par; - - /** - * The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet; - * therefore, it is essential to insert the SPS and PPS before each IDR frame - * in such cases. - */ - int h264_annexb_insert_sps_pps; - - /* The random number generator. */ - AVLFG rnd; - - /* The ICE username and pwd fragment generated by the muxer. */ - char ice_ufrag_local[9]; - char ice_pwd_local[33]; - /* The SSRC of the audio and video stream, generated by the muxer. */ - uint32_t audio_ssrc; - uint32_t video_ssrc; - uint32_t video_rtx_ssrc; - - uint16_t audio_first_seq; - uint16_t video_first_seq; - /* The PT(Payload Type) of stream, generated by the muxer. */ - uint8_t audio_payload_type; - uint8_t video_payload_type; - uint8_t video_rtx_payload_type; - /** - * This is the SDP offer generated by the muxer based on the codec parameters, - * DTLS, and ICE information. - */ - char *sdp_offer; - - int is_peer_ice_lite; - uint64_t ice_tie_breaker; // random 64 bit, for ICE-CONTROLLING - /* The ICE username and pwd from remote server. */ - char *ice_ufrag_remote; - char *ice_pwd_remote; - /** - * This represents the ICE candidate protocol, priority, host and port. - * Currently, we only support one candidate and choose the first UDP candidate. - * However, we plan to support multiple candidates in the future. - */ - char *ice_protocol; - char *ice_host; - int ice_port; - - /* The SDP answer received from the WebRTC server. */ - char *sdp_answer; - /* The resource URL returned in the Location header of WHIP HTTP response. */ - char *whip_resource_url; - - /* These variables represent timestamps used for calculating and tracking the cost. */ - int64_t whip_starttime; - int64_t whip_init_time; - int64_t whip_offer_time; - int64_t whip_answer_time; - int64_t whip_udp_time; - int64_t whip_ice_time; - int64_t whip_dtls_time; - int64_t whip_srtp_time; - int64_t whip_last_consent_tx_time; - int64_t whip_last_consent_rx_time; - - /* The certificate and private key content used for DTLS handshake */ - char cert_buf[MAX_CERTIFICATE_SIZE]; - char key_buf[MAX_CERTIFICATE_SIZE]; - /* The fingerprint of certificate, used in SDP offer. */ - char *dtls_fingerprint; - /** - * This represents the material used to build the SRTP master key. It is - * generated by DTLS and has the following layout: - * 16B 16B 14B 14B - * client_key | server_key | client_salt | server_salt - */ - uint8_t dtls_srtp_materials[(DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN) * 2]; - - /* TODO: Use AVIOContext instead of URLContext */ - URLContext *dtls_uc; - - /* The SRTP send context, to encrypt outgoing packets. */ - SRTPContext srtp_audio_send; - SRTPContext srtp_video_send; - SRTPContext srtp_video_rtx_send; - SRTPContext srtp_rtcp_send; - /* The SRTP receive context, to decrypt incoming packets. */ - SRTPContext srtp_recv; - - /* The UDP transport is used for delivering ICE, DTLS and SRTP packets. */ - URLContext *udp; - /* The buffer for UDP transmission. */ - char buf[MAX_UDP_BUFFER_SIZE]; - - /* The timeout in milliseconds for ICE and DTLS handshake. */ - int handshake_timeout; - /** - * The size of RTP packet, should generally be set to MTU. - * Note that pion requires a smaller value, for example, 1200. - */ - int pkt_size; - int ts_buffer_size;/* Underlying protocol send/receive buffer size */ - /** - * The optional Bearer token for WHIP Authorization. - * See https://www.ietf.org/archive/id/draft-ietf-wish-whip-08.html#name-authentic… - */ - char* authorization; - /* The certificate and private key used for DTLS handshake. */ - char* cert_file; - char* key_file; -} WHIPContext; - -/** - * Whether the packet is a DTLS packet. - */ -static int is_dtls_packet(uint8_t *b, int size) -{ - int ret = 0; - if (size > DTLS_RECORD_LAYER_HEADER_LEN) { - uint16_t version = AV_RB16(&b[1]); - ret = b[0] >= DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC && - (version == DTLS_VERSION_10 || version == DTLS_VERSION_12); - } - return ret; -} - - -/** - * Get or Generate a self-signed certificate and private key for DTLS, - * fingerprint for SDP - */ -static av_cold int certificate_key_init(AVFormatContext *s) -{ - int ret = 0; - WHIPContext *whip = s->priv_data; - - if (whip->cert_file && whip->key_file) { - /* Read the private key and certificate from the file. */ - if ((ret = ff_ssl_read_key_cert(whip->key_file, whip->cert_file, - whip->key_buf, sizeof(whip->key_buf), - whip->cert_buf, sizeof(whip->cert_buf), - &whip->dtls_fingerprint)) < 0) { - av_log(s, AV_LOG_ERROR, "Failed to read DTLS certificate from cert=%s, key=%s\n", - whip->cert_file, whip->key_file); - return ret; - } - } else { - /* Generate a private key to ctx->dtls_pkey and self-signed certificate. */ - if ((ret = ff_ssl_gen_key_cert(whip->key_buf, sizeof(whip->key_buf), - whip->cert_buf, sizeof(whip->cert_buf), - &whip->dtls_fingerprint)) < 0) { - av_log(s, AV_LOG_ERROR, "Failed to generate DTLS private key and certificate\n"); - return ret; - } - } - - return ret; -} - -static av_cold int dtls_initialize(AVFormatContext *s) -{ - int ret = 0; - WHIPContext *whip = s->priv_data; - int is_dtls_active = whip->flags & WHIP_DTLS_ACTIVE; - AVDictionary *opts = NULL; - char buf[256]; - - ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL); - av_dict_set_int(&opts, "mtu", whip->pkt_size, 0); - if (whip->cert_file) { - av_dict_set(&opts, "cert_file", whip->cert_file, 0); - } else - av_dict_set(&opts, "cert_pem", whip->cert_buf, 0); - - if (whip->key_file) { - av_dict_set(&opts, "key_file", whip->key_file, 0); - } else - av_dict_set(&opts, "key_pem", whip->key_buf, 0); - av_dict_set_int(&opts, "external_sock", 1, 0); - av_dict_set_int(&opts, "use_srtp", 1, 0); - av_dict_set_int(&opts, "listen", is_dtls_active ? 0 : 1, 0); - ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, - &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); - av_dict_free(&opts); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to open DTLS url:%s\n", buf); - goto end; - } - /* reuse the udp created by whip */ - ff_tls_set_external_socket(whip->dtls_uc, whip->udp); -end: - return ret; -} - -/** - * Initialize and check the options for the WebRTC muxer. - */ -static av_cold int initialize(AVFormatContext *s) -{ - int ret, ideal_pkt_size = 532; - WHIPContext *whip = s->priv_data; - uint32_t seed; - - whip->whip_starttime = av_gettime_relative(); - - ret = certificate_key_init(s); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to init certificate and key\n"); - return ret; - } - - /* Initialize the random number generator. */ - seed = av_get_random_seed(); - av_lfg_init(&whip->rnd, seed); - - /* 64 bit tie breaker for ICE-CONTROLLING (RFC 8445 16.1) */ - ret = av_random_bytes((uint8_t *)&whip->ice_tie_breaker, sizeof(whip->ice_tie_breaker)); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Couldn't generate random bytes for ICE tie breaker\n"); - return ret; - } - - whip->audio_first_seq = av_lfg_get(&whip->rnd) & 0x0fff; - whip->video_first_seq = whip->audio_first_seq + 1; - - if (whip->pkt_size < ideal_pkt_size) - av_log(whip, AV_LOG_WARNING, "pkt_size=%d(<%d) is too small, may cause packet loss\n", - whip->pkt_size, ideal_pkt_size); - - if (whip->state < WHIP_STATE_INIT) - whip->state = WHIP_STATE_INIT; - whip->whip_init_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "Init state=%d, handshake_timeout=%dms, pkt_size=%d, seed=%d, elapsed=%.2fms\n", - whip->state, whip->handshake_timeout, whip->pkt_size, seed, ELAPSED(whip->whip_starttime, av_gettime_relative())); - - return 0; -} /** * When duplicating a stream, the demuxer has already set the extradata, profile, and @@ -477,7 +53,7 @@ static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par) const uint8_t *r = par->extradata, *r1, *end = par->extradata + par->extradata_size; H264SPS seq, *const sps = &seq; uint32_t state; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; if (par->codec_id != AV_CODEC_ID_H264) return ret; @@ -486,7 +62,7 @@ static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par) return ret; if (!par->extradata || par->extradata_size <= 0) { - av_log(whip, AV_LOG_ERROR, "Unable to parse profile from empty extradata=%p, size=%d\n", + av_log(rtc, AV_LOG_ERROR, "Unable to parse profile from empty extradata=%p, size=%d\n", par->extradata, par->extradata_size); return AVERROR(EINVAL); } @@ -500,12 +76,12 @@ static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par) if ((state & 0x1f) == H264_NAL_SPS) { ret = ff_avc_decode_sps(sps, r, r1 - r); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to decode SPS, state=%x, size=%d\n", + av_log(rtc, AV_LOG_ERROR, "Failed to decode SPS, state=%x, size=%d\n", state, (int)(r1 - r)); return ret; } - av_log(whip, AV_LOG_VERBOSE, "Parse profile=%d, level=%d from SPS\n", + av_log(rtc, AV_LOG_VERBOSE, "Parse profile=%d, level=%d from SPS\n", sps->profile_idc, sps->level_idc); par->profile = sps->profile_idc; par->level = sps->level_idc; @@ -541,44 +117,44 @@ static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par) static int parse_codec(AVFormatContext *s) { int i, ret = 0; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; for (i = 0; i < s->nb_streams; i++) { AVCodecParameters *par = s->streams[i]->codecpar; switch (par->codec_type) { case AVMEDIA_TYPE_VIDEO: - whip->video_par = par; + rtc->video_par = par; if (par->video_delay > 0) { - av_log(whip, AV_LOG_ERROR, "Unsupported B frames by RTC\n"); + av_log(rtc, AV_LOG_ERROR, "Unsupported B frames by RTC\n"); return AVERROR_PATCHWELCOME; } if ((ret = parse_profile_level(s, par)) < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to parse SPS/PPS from extradata\n"); + av_log(rtc, AV_LOG_ERROR, "Failed to parse SPS/PPS from extradata\n"); return AVERROR(EINVAL); } if (par->profile == AV_PROFILE_UNKNOWN) { - av_log(whip, AV_LOG_WARNING, "No profile found in extradata, consider baseline\n"); + av_log(rtc, AV_LOG_WARNING, "No profile found in extradata, consider baseline\n"); return AVERROR(EINVAL); } if (par->level == AV_LEVEL_UNKNOWN) { - av_log(whip, AV_LOG_WARNING, "No level found in extradata, consider 3.1\n"); + av_log(rtc, AV_LOG_WARNING, "No level found in extradata, consider 3.1\n"); return AVERROR(EINVAL); } break; case AVMEDIA_TYPE_AUDIO: - whip->audio_par = par; + rtc->audio_par = par; if (par->ch_layout.nb_channels != 2) { - av_log(whip, AV_LOG_ERROR, "Unsupported audio channels %d by RTC, choose stereo\n", + av_log(rtc, AV_LOG_ERROR, "Unsupported audio channels %d by RTC, choose stereo\n", par->ch_layout.nb_channels); return AVERROR_PATCHWELCOME; } if (par->sample_rate != 48000) { - av_log(whip, AV_LOG_ERROR, "Unsupported audio sample rate %d by RTC, choose 48000\n", par->sample_rate); + av_log(rtc, AV_LOG_ERROR, "Unsupported audio sample rate %d by RTC, choose 48000\n", par->sample_rate); return AVERROR_PATCHWELCOME; } break; @@ -605,37 +181,37 @@ static int generate_sdp_offer(AVFormatContext *s) char bundle[4]; int bundle_index = 0; AVBPrint bp; - WHIPContext *whip = s->priv_data; - int is_dtls_active = whip->flags & WHIP_DTLS_ACTIVE; + RTCContext *rtc = s->priv_data; + int is_dtls_active = rtc->flags & RTC_DTLS_ACTIVE; /* To prevent a crash during cleanup, always initialize it. */ - av_bprint_init(&bp, 1, MAX_SDP_SIZE); + av_bprint_init(&bp, 1, RTC_MAX_SDP_SIZE); - if (whip->sdp_offer) { - av_log(whip, AV_LOG_ERROR, "SDP offer is already set\n"); + if (rtc->sdp_offer) { + av_log(rtc, AV_LOG_ERROR, "SDP offer is already set\n"); ret = AVERROR(EINVAL); goto end; } - snprintf(whip->ice_ufrag_local, sizeof(whip->ice_ufrag_local), "%08x", - av_lfg_get(&whip->rnd)); - snprintf(whip->ice_pwd_local, sizeof(whip->ice_pwd_local), "%08x%08x%08x%08x", - av_lfg_get(&whip->rnd), av_lfg_get(&whip->rnd), av_lfg_get(&whip->rnd), - av_lfg_get(&whip->rnd)); + snprintf(rtc->ice_ufrag_local, sizeof(rtc->ice_ufrag_local), "%08x", + av_lfg_get(&rtc->rnd)); + snprintf(rtc->ice_pwd_local, sizeof(rtc->ice_pwd_local), "%08x%08x%08x%08x", + av_lfg_get(&rtc->rnd), av_lfg_get(&rtc->rnd), av_lfg_get(&rtc->rnd), + av_lfg_get(&rtc->rnd)); - whip->audio_ssrc = av_lfg_get(&whip->rnd); - whip->video_ssrc = whip->audio_ssrc + 1; - whip->video_rtx_ssrc = whip->video_ssrc + 1; + rtc->audio_ssrc = av_lfg_get(&rtc->rnd); + rtc->video_ssrc = rtc->audio_ssrc + 1; + rtc->video_rtx_ssrc = rtc->video_ssrc + 1; - whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS; - whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264; - whip->video_rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_VIDEO_RTX; + rtc->audio_payload_type = RTC_RTP_PAYLOAD_TYPE_OPUS; + rtc->video_payload_type = RTC_RTP_PAYLOAD_TYPE_H264; + rtc->video_rtx_payload_type = RTC_RTP_PAYLOAD_TYPE_VIDEO_RTX; - if (whip->audio_par) { + if (rtc->audio_par) { bundle[bundle_index++] = '0'; bundle[bundle_index++] = ' '; } - if (whip->video_par) { + if (rtc->video_par) { bundle[bundle_index++] = '1'; bundle[bundle_index++] = ' '; } @@ -649,12 +225,12 @@ static int generate_sdp_offer(AVFormatContext *s) "a=group:BUNDLE %s\r\n" "a=extmap-allow-mixed\r\n" "a=msid-semantic: WMS\r\n", - WHIP_SDP_SESSION_ID, - WHIP_SDP_CREATOR_IP, + RTC_SDP_SESSION_ID, + RTC_SDP_CREATOR_IP, bundle); - if (whip->audio_par) { - if (whip->audio_par->codec_id == AV_CODEC_ID_OPUS) + if (rtc->audio_par) { + if (rtc->audio_par->codec_id == AV_CODEC_ID_OPUS) acodec_name = "opus"; av_bprintf(&bp, "" @@ -671,26 +247,26 @@ static int generate_sdp_offer(AVFormatContext *s) "a=rtpmap:%u %s/%d/%d\r\n" "a=ssrc:%u cname:FFmpeg\r\n" "a=ssrc:%u msid:FFmpeg audio\r\n", - whip->audio_payload_type, - whip->ice_ufrag_local, - whip->ice_pwd_local, - whip->dtls_fingerprint, + rtc->audio_payload_type, + rtc->ice_ufrag_local, + rtc->ice_pwd_local, + rtc->dtls_fingerprint, is_dtls_active ? "active" : "passive", - whip->audio_payload_type, + rtc->audio_payload_type, acodec_name, - whip->audio_par->sample_rate, - whip->audio_par->ch_layout.nb_channels, - whip->audio_ssrc, - whip->audio_ssrc); + rtc->audio_par->sample_rate, + rtc->audio_par->ch_layout.nb_channels, + rtc->audio_ssrc, + rtc->audio_ssrc); } - if (whip->video_par) { - level = whip->video_par->level; - if (whip->video_par->codec_id == AV_CODEC_ID_H264) { + if (rtc->video_par) { + level = rtc->video_par->level; + if (rtc->video_par->codec_id == AV_CODEC_ID_H264) { vcodec_name = "H264"; - profile_iop |= whip->video_par->profile & AV_PROFILE_H264_CONSTRAINED ? 1 << 6 : 0; - profile_iop |= whip->video_par->profile & AV_PROFILE_H264_INTRA ? 1 << 4 : 0; - profile_idc = whip->video_par->profile & 0x00ff; + profile_iop |= rtc->video_par->profile & AV_PROFILE_H264_CONSTRAINED ? 1 << 6 : 0; + profile_iop |= rtc->video_par->profile & AV_PROFILE_H264_INTRA ? 1 << 4 : 0; + profile_idc = rtc->video_par->profile & 0x00ff; } av_bprintf(&bp, "" @@ -713,768 +289,50 @@ static int generate_sdp_offer(AVFormatContext *s) "a=ssrc-group:FID %u %u\r\n" "a=ssrc:%u cname:FFmpeg\r\n" "a=ssrc:%u msid:FFmpeg video\r\n", - whip->video_payload_type, - whip->video_rtx_payload_type, - whip->ice_ufrag_local, - whip->ice_pwd_local, - whip->dtls_fingerprint, + rtc->video_payload_type, + rtc->video_rtx_payload_type, + rtc->ice_ufrag_local, + rtc->ice_pwd_local, + rtc->dtls_fingerprint, is_dtls_active ? "active" : "passive", - whip->video_payload_type, + rtc->video_payload_type, vcodec_name, - whip->video_payload_type, + rtc->video_payload_type, profile_idc, profile_iop, level, - whip->video_payload_type, - whip->video_rtx_payload_type, - whip->video_rtx_payload_type, - whip->video_payload_type, - whip->video_ssrc, - whip->video_rtx_ssrc, - whip->video_ssrc, - whip->video_ssrc); + rtc->video_payload_type, + rtc->video_rtx_payload_type, + rtc->video_rtx_payload_type, + rtc->video_payload_type, + rtc->video_ssrc, + rtc->video_rtx_ssrc, + rtc->video_ssrc, + rtc->video_ssrc); } if (!av_bprint_is_complete(&bp)) { - av_log(whip, AV_LOG_ERROR, "Offer exceed max %d, %s\n", MAX_SDP_SIZE, bp.str); + av_log(rtc, AV_LOG_ERROR, "Offer exceed max %d, %s\n", RTC_MAX_SDP_SIZE, bp.str); ret = AVERROR(EIO); goto end; } - whip->sdp_offer = av_strdup(bp.str); - if (!whip->sdp_offer) { + rtc->sdp_offer = av_strdup(bp.str); + if (!rtc->sdp_offer) { ret = AVERROR(ENOMEM); goto end; } - if (whip->state < WHIP_STATE_OFFER) - whip->state = WHIP_STATE_OFFER; - whip->whip_offer_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "Generated state=%d, offer: %s\n", whip->state, whip->sdp_offer); + if (rtc->state < RTC_STATE_OFFER) + rtc->state = RTC_STATE_OFFER; + rtc->rtc_offer_time = av_gettime_relative(); + av_log(rtc, AV_LOG_VERBOSE, "Generated state=%d, offer: %s\n", rtc->state, rtc->sdp_offer); end: av_bprint_finalize(&bp, NULL); return ret; } -/** - * Exchange SDP offer with WebRTC peer to get the answer. - * - * @return 0 if OK, AVERROR_xxx on error - */ -static int exchange_sdp(AVFormatContext *s) -{ - int ret; - char buf[MAX_URL_SIZE]; - AVBPrint bp; - WHIPContext *whip = s->priv_data; - /* The URL context is an HTTP transport layer for the WHIP protocol. */ - URLContext *whip_uc = NULL; - AVDictionary *opts = NULL; - char *hex_data = NULL; - const char *proto_name = avio_find_protocol_name(s->url); - - /* To prevent a crash during cleanup, always initialize it. */ - av_bprint_init(&bp, 1, MAX_SDP_SIZE); - - if (!av_strstart(proto_name, "http", NULL)) { - av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose http, url is %s\n", - proto_name, s->url); - ret = AVERROR(EINVAL); - goto end; - } - - if (!whip->sdp_offer || !strlen(whip->sdp_offer)) { - av_log(whip, AV_LOG_ERROR, "No offer to exchange\n"); - ret = AVERROR(EINVAL); - goto end; - } - - ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\nContent-Type: application/sdp\r\n"); - if (whip->authorization) - ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization); - if (ret <= 0 || ret >= sizeof(buf)) { - av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); - ret = AVERROR(EINVAL); - goto end; - } - - av_dict_set(&opts, "headers", buf, 0); - av_dict_set_int(&opts, "chunked_post", 0, 0); - - hex_data = av_mallocz(2 * strlen(whip->sdp_offer) + 1); - if (!hex_data) { - ret = AVERROR(ENOMEM); - goto end; - } - ff_data_to_hex(hex_data, whip->sdp_offer, strlen(whip->sdp_offer), 0); - av_dict_set(&opts, "post_data", hex_data, 0); - - ret = ffurl_open_whitelist(&whip_uc, s->url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, - &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to request url=%s, offer: %s\n", s->url, whip->sdp_offer); - goto end; - } - - if (ff_http_get_new_location(whip_uc)) { - whip->whip_resource_url = av_strdup(ff_http_get_new_location(whip_uc)); - if (!whip->whip_resource_url) { - ret = AVERROR(ENOMEM); - goto end; - } - } - - while (1) { - ret = ffurl_read(whip_uc, buf, sizeof(buf)); - if (ret == AVERROR_EOF) { - /* Reset the error because we read all response as answer util EOF. */ - ret = 0; - break; - } - if (ret <= 0) { - av_log(whip, AV_LOG_ERROR, "Failed to read response from url=%s, offer is %s, answer is %s\n", - s->url, whip->sdp_offer, whip->sdp_answer); - goto end; - } - - av_bprintf(&bp, "%.*s", ret, buf); - if (!av_bprint_is_complete(&bp)) { - av_log(whip, AV_LOG_ERROR, "Answer exceed max size %d, %.*s, %s\n", MAX_SDP_SIZE, ret, buf, bp.str); - ret = AVERROR(EIO); - goto end; - } - } - - if (!av_strstart(bp.str, "v=", NULL)) { - av_log(whip, AV_LOG_ERROR, "Invalid answer: %s\n", bp.str); - ret = AVERROR(EINVAL); - goto end; - } - - whip->sdp_answer = av_strdup(bp.str); - if (!whip->sdp_answer) { - ret = AVERROR(ENOMEM); - goto end; - } - - if (whip->state < WHIP_STATE_ANSWER) - whip->state = WHIP_STATE_ANSWER; - av_log(whip, AV_LOG_VERBOSE, "Got state=%d, answer: %s\n", whip->state, whip->sdp_answer); - -end: - ffurl_closep(&whip_uc); - av_bprint_finalize(&bp, NULL); - av_dict_free(&opts); - av_freep(&hex_data); - return ret; -} - -/** - * Parses the ICE ufrag, pwd, and candidates from the SDP answer. - * - * This function is used to extract the ICE ufrag, pwd, and candidates from the SDP answer. - * It returns an error if any of these fields is NULL. The function only uses the first - * candidate if there are multiple candidates. However, support for multiple candidates - * will be added in the future. - * - * @param s Pointer to the AVFormatContext - * @returns Returns 0 if successful or AVERROR_xxx if an error occurs. - */ -static int parse_answer(AVFormatContext *s) -{ - int ret = 0; - AVIOContext *pb; - char line[MAX_URL_SIZE]; - const char *ptr; - int i; - WHIPContext *whip = s->priv_data; - - if (!whip->sdp_answer || !strlen(whip->sdp_answer)) { - av_log(whip, AV_LOG_ERROR, "No answer to parse\n"); - ret = AVERROR(EINVAL); - goto end; - } - - pb = avio_alloc_context(whip->sdp_answer, strlen(whip->sdp_answer), 0, NULL, NULL, NULL, NULL); - if (!pb) - return AVERROR(ENOMEM); - - for (i = 0; !avio_feof(pb); i++) { - ff_get_chomp_line(pb, line, sizeof(line)); - if (av_strstart(line, "a=ice-lite", &ptr)) - whip->is_peer_ice_lite = 1; - if (av_strstart(line, "a=ice-ufrag:", &ptr) && !whip->ice_ufrag_remote) { - whip->ice_ufrag_remote = av_strdup(ptr); - if (!whip->ice_ufrag_remote) { - ret = AVERROR(ENOMEM); - goto end; - } - } else if (av_strstart(line, "a=ice-pwd:", &ptr) && !whip->ice_pwd_remote) { - whip->ice_pwd_remote = av_strdup(ptr); - if (!whip->ice_pwd_remote) { - ret = AVERROR(ENOMEM); - goto end; - } - } else if (av_strstart(line, "a=candidate:", &ptr) && !whip->ice_protocol) { - if (ptr && av_stristr(ptr, "host")) { - /* Refer to RFC 5245 15.1 */ - char foundation[33], protocol[17], host[129]; - int component_id, priority, port; - ret = sscanf(ptr, "%32s %d %16s %d %128s %d typ host", foundation, &component_id, protocol, &priority, host, &port); - if (ret != 6) { - av_log(whip, AV_LOG_ERROR, "Failed %d to parse line %d %s from %s\n", - ret, i, line, whip->sdp_answer); - ret = AVERROR(EIO); - goto end; - } - - if (av_strcasecmp(protocol, "udp")) { - av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n", - protocol, i, line, whip->sdp_answer); - ret = AVERROR(EIO); - goto end; - } - - whip->ice_protocol = av_strdup(protocol); - whip->ice_host = av_strdup(host); - whip->ice_port = port; - if (!whip->ice_protocol || !whip->ice_host) { - ret = AVERROR(ENOMEM); - goto end; - } - } - } - } - - if (!whip->ice_pwd_remote || !strlen(whip->ice_pwd_remote)) { - av_log(whip, AV_LOG_ERROR, "No remote ice pwd parsed from %s\n", whip->sdp_answer); - ret = AVERROR(EINVAL); - goto end; - } - - if (!whip->ice_ufrag_remote || !strlen(whip->ice_ufrag_remote)) { - av_log(whip, AV_LOG_ERROR, "No remote ice ufrag parsed from %s\n", whip->sdp_answer); - ret = AVERROR(EINVAL); - goto end; - } - - if (!whip->ice_protocol || !whip->ice_host || !whip->ice_port) { - av_log(whip, AV_LOG_ERROR, "No ice candidate parsed from %s\n", whip->sdp_answer); - ret = AVERROR(EINVAL); - goto end; - } - - if (whip->state < WHIP_STATE_NEGOTIATED) - whip->state = WHIP_STATE_NEGOTIATED; - whip->whip_answer_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "SDP state=%d, offer=%zuB, answer=%zuB, ufrag=%s, pwd=%zuB, transport=%s://%s:%d, elapsed=%.2fms\n", - whip->state, strlen(whip->sdp_offer), strlen(whip->sdp_answer), whip->ice_ufrag_remote, strlen(whip->ice_pwd_remote), - whip->ice_protocol, whip->ice_host, whip->ice_port, ELAPSED(whip->whip_starttime, av_gettime_relative())); - -end: - avio_context_free(&pb); - return ret; -} - -/** - * Creates and marshals an ICE binding request packet. - * - * This function creates and marshals an ICE binding request packet. The function only - * generates the username attribute and does not include goog-network-info, - * use-candidate. However, some of these attributes may be added in the future. - * - * @param s Pointer to the AVFormatContext - * @param buf Pointer to memory buffer to store the request packet - * @param buf_size Size of the memory buffer - * @param request_size Pointer to an integer that receives the size of the request packet - * @return Returns 0 if successful or AVERROR_xxx if an error occurs. - */ -static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, int *request_size) -{ - int ret, size, crc32; - char username[128]; - AVIOContext *pb = NULL; - AVHMAC *hmac = NULL; - WHIPContext *whip = s->priv_data; - - pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL); - if (!pb) - return AVERROR(ENOMEM); - - hmac = av_hmac_alloc(AV_HMAC_SHA1); - if (!hmac) { - ret = AVERROR(ENOMEM); - goto end; - } - - /* Write 20 bytes header */ - avio_wb16(pb, 0x0001); /* STUN binding request */ - avio_wb16(pb, 0); /* length */ - avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */ - avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */ - avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */ - avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */ - - /* The username is the concatenation of the two ICE ufrag */ - ret = snprintf(username, sizeof(username), "%s:%s", whip->ice_ufrag_remote, whip->ice_ufrag_local); - if (ret <= 0 || ret >= sizeof(username)) { - av_log(whip, AV_LOG_ERROR, "Failed to build username %s:%s, max=%zu, ret=%d\n", - whip->ice_ufrag_remote, whip->ice_ufrag_local, sizeof(username), ret); - ret = AVERROR(EIO); - goto end; - } - - /* Write the username attribute */ - avio_wb16(pb, STUN_ATTR_USERNAME); /* attribute type username */ - avio_wb16(pb, ret); /* size of username */ - avio_write(pb, username, ret); /* bytes of username */ - ffio_fill(pb, 0, (4 - (ret % 4)) % 4); /* padding */ - - /* Write the use-candidate attribute */ - avio_wb16(pb, STUN_ATTR_USE_CANDIDATE); /* attribute type use-candidate */ - avio_wb16(pb, 0); /* size of use-candidate */ - - avio_wb16(pb, STUN_ATTR_PRIORITY); - avio_wb16(pb, 4); - avio_wb32(pb, STUN_HOST_CANDIDATE_PRIORITY); - - avio_wb16(pb, STUN_ATTR_ICE_CONTROLLING); - avio_wb16(pb, 8); - avio_wb64(pb, whip->ice_tie_breaker); - - /* Build and update message integrity */ - avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */ - avio_wb16(pb, 20); /* size of message integrity */ - ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */ - size = avio_tell(pb); - buf[2] = (size - 20) >> 8; - buf[3] = (size - 20) & 0xFF; - av_hmac_init(hmac, whip->ice_pwd_remote, strlen(whip->ice_pwd_remote)); - av_hmac_update(hmac, buf, size - 24); - av_hmac_final(hmac, buf + size - 20, 20); - - /* Write the fingerprint attribute */ - avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */ - avio_wb16(pb, 4); /* size of fingerprint */ - ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */ - size = avio_tell(pb); - buf[2] = (size - 20) >> 8; - buf[3] = (size - 20) & 0xFF; - /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */ - crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF; - avio_skip(pb, -4); - avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */ - - *request_size = size; - -end: - avio_context_free(&pb); - av_hmac_free(hmac); - return ret; -} - -/** - * Create an ICE binding response. - * - * This function generates an ICE binding response and writes it to the provided - * buffer. The response is signed using the local password for message integrity. - * - * @param s Pointer to the AVFormatContext structure. - * @param tid Pointer to the transaction ID of the binding request. The tid_size should be 12. - * @param tid_size The size of the transaction ID, should be 12. - * @param buf Pointer to the buffer where the response will be written. - * @param buf_size The size of the buffer provided for the response. - * @param response_size Pointer to an integer that will store the size of the generated response. - * @return Returns 0 if successful or AVERROR_xxx if an error occurs. - */ -static int ice_create_response(AVFormatContext *s, char *tid, int tid_size, uint8_t *buf, int buf_size, int *response_size) -{ - int ret = 0, size, crc32; - AVIOContext *pb = NULL; - AVHMAC *hmac = NULL; - WHIPContext *whip = s->priv_data; - - if (tid_size != 12) { - av_log(whip, AV_LOG_ERROR, "Invalid transaction ID size. Expected 12, got %d\n", tid_size); - return AVERROR(EINVAL); - } - - pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL); - if (!pb) - return AVERROR(ENOMEM); - - hmac = av_hmac_alloc(AV_HMAC_SHA1); - if (!hmac) { - ret = AVERROR(ENOMEM); - goto end; - } - - /* Write 20 bytes header */ - avio_wb16(pb, 0x0101); /* STUN binding response */ - avio_wb16(pb, 0); /* length */ - avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */ - avio_write(pb, tid, tid_size); /* transaction ID */ - - /* Build and update message integrity */ - avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */ - avio_wb16(pb, 20); /* size of message integrity */ - ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */ - size = avio_tell(pb); - buf[2] = (size - 20) >> 8; - buf[3] = (size - 20) & 0xFF; - av_hmac_init(hmac, whip->ice_pwd_local, strlen(whip->ice_pwd_local)); - av_hmac_update(hmac, buf, size - 24); - av_hmac_final(hmac, buf + size - 20, 20); - - /* Write the fingerprint attribute */ - avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */ - avio_wb16(pb, 4); /* size of fingerprint */ - ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */ - size = avio_tell(pb); - buf[2] = (size - 20) >> 8; - buf[3] = (size - 20) & 0xFF; - /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */ - crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF; - avio_skip(pb, -4); - avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */ - - *response_size = size; - -end: - avio_context_free(&pb); - av_hmac_free(hmac); - return ret; -} - -/** - * A Binding request has class=0b00 (request) and method=0b000000000001 (Binding) - * and is encoded into the first 16 bits as 0x0001. - * See https://datatracker.ietf.org/doc/html/rfc5389#section-6 - */ -static int ice_is_binding_request(uint8_t *b, int size) -{ - return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0001; -} - -/** - * A Binding response has class=0b10 (success response) and method=0b000000000001, - * and is encoded into the first 16 bits as 0x0101. - */ -static int ice_is_binding_response(uint8_t *b, int size) -{ - return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0101; -} - -/** - * In RTP packets, the first byte is represented as 0b10xxxxxx, where the initial - * two bits (0b10) indicate the RTP version, - * see https://www.rfc-editor.org/rfc/rfc3550#section-5.1 - * The RTCP packet header is similar to RTP, - * see https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1 - */ -static int media_is_rtp_rtcp(const uint8_t *b, int size) -{ - return size >= WHIP_RTP_HEADER_SIZE && (b[0] & 0xC0) == 0x80; -} - -/* Whether the packet is RTCP. */ -static int media_is_rtcp(const uint8_t *b, int size) -{ - return size >= WHIP_RTP_HEADER_SIZE && b[1] >= WHIP_RTCP_PT_START && b[1] <= WHIP_RTCP_PT_END; -} - -/** - * This function handles incoming binding request messages by responding to them. - * If the message is not a binding request, it will be ignored. - */ -static int ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_size) -{ - int ret = 0, size; - char tid[12]; - WHIPContext *whip = s->priv_data; - - /* Ignore if not a binding request. */ - if (!ice_is_binding_request(buf, buf_size)) - return ret; - - if (buf_size < ICE_STUN_HEADER_SIZE) { - av_log(whip, AV_LOG_ERROR, "Invalid STUN message, expected at least %d, got %d\n", - ICE_STUN_HEADER_SIZE, buf_size); - return AVERROR(EINVAL); - } - - /* Parse transaction id from binding request in buf. */ - memcpy(tid, buf + 8, 12); - - /* Build the STUN binding response. */ - ret = ice_create_response(s, tid, sizeof(tid), whip->buf, sizeof(whip->buf), &size); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding response, size=%d\n", size); - return ret; - } - - ret = ffurl_write(whip->udp, whip->buf, size); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding response, size=%d\n", size); - return ret; - } - - return 0; -} - -/** - * To establish a connection with the UDP server, we utilize ICE-LITE in a Client-Server - * mode. In this setup, FFmpeg acts as the UDP client, while the peer functions as the - * UDP server. - */ -static int udp_connect(AVFormatContext *s) -{ - int ret = 0; - char url[256]; - AVDictionary *opts = NULL; - WHIPContext *whip = s->priv_data; - - /* Build UDP URL and create the UDP context as transport. */ - ff_url_join(url, sizeof(url), "udp", NULL, whip->ice_host, whip->ice_port, NULL); - - av_dict_set_int(&opts, "connect", 1, 0); - av_dict_set_int(&opts, "fifo_size", 0, 0); - /* Pass through the pkt_size and buffer_size to underling protocol */ - av_dict_set_int(&opts, "pkt_size", whip->pkt_size, 0); - av_dict_set_int(&opts, "buffer_size", whip->ts_buffer_size, 0); - - ret = ffurl_open_whitelist(&whip->udp, url, AVIO_FLAG_WRITE, &s->interrupt_callback, - &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to connect udp://%s:%d\n", whip->ice_host, whip->ice_port); - goto end; - } - - /* Make the socket non-blocking, set to READ and WRITE mode after connected */ - ff_socket_nonblock(ffurl_get_file_handle(whip->udp), 1); - whip->udp->flags |= AVIO_FLAG_READ | AVIO_FLAG_NONBLOCK; - - if (whip->state < WHIP_STATE_UDP_CONNECTED) - whip->state = WHIP_STATE_UDP_CONNECTED; - whip->whip_udp_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "UDP state=%d, elapsed=%.2fms, connected to udp://%s:%d\n", - whip->state, ELAPSED(whip->whip_starttime, av_gettime_relative()), whip->ice_host, whip->ice_port); - -end: - av_dict_free(&opts); - return ret; -} - -static int ice_dtls_handshake(AVFormatContext *s) -{ - int ret = 0, size, i; - int64_t starttime = av_gettime_relative(), now; - WHIPContext *whip = s->priv_data; - int is_dtls_active = whip->flags & WHIP_DTLS_ACTIVE; - - if (whip->state < WHIP_STATE_UDP_CONNECTED || !whip->udp) { - av_log(whip, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp); - return AVERROR(EINVAL); - } - - while (1) { - if (whip->state <= WHIP_STATE_ICE_CONNECTING) { - /* Build the STUN binding request. */ - ret = ice_create_request(s, whip->buf, sizeof(whip->buf), &size); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); - goto end; - } - - ret = ffurl_write(whip->udp, whip->buf, size); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); - goto end; - } - - if (whip->state < WHIP_STATE_ICE_CONNECTING) - whip->state = WHIP_STATE_ICE_CONNECTING; - } - -next_packet: - if (whip->state >= WHIP_STATE_DTLS_FINISHED) - /* DTLS handshake is done, exit the loop. */ - break; - - now = av_gettime_relative(); - if (now - starttime >= whip->handshake_timeout * WHIP_US_PER_MS) { - av_log(whip, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%.2fms, elapsed=%.2fms, state=%d\n", - whip->handshake_timeout, ELAPSED(starttime, now), ELAPSED(whip->whip_starttime, now), whip->state); - ret = AVERROR(ETIMEDOUT); - goto end; - } - - /* Read the STUN or DTLS messages from peer. */ - for (i = 0; i < ICE_DTLS_READ_MAX_RETRY; i++) { - if (whip->state > WHIP_STATE_ICE_CONNECTED) - break; - ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf)); - if (ret > 0) - break; - if (ret == AVERROR(EAGAIN)) { - av_usleep(ICE_DTLS_READ_SLEEP_DURATION * WHIP_US_PER_MS); - continue; - } - if (is_dtls_active) - break; - av_log(whip, AV_LOG_ERROR, "Failed to read message\n"); - goto end; - } - - /* Handle the ICE binding response. */ - if (ice_is_binding_response(whip->buf, ret)) { - if (whip->state < WHIP_STATE_ICE_CONNECTED) { - if (whip->is_peer_ice_lite) - whip->state = WHIP_STATE_ICE_CONNECTED; - } - goto next_packet; - } - - /* When a binding request is received, it is necessary to respond immediately. */ - if (ice_is_binding_request(whip->buf, ret)) { - if ((ret = ice_handle_binding_request(s, whip->buf, ret)) < 0) - goto end; - goto next_packet; - } - - /* Handle DTLS handshake */ - if (is_dtls_packet(whip->buf, ret) || is_dtls_active) { - whip->whip_ice_time = av_gettime_relative(); - /* Start consent timer when ICE selected */ - whip->whip_last_consent_tx_time = whip->whip_last_consent_rx_time = whip->whip_ice_time; - whip->state = WHIP_STATE_ICE_CONNECTED; - av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%.2fms\n", - whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "", - whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, whip->whip_ice_time)); - - ret = dtls_initialize(s); - if (ret < 0) - goto end; - ret = ffurl_handshake(whip->dtls_uc); - if (ret < 0) { - whip->state = WHIP_STATE_FAILED; - av_log(whip, AV_LOG_ERROR, "DTLS session failed\n"); - goto end; - } - if (!ret) { - whip->state = WHIP_STATE_DTLS_FINISHED; - whip->whip_dtls_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%.2fms\n", - ELAPSED(whip->whip_starttime, whip->whip_dtls_time)); - } - goto next_packet; - } - } - -end: - return ret; -} - -/** - * Establish the SRTP context using the keying material exported from DTLS. - * - * Create separate SRTP contexts for sending video and audio, as their sequences differ - * and should not share a single context. Generate a single SRTP context for receiving - * RTCP only. - * - * @return 0 if OK, AVERROR_xxx on error - */ -static int setup_srtp(AVFormatContext *s) -{ - int ret; - char recv_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN]; - char send_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN]; - char buf[AV_BASE64_SIZE(DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN)]; - /** - * The profile for OpenSSL's SRTP is SRTP_AES128_CM_SHA1_80, see ssl/d1_srtp.c. - * The profile for FFmpeg's SRTP is SRTP_AES128_CM_HMAC_SHA1_80, see libavformat/srtp.c. - */ - const char* suite = "SRTP_AES128_CM_HMAC_SHA1_80"; - WHIPContext *whip = s->priv_data; - int is_dtls_active = whip->flags & WHIP_DTLS_ACTIVE; - char *cp = is_dtls_active ? send_key : recv_key; - char *sp = is_dtls_active ? recv_key : send_key; - - ret = ff_dtls_export_materials(whip->dtls_uc, whip->dtls_srtp_materials, sizeof(whip->dtls_srtp_materials)); - if (ret < 0) - goto end; - /** - * This represents the material used to build the SRTP master key. It is - * generated by DTLS and has the following layout: - * 16B 16B 14B 14B - * client_key | server_key | client_salt | server_salt - */ - char *client_key = whip->dtls_srtp_materials; - char *server_key = whip->dtls_srtp_materials + DTLS_SRTP_KEY_LEN; - char *client_salt = server_key + DTLS_SRTP_KEY_LEN; - char *server_salt = client_salt + DTLS_SRTP_SALT_LEN; - - memcpy(cp, client_key, DTLS_SRTP_KEY_LEN); - memcpy(cp + DTLS_SRTP_KEY_LEN, client_salt, DTLS_SRTP_SALT_LEN); - - memcpy(sp, server_key, DTLS_SRTP_KEY_LEN); - memcpy(sp + DTLS_SRTP_KEY_LEN, server_salt, DTLS_SRTP_SALT_LEN); - - /* Setup SRTP context for outgoing packets */ - if (!av_base64_encode(buf, sizeof(buf), send_key, sizeof(send_key))) { - av_log(whip, AV_LOG_ERROR, "Failed to encode send key\n"); - ret = AVERROR(EIO); - goto end; - } - - ret = ff_srtp_set_crypto(&whip->srtp_audio_send, suite, buf); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to set crypto for audio send\n"); - goto end; - } - - ret = ff_srtp_set_crypto(&whip->srtp_video_send, suite, buf); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to set crypto for video send\n"); - goto end; - } - - ret = ff_srtp_set_crypto(&whip->srtp_video_rtx_send, suite, buf); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to set crypto for video rtx send\n"); - goto end; - } - - ret = ff_srtp_set_crypto(&whip->srtp_rtcp_send, suite, buf); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to set crypto for rtcp send\n"); - goto end; - } - - /* Setup SRTP context for incoming packets */ - if (!av_base64_encode(buf, sizeof(buf), recv_key, sizeof(recv_key))) { - av_log(whip, AV_LOG_ERROR, "Failed to encode recv key\n"); - ret = AVERROR(EIO); - goto end; - } - - ret = ff_srtp_set_crypto(&whip->srtp_recv, suite, buf); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to set crypto for recv\n"); - goto end; - } - - if (whip->state < WHIP_STATE_SRTP_FINISHED) - whip->state = WHIP_STATE_SRTP_FINISHED; - whip->whip_srtp_time = av_gettime_relative(); - av_log(whip, AV_LOG_VERBOSE, "SRTP setup done, state=%d, suite=%s, key=%zuB, elapsed=%.2fms\n", - whip->state, suite, sizeof(send_key), ELAPSED(whip->whip_starttime, av_gettime_relative())); - -end: - return ret; -} - /** * Callback triggered by the RTP muxer when it creates and sends out an RTP packet. * @@ -1487,33 +345,33 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size) int ret, cipher_size, is_rtcp, is_video; uint8_t payload_type; AVFormatContext *s = opaque; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; SRTPContext *srtp; /* Ignore if not RTP or RTCP packet. */ - if (!media_is_rtp_rtcp(buf, buf_size)) + if (!ff_rtc_media_is_rtp_rtcp(buf, buf_size)) return 0; /* Only support audio, video and rtcp. */ - is_rtcp = media_is_rtcp(buf, buf_size); + is_rtcp = ff_rtc_media_is_rtcp(buf, buf_size); payload_type = buf[1] & 0x7f; - is_video = payload_type == whip->video_payload_type; - if (!is_rtcp && payload_type != whip->video_payload_type && payload_type != whip->audio_payload_type) + is_video = payload_type == rtc->video_payload_type; + if (!is_rtcp && payload_type != rtc->video_payload_type && payload_type != rtc->audio_payload_type) return 0; /* Get the corresponding SRTP context. */ - srtp = is_rtcp ? &whip->srtp_rtcp_send : (is_video? &whip->srtp_video_send : &whip->srtp_audio_send); + srtp = is_rtcp ? &rtc->srtp_rtcp_send : (is_video? &rtc->srtp_video_send : &rtc->srtp_audio_send); /* Encrypt by SRTP and send out. */ - cipher_size = ff_srtp_encrypt(srtp, buf, buf_size, whip->buf, sizeof(whip->buf)); + cipher_size = ff_srtp_encrypt(srtp, buf, buf_size, rtc->buf, sizeof(rtc->buf)); if (cipher_size <= 0 || cipher_size < buf_size) { - av_log(whip, AV_LOG_WARNING, "Failed to encrypt packet=%dB, cipher=%dB\n", buf_size, cipher_size); + av_log(rtc, AV_LOG_WARNING, "Failed to encrypt packet=%dB, cipher=%dB\n", buf_size, cipher_size); return 0; } - ret = ffurl_write(whip->udp, whip->buf, cipher_size); + ret = ffurl_write(rtc->udp, rtc->buf, cipher_size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret); + av_log(rtc, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret); return ret; } @@ -1538,20 +396,20 @@ static int create_rtp_muxer(AVFormatContext *s) AVDictionary *opts = NULL; uint8_t *buffer = NULL; char buf[64]; - WHIPContext *whip = s->priv_data; - whip->udp->flags |= AVIO_FLAG_NONBLOCK; + RTCContext *rtc = s->priv_data; + rtc->udp->flags |= AVIO_FLAG_NONBLOCK; const AVOutputFormat *rtp_format = av_guess_format("rtp", NULL, NULL); if (!rtp_format) { - av_log(whip, AV_LOG_ERROR, "Failed to guess rtp muxer\n"); + av_log(rtc, AV_LOG_ERROR, "Failed to guess rtp muxer\n"); ret = AVERROR(ENOSYS); goto end; } /* The UDP buffer size, may greater than MTU. */ - buffer_size = MAX_UDP_BUFFER_SIZE; + buffer_size = RTC_MAX_UDP_BUFFER_SIZE; /* The RTP payload max size. Reserved some bytes for SRTP checksum and padding. */ - max_packet_size = whip->pkt_size - DTLS_SRTP_CHECKSUM_LEN; + max_packet_size = rtc->pkt_size - DTLS_SRTP_CHECKSUM_LEN; for (i = 0; i < s->nb_streams; i++) { rtp_ctx = avformat_alloc_context(); @@ -1604,15 +462,15 @@ static int create_rtp_muxer(AVFormatContext *s) rtp_ctx->pb->av_class = &ff_avio_class; is_video = s->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO; - snprintf(buf, sizeof(buf), "%d", is_video? whip->video_payload_type : whip->audio_payload_type); + snprintf(buf, sizeof(buf), "%d", is_video? rtc->video_payload_type : rtc->audio_payload_type); av_dict_set(&opts, "payload_type", buf, 0); - snprintf(buf, sizeof(buf), "%d", is_video? whip->video_ssrc : whip->audio_ssrc); + snprintf(buf, sizeof(buf), "%d", is_video? rtc->video_ssrc : rtc->audio_ssrc); av_dict_set(&opts, "ssrc", buf, 0); - av_dict_set_int(&opts, "seq", is_video ? whip->video_first_seq : whip->audio_first_seq, 0); + av_dict_set_int(&opts, "seq", is_video ? rtc->video_first_seq : rtc->audio_first_seq, 0); ret = avformat_write_header(rtp_ctx, &opts); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to write rtp header\n"); + av_log(rtc, AV_LOG_ERROR, "Failed to write rtp header\n"); goto end; } @@ -1622,18 +480,18 @@ static int create_rtp_muxer(AVFormatContext *s) rtp_ctx = NULL; } - if (whip->state < WHIP_STATE_READY) - whip->state = WHIP_STATE_READY; - av_log(whip, AV_LOG_INFO, "Muxer state=%d, buffer_size=%d, max_packet_size=%d, " + if (rtc->state < RTC_STATE_READY) + rtc->state = RTC_STATE_READY; + av_log(rtc, AV_LOG_INFO, "Muxer state=%d, buffer_size=%d, max_packet_size=%d, " "elapsed=%.2fms(init:%.2f,offer:%.2f,answer:%.2f,udp:%.2f,ice:%.2f,dtls:%.2f,srtp:%.2f)\n", - whip->state, buffer_size, max_packet_size, ELAPSED(whip->whip_starttime, av_gettime_relative()), - ELAPSED(whip->whip_starttime, whip->whip_init_time), - ELAPSED(whip->whip_init_time, whip->whip_offer_time), - ELAPSED(whip->whip_offer_time, whip->whip_answer_time), - ELAPSED(whip->whip_answer_time, whip->whip_udp_time), - ELAPSED(whip->whip_udp_time, whip->whip_ice_time), - ELAPSED(whip->whip_ice_time, whip->whip_dtls_time), - ELAPSED(whip->whip_dtls_time, whip->whip_srtp_time)); + rtc->state, buffer_size, max_packet_size, RTC_ELAPSED(rtc->rtc_starttime, av_gettime_relative()), + RTC_ELAPSED(rtc->rtc_starttime, rtc->rtc_init_time), + RTC_ELAPSED(rtc->rtc_init_time, rtc->rtc_offer_time), + RTC_ELAPSED(rtc->rtc_offer_time, rtc->rtc_answer_time), + RTC_ELAPSED(rtc->rtc_answer_time, rtc->rtc_udp_time), + RTC_ELAPSED(rtc->rtc_udp_time, rtc->rtc_ice_time), + RTC_ELAPSED(rtc->rtc_ice_time, rtc->rtc_dtls_time), + RTC_ELAPSED(rtc->rtc_dtls_time, rtc->rtc_srtp_time)); end: if (rtp_ctx) { @@ -1646,64 +504,69 @@ end: return ret; } -/** - * RTC is connectionless, for it's based on UDP, so it check whether sesison is - * timeout. In such case, publishers can't republish the stream util the session - * is timeout. - * This function is called to notify the server that the stream is ended, server - * should expire and close the session immediately, so that publishers can republish - * the stream quickly. - */ -static int dispose_session(AVFormatContext *s) +static av_cold int whip_init(AVFormatContext *s) { int ret; - char buf[MAX_URL_SIZE]; - URLContext *whip_uc = NULL; - AVDictionary *opts = NULL; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; - if (!whip->whip_resource_url) - return 0; - - ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\n"); - if (whip->authorization) - ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization); - if (ret <= 0 || ret >= sizeof(buf)) { - av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); - ret = AVERROR(EINVAL); + if ((ret = ff_rtc_initialize(s)) < 0) goto end; - } - av_dict_set(&opts, "headers", buf, 0); - av_dict_set_int(&opts, "chunked_post", 0, 0); - av_dict_set(&opts, "method", "DELETE", 0); - ret = ffurl_open_whitelist(&whip_uc, whip->whip_resource_url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, - &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to DELETE url=%s\n", whip->whip_resource_url); + if ((ret = parse_codec(s)) < 0) goto end; - } - while (1) { - ret = ffurl_read(whip_uc, buf, sizeof(buf)); - if (ret == AVERROR_EOF) { - ret = 0; - break; - } - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to read response from DELETE url=%s\n", whip->whip_resource_url); - goto end; - } - } + if ((ret = generate_sdp_offer(s)) < 0) + goto end; - av_log(whip, AV_LOG_INFO, "Dispose resource %s ok\n", whip->whip_resource_url); + if ((ret = ff_rtc_connect(s)) < 0) + goto end; + + if ((ret = create_rtp_muxer(s)) < 0) + goto end; end: - ffurl_closep(&whip_uc); - av_dict_free(&opts); + if (ret < 0) + rtc->state = RTC_STATE_FAILED; return ret; } +static void handle_nack_rtx(AVFormatContext *s, int size) +{ + int ret; + RTCContext *rtc = s->priv_data; + uint8_t *buf = NULL; + int rtcp_len, srtcp_len, header_len = 12/*RFC 4585 6.1*/; + + /** + * Refer to RFC 3550 6.4.1 + * The length of this RTCP packet in 32 bit words minus one, + * including the header and any padding. + */ + rtcp_len = (AV_RB16(&rtc->buf[2]) + 1) * 4; + if (rtcp_len <= header_len) { + av_log(rtc, AV_LOG_WARNING, "NACK packet is broken, size: %d\n", rtcp_len); + goto error; + } + /* SRTCP index(4 bytes) + HMAC(SRTP_ARS128_CM_SHA1_80) 10bytes */ + srtcp_len = rtcp_len + 4 + 10; + if (srtcp_len != size) { + av_log(rtc, AV_LOG_WARNING, "NACK packet size not match, srtcp_len:%d, size:%d\n", srtcp_len, size); + goto error; + } + buf = av_memdup(rtc->buf, srtcp_len); + if (!buf) + goto error; + if ((ret = ff_srtp_decrypt(&rtc->srtp_recv, buf, &srtcp_len)) < 0) { + av_log(rtc, AV_LOG_WARNING, "NACK packet decrypt failed: %d\n", ret); + goto error; + } + goto end; +error: + av_log(rtc, AV_LOG_WARNING, "Failed to handle NACK and RTX, Skip...\n"); +end: + av_freep(&buf); +} + /** * Since the h264_mp4toannexb filter only processes the MP4 ISOM format and bypasses * the annexb format, it is necessary to manually insert encoder metadata before each @@ -1785,85 +648,10 @@ fail: return ret; } -static av_cold int whip_init(AVFormatContext *s) -{ - int ret; - WHIPContext *whip = s->priv_data; - - if ((ret = initialize(s)) < 0) - goto end; - - if ((ret = parse_codec(s)) < 0) - goto end; - - if ((ret = generate_sdp_offer(s)) < 0) - goto end; - - if ((ret = exchange_sdp(s)) < 0) - goto end; - - if ((ret = parse_answer(s)) < 0) - goto end; - - if ((ret = udp_connect(s)) < 0) - goto end; - - if ((ret = ice_dtls_handshake(s)) < 0) - goto end; - - if ((ret = setup_srtp(s)) < 0) - goto end; - - if ((ret = create_rtp_muxer(s)) < 0) - goto end; - -end: - if (ret < 0) - whip->state = WHIP_STATE_FAILED; - return ret; -} - -static void handle_nack_rtx(AVFormatContext *s, int size) -{ - int ret; - WHIPContext *whip = s->priv_data; - uint8_t *buf = NULL; - int rtcp_len, srtcp_len, header_len = 12/*RFC 4585 6.1*/; - - /** - * Refer to RFC 3550 6.4.1 - * The length of this RTCP packet in 32 bit words minus one, - * including the header and any padding. - */ - rtcp_len = (AV_RB16(&whip->buf[2]) + 1) * 4; - if (rtcp_len <= header_len) { - av_log(whip, AV_LOG_WARNING, "NACK packet is broken, size: %d\n", rtcp_len); - goto error; - } - /* SRTCP index(4 bytes) + HMAC(SRTP_ARS128_CM_SHA1_80) 10bytes */ - srtcp_len = rtcp_len + 4 + 10; - if (srtcp_len != size) { - av_log(whip, AV_LOG_WARNING, "NACK packet size not match, srtcp_len:%d, size:%d\n", srtcp_len, size); - goto error; - } - buf = av_memdup(whip->buf, srtcp_len); - if (!buf) - goto error; - if ((ret = ff_srtp_decrypt(&whip->srtp_recv, buf, &srtcp_len)) < 0) { - av_log(whip, AV_LOG_WARNING, "NACK packet decrypt failed: %d\n", ret); - goto error; - } - goto end; -error: - av_log(whip, AV_LOG_WARNING, "Failed to handle NACK and RTX, Skip...\n"); -end: - av_freep(&buf); -} - static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) { int ret; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; AVStream *st = s->streams[pkt->stream_index]; AVFormatContext *rtp_ctx = st->priv_data; int64_t now = av_gettime_relative(); @@ -1871,50 +659,50 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) * Refer to RFC 7675 * Periodically send Consent Freshness STUN Binding Request */ - if (now - whip->whip_last_consent_tx_time > WHIP_ICE_CONSENT_CHECK_INTERVAL * WHIP_US_PER_MS) { + if (now - rtc->rtc_last_consent_tx_time > RTC_ICE_CONSENT_CHECK_INTERVAL * RTC_US_PER_MS) { int size; - ret = ice_create_request(s, whip->buf, sizeof(whip->buf), &size); + ret = ff_rtc_ice_create_request(s, rtc->buf, sizeof(rtc->buf), &size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); + av_log(rtc, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); goto end; } - ret = ffurl_write(whip->udp, whip->buf, size); + ret = ffurl_write(rtc->udp, rtc->buf, size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); + av_log(rtc, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); goto end; } - whip->whip_last_consent_tx_time = now; - av_log(whip, AV_LOG_DEBUG, "Consent Freshness check sent\n"); + rtc->rtc_last_consent_tx_time = now; + av_log(rtc, AV_LOG_DEBUG, "Consent Freshness check sent\n"); } /** * Receive packets from the server such as ICE binding requests, DTLS messages, * and RTCP like PLI requests, then respond to them. */ - ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf)); + ret = ffurl_read(rtc->udp, rtc->buf, sizeof(rtc->buf)); if (ret < 0) { if (ret == AVERROR(EAGAIN)) goto write_packet; - av_log(whip, AV_LOG_ERROR, "Failed to read from UDP socket\n"); + av_log(rtc, AV_LOG_ERROR, "Failed to read from UDP socket\n"); goto end; } if (!ret) { - av_log(whip, AV_LOG_ERROR, "Receive EOF from UDP socket\n"); + av_log(rtc, AV_LOG_ERROR, "Receive EOF from UDP socket\n"); goto end; } - if (ice_is_binding_response(whip->buf, ret)) { - whip->whip_last_consent_rx_time = av_gettime_relative(); - av_log(whip, AV_LOG_DEBUG, "Consent Freshness check received\n"); + if (ff_rtc_ice_is_binding_response(rtc->buf, ret)) { + rtc->rtc_last_consent_rx_time = av_gettime_relative(); + av_log(rtc, AV_LOG_DEBUG, "Consent Freshness check received\n"); } - if (is_dtls_packet(whip->buf, ret)) { - if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to handle DTLS message\n"); + if (ff_rtc_is_dtls_packet(rtc->buf, ret)) { + if ((ret = ffurl_write(rtc->dtls_uc, rtc->buf, ret)) < 0) { + av_log(rtc, AV_LOG_ERROR, "Failed to handle DTLS message\n"); goto end; } } - if (media_is_rtcp(whip->buf, ret)) { - uint8_t fmt = whip->buf[0] & 0x1f; - uint8_t pt = whip->buf[1]; + if (ff_rtc_media_is_rtcp(rtc->buf, ret)) { + uint8_t fmt = rtc->buf[0] & 0x1f; + uint8_t pt = rtc->buf[1]; /** * Handle RTCP NACK packet * Refer to RFC 4585 6.2.1 @@ -1927,16 +715,16 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) } write_packet: now = av_gettime_relative(); - if (now - whip->whip_last_consent_rx_time > WHIP_ICE_CONSENT_EXPIRED_TIMER * WHIP_US_PER_MS) { - av_log(whip, AV_LOG_ERROR, + if (now - rtc->rtc_last_consent_rx_time > RTC_ICE_CONSENT_EXPIRED_TIMER * RTC_US_PER_MS) { + av_log(rtc, AV_LOG_ERROR, "Consent Freshness expired after %.2fms (limited %dms), terminate session\n", - ELAPSED(whip->whip_last_consent_rx_time, now), WHIP_ICE_CONSENT_EXPIRED_TIMER); + RTC_ELAPSED(rtc->rtc_last_consent_rx_time, now), RTC_ICE_CONSENT_EXPIRED_TIMER); ret = AVERROR(ETIMEDOUT); goto end; } - if (whip->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) { + if (rtc->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) { if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) { - av_log(whip, AV_LOG_ERROR, "Failed to insert SPS/PPS before IDR\n"); + av_log(rtc, AV_LOG_ERROR, "Failed to insert SPS/PPS before IDR\n"); goto end; } } @@ -1944,104 +732,44 @@ write_packet: ret = ff_write_chained(rtp_ctx, 0, pkt, s, 0); if (ret < 0) { if (ret == AVERROR(EINVAL)) { - av_log(whip, AV_LOG_WARNING, "Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret); + av_log(rtc, AV_LOG_WARNING, "Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret); ret = 0; } else if (ret == AVERROR(EAGAIN)) { - av_log(whip, AV_LOG_ERROR, "UDP send blocked, please increase the buffer via -ts_buffer_size\n"); + av_log(rtc, AV_LOG_ERROR, "UDP send blocked, please increase the buffer via -ts_buffer_size\n"); } else - av_log(whip, AV_LOG_ERROR, "Failed to write packet, size=%d, ret=%d\n", pkt->size, ret); + av_log(rtc, AV_LOG_ERROR, "Failed to write packet, size=%d, ret=%d\n", pkt->size, ret); goto end; } end: if (ret < 0) - whip->state = WHIP_STATE_FAILED; + rtc->state = RTC_STATE_FAILED; return ret; } -static av_cold void whip_deinit(AVFormatContext *s) -{ - int i, ret; - WHIPContext *whip = s->priv_data; - - ret = dispose_session(s); - if (ret < 0) - av_log(whip, AV_LOG_WARNING, "Failed to dispose resource, ret=%d\n", ret); - - for (i = 0; i < s->nb_streams; i++) { - AVFormatContext* rtp_ctx = s->streams[i]->priv_data; - if (!rtp_ctx) - continue; - - av_write_trailer(rtp_ctx); - /** - * Keep in mind that it is necessary to free the buffer of pb since we allocate - * it and pass it to pb using avio_alloc_context, while avio_context_free does - * not perform this action. - */ - av_freep(&rtp_ctx->pb->buffer); - avio_context_free(&rtp_ctx->pb); - avformat_free_context(rtp_ctx); - s->streams[i]->priv_data = NULL; - } - - av_freep(&whip->sdp_offer); - av_freep(&whip->sdp_answer); - av_freep(&whip->whip_resource_url); - av_freep(&whip->ice_ufrag_remote); - av_freep(&whip->ice_pwd_remote); - av_freep(&whip->ice_protocol); - av_freep(&whip->ice_host); - av_freep(&whip->authorization); - av_freep(&whip->cert_file); - av_freep(&whip->key_file); - ff_srtp_free(&whip->srtp_audio_send); - ff_srtp_free(&whip->srtp_video_send); - ff_srtp_free(&whip->srtp_video_rtx_send); - ff_srtp_free(&whip->srtp_rtcp_send); - ff_srtp_free(&whip->srtp_recv); - ffurl_close(whip->dtls_uc); - ffurl_closep(&whip->udp); - av_freep(&whip->dtls_fingerprint); -} - static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket *pkt) { int ret = 1, extradata_isom = 0; uint8_t *b = pkt->data; - WHIPContext *whip = s->priv_data; + RTCContext *rtc = s->priv_data; if (st->codecpar->codec_id == AV_CODEC_ID_H264) { extradata_isom = st->codecpar->extradata_size > 0 && st->codecpar->extradata[0] == 1; if (pkt->size >= 5 && AV_RB32(b) != 0x0000001 && (AV_RB24(b) != 0x000001 || extradata_isom)) { ret = ff_stream_add_bitstream_filter(st, "h264_mp4toannexb", NULL); - av_log(whip, AV_LOG_VERBOSE, "Enable BSF h264_mp4toannexb, packet=[%x %x %x %x %x ...], extradata_isom=%d\n", + av_log(rtc, AV_LOG_VERBOSE, "Enable BSF h264_mp4toannexb, packet=[%x %x %x %x %x ...], extradata_isom=%d\n", b[0], b[1], b[2], b[3], b[4], extradata_isom); } else - whip->h264_annexb_insert_sps_pps = 1; + rtc->h264_annexb_insert_sps_pps = 1; } return ret; } -#define OFFSET(x) offsetof(WHIPContext, x) -#define ENC AV_OPT_FLAG_ENCODING_PARAM -static const AVOption options[] = { - { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC }, - { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC }, - { "ts_buffer_size", "The buffer size, in bytes, of underlying protocol", OFFSET(ts_buffer_size), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, ENC }, - { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0}, 0, UINT_MAX, ENC, .unit = "flags" }, - { "dtls_active", "Set dtls role as active", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_DTLS_ACTIVE}, 0, UINT_MAX, ENC, .unit = "flags" }, - { "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, - { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, - { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, - { NULL }, -}; - static const AVClass whip_muxer_class = { .class_name = "WHIP muxer", .item_name = av_default_item_name, - .option = options, + .option = ff_rtc_options, .version = LIBAVUTIL_VERSION_INT, }; @@ -2054,9 +782,9 @@ const FFOutputFormat ff_whip_muxer = { .p.flags = AVFMT_GLOBALHEADER | AVFMT_NOFILE | AVFMT_EXPERIMENTAL, .p.priv_class = &whip_muxer_class, .flags_internal = FF_OFMT_FLAG_ONLY_DEFAULT_CODECS | FF_OFMT_FLAG_MAX_ONE_OF_EACH, - .priv_data_size = sizeof(WHIPContext), + .priv_data_size = sizeof(RTCContext), .init = whip_init, .write_packet = whip_write_packet, - .deinit = whip_deinit, + .deinit = ff_rtc_close, .check_bitstream = whip_check_bitstream, -}; +}; \ No newline at end of file -- 2.52.0

1 0

[PR] avfilter/mestimate_d3d12: add documentation for the filter (PR #21602)
by Steven Xiao 29 Jan '26

29 Jan '26

PR #21602 opened by Steven Xiao (younengxiao) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21602 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21602.patch Add documentation for filter "mestimate_d3d12". Also bump library version. This is the follow-up for commit https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21217/commits From 5459d41f7170dea320ad330a7b4eb552c2f16953 Mon Sep 17 00:00:00 2001 From: stevxiao <steven.xiao(a)amd.com> Date: Wed, 28 Jan 2026 20:29:31 -0500 Subject: [PATCH] avfilter/mestimate_d3d12: add documentation for the filter Add documentation for filter "mestimate_d3d12". Also bump library version. This is the follow-up for commit https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21217/commits Signed-off-by: younengxiao <steven.xiao(a)amd.com> --- doc/filters.texi | 26 ++++++++++++++++++++++++++ libavfilter/version.h | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/filters.texi b/doc/filters.texi index 0f64b4a3fa..9aecea7d88 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -18096,6 +18096,32 @@ Macroblock size. Default @code{16}. Search parameter. Default @code{7}. @end table +@section mestimate_d3d12 + +Estimate and export motion vectors using D3D12 hardware-accelerated motion estimation. +This filter uses GPU hardware motion estimation capabilities available in +DirectX 12 Video APIs to achieve significant performance improvements +over the software-based @code{mestimate} filter. + +Motion vectors are stored in frame side data to be used by other filters. + +This filter requires the input to be in @code{d3d12} hardware pixel format. +The filter uses quarter-pixel precision for motion vector estimation. + +This filter accepts the following options: +@table @option +@item mb_size +Macroblock size. Only @code{8} and @code{16} are supported. +Default @code{16}. +@end table + +@subsection Examples + +Estimate motion vectors using D3D12 hardware acceleration with 8x8 blocks: +@example +ffmpeg -hwaccel d3d12va -hwaccel_output_format d3d12 -i input.mp4 -vf mestimate_d3d12=mb_size=8 -f null - +@end example + @section midequalizer Apply Midway Image Equalization effect using two video streams. diff --git a/libavfilter/version.h b/libavfilter/version.h index 44dacc29be..4239736a2d 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -31,7 +31,7 @@ #include "version_major.h" -#define LIBAVFILTER_VERSION_MINOR 11 +#define LIBAVFILTER_VERSION_MINOR 12 #define LIBAVFILTER_VERSION_MICRO 101 -- 2.52.0

1 0

[PR] tools/target_sws_fuzzer: add sws filters to fuzzing (PR #21601)
by Kacper Michajłow 28 Jan '26

28 Jan '26

PR #21601 opened by Kacper Michajłow (kasper93) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21601 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21601.patch From cc99f1a3d1db2b1823c65ac9b941994356c7e456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93(a)gmail.com> Date: Wed, 28 Jan 2026 13:20:31 +0100 Subject: [PATCH 1/2] tools/target_sws_fuzzer: don't zero init dest buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Destination is destination and should be initialized by the sws itself. Signed-off-by: Kacper Michajłow <kasper93(a)gmail.com> --- tools/target_sws_fuzzer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/target_sws_fuzzer.c b/tools/target_sws_fuzzer.c index 0e14adb1be..2aca0f3254 100644 --- a/tools/target_sws_fuzzer.c +++ b/tools/target_sws_fuzzer.c @@ -40,7 +40,7 @@ static void error(const char *err) exit(1); } -static int alloc_plane(uint8_t *data[AV_VIDEO_MAX_PLANES], int stride[AV_VIDEO_MAX_PLANES], int w, int h, int format, int *hshift, int *vshift) +static int alloc_plane(uint8_t *data[AV_VIDEO_MAX_PLANES], int stride[AV_VIDEO_MAX_PLANES], int w, int h, int format, int *hshift, int *vshift, int zero_init) { size_t size[AV_VIDEO_MAX_PLANES]; ptrdiff_t ptrdiff_stride[AV_VIDEO_MAX_PLANES]; @@ -62,7 +62,7 @@ static int alloc_plane(uint8_t *data[AV_VIDEO_MAX_PLANES], int stride[AV_VIDEO_M for(int p=0; p<AV_VIDEO_MAX_PLANES; p++) { if (size[p]) { - data[p] = av_mallocz(size[p] + 32); + data[p] = zero_init ? av_mallocz(size[p] + 32) : av_malloc(size[p] + 32); if (!data[p]) return -1; } else @@ -153,11 +153,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // fprintf(stderr, "%d x %d %s -> %d x %d %s\n", srcW, srcH, desc_src->name, dstW, dstH, desc_dst->name); - ret = alloc_plane(src, srcStride, srcW, srcH, srcFormat, &srcHShift, &srcVShift); + ret = alloc_plane(src, srcStride, srcW, srcH, srcFormat, &srcHShift, &srcVShift, 1); if (ret < 0) goto end; - ret = alloc_plane(dst, dstStride, dstW, dstH, dstFormat, &dstHShift, &dstVShift); + ret = alloc_plane(dst, dstStride, dstW, dstH, dstFormat, &dstHShift, &dstVShift, 0); if (ret < 0) goto end; -- 2.52.0 From 329f6b8f01582390896abac55eacf21758256fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93(a)gmail.com> Date: Wed, 28 Jan 2026 13:22:50 +0100 Subject: [PATCH 2/2] tools/target_sws_fuzzer: add sws filters to fuzzing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent regression in commit 795bb37a39e163cbb5b6d897ebb0ca90ea7b449a that skipped clearing some internal buffers, caused pretty significant security implications: 478212631/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-5143269947539456 478212631/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-5389230846312448 478212631/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-4732512093143040 478212631/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-4791771971518464 478212631/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-6030282833854464 478301104/clusterfuzz-testcase-minimized-fuzzer_set_property_MPV_FORMAT_STRING_1-5053287094353920 Found-by: https://github.com/google/oss-fuzz/tree/master/projects/mpv None of which were found by FFmpeg's fuzzing infrastructure, so update the SWS fuzzer to account for that. Signed-off-by: Kacper Michajłow <kasper93(a)gmail.com> --- tools/target_sws_fuzzer.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tools/target_sws_fuzzer.c b/tools/target_sws_fuzzer.c index 2aca0f3254..4a0b639526 100644 --- a/tools/target_sws_fuzzer.c +++ b/tools/target_sws_fuzzer.c @@ -102,6 +102,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { uint8_t *dst[AV_VIDEO_MAX_PLANES] = { 0 }; struct SwsContext *sws = NULL; const AVPixFmtDescriptor *desc_src, *desc_dst; + SwsFilter *src_filter = NULL; + SwsFilter *dst_filter = NULL; + int use_filter = 0; + float filter_luma_gblur = 0, filter_chroma_gblur = 0; + float filter_luma_sharpen = 0, filter_chroma_sharpen = 0; + float filter_chroma_hshift = 0, filter_chroma_vshift = 0; if (size > 128) { GetByteContext gbc; @@ -141,6 +147,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if (flags64 & 0x10) av_force_cpu_flags(0); + use_filter = flags64 & 0x20; + if (use_filter) { + const uint32_t param_max = 0xFF; + uint32_t luma_gblur = bytestream2_get_le32(&gbc) % param_max; + uint32_t chroma_gblur = bytestream2_get_le32(&gbc) % param_max; + uint32_t luma_sharpen = bytestream2_get_le32(&gbc) % param_max; + uint32_t chroma_sharpen = bytestream2_get_le32(&gbc) % param_max; + uint32_t chroma_hshift = bytestream2_get_le32(&gbc) % param_max; + uint32_t chroma_vshift = bytestream2_get_le32(&gbc) % param_max; + + filter_luma_gblur = (float)luma_gblur / param_max * 200.0f - 100.0f; + filter_chroma_gblur = (float)chroma_gblur / param_max * 200.0f - 100.0f; + filter_luma_sharpen = (float)luma_sharpen / param_max * 200.0f - 100.0f; + filter_chroma_sharpen = (float)chroma_sharpen / param_max * 200.0f - 100.0f; + filter_chroma_hshift = (float)chroma_hshift / param_max * 200.0f - 100.0f; + filter_chroma_vshift = (float)chroma_vshift / param_max * 200.0f - 100.0f; + } + if (av_image_check_size(srcW, srcH, srcFormat, NULL) < 0) srcW = srcH = 23; if (av_image_check_size(dstW, dstH, dstFormat, NULL) < 0) @@ -186,7 +210,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { av_opt_set_int(sws, "dst_format", dstFormat, 0); av_opt_set(sws, "alphablend", "none", 0); - ret = sws_init_context(sws, NULL, NULL); + if (use_filter) { + src_filter = sws_getDefaultFilter(filter_luma_gblur, filter_chroma_gblur, + filter_luma_sharpen, filter_chroma_sharpen, + filter_chroma_hshift, filter_chroma_vshift, 0); + dst_filter = sws_getDefaultFilter(filter_luma_gblur, filter_chroma_gblur, + filter_luma_sharpen, filter_chroma_sharpen, + filter_chroma_hshift, filter_chroma_vshift, 0); + } + + ret = sws_init_context(sws, src_filter, dst_filter); if (ret < 0) goto end; @@ -194,6 +227,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { sws_scale(sws, (const uint8_t * const*)src, srcStride, 0, srcH, dst, dstStride); end: + sws_freeFilter(dst_filter); + sws_freeFilter(src_filter); sws_freeContext(sws); free_plane(src); -- 2.52.0

1 0

[PR] avformat: add shared concurrent block cache protocol (PR #21600)
by Niklas Haas 28 Jan '26

28 Jan '26

PR #21600 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21600 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21600.patch This adds a new protocol shared:URI which is distinct from the existing `cache:` in that it is explicity designed to be thread-safe and cross-process, enabling multiple ffmpeg processes (or multiple ffmpeg decoders within the same process) to share a single cache file, for e.g. a remote HTTP stream. As such, it uses a radically different internal design. To facilitate zero-knowledge cross-process interoperability, the cache file itself is just a memory-mapped representation of the underlying file data, which has the side benefit that the resulting cache file will contain a working copy of the streamed file (assuming the stream was read to completion). To keep track of which regions are cached and which are not, we use a secondary file that contains a minimal header along with a static bitmap of blocks within the file. This secondary file is also used to store metadata such as the filesize, if known. Both files can grow dynamically in order to accomodate larger/growing files, and can be atomically updated (through the use of shared space maps). I have extensively checked the space map initalization and update code for race conditions, and I believe the current design to be solid. That said, it is the user's responsibility to ensure that the same cache file is not used for different streams, as we have no way to identify sources uniquely. The lack of any implicit default also means that `shared:` can't be abused via URL injection to access random files on the disk (or leak content from other streams). From 83c7ce0fd69a90f45d47fbe22ff43c297c0e9b6b Mon Sep 17 00:00:00 2001 From: Niklas Haas <git(a)haasn.dev> Date: Mon, 26 Jan 2026 19:46:49 +0100 Subject: [PATCH] avformat: add shared concurrent block cache protocol This adds a new protocol shared:URI which is distinct from the existing `cache:` in that it is explicity designed to be thread-safe and cross-process, enabling multiple ffmpeg processes (or multiple ffmpeg decoders within the same process) to share a single cache file, for e.g. a remote HTTP stream. As such, it uses a radically different internal design. To facilitate zero-knowledge cross-process interoperability, the cache file itself is just a memory-mapped representation of the underlying file data, which has the side benefit that the resulting cache file will contain a working copy of the streamed file (assuming the stream was read to completion). To keep track of which regions are cached and which are not, we use a secondary file that contains a minimal header along with a static bitmap of blocks within the file. This secondary file is also used to store metadata such as the filesize, if known. Both files can grow dynamically in order to accomodate larger/growing files, and can be atomically updated (through the use of shared space maps). I have extensively checked the space map initalization and update code for race conditions, and I believe the current design to be solid. That said, it is the user's responsibility to ensure that the same cache file is not used for different streams, as we have no way to identify sources uniquely. The lack of any implicit default also means that `shared:` can't be abused via URL injection to access random files on the disk (or leak content from other streams). --- configure | 1 + doc/protocols.texi | 25 +++ libavformat/Makefile | 1 + libavformat/protocols.c | 1 + libavformat/shared.c | 467 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 495 insertions(+) create mode 100644 libavformat/shared.c diff --git a/configure b/configure index 6d6e6673dc..0dfc02b2ae 100755 --- a/configure +++ b/configure @@ -3998,6 +3998,7 @@ schannel_conflict="openssl gnutls libtls mbedtls" sctp_protocol_deps="struct_sctp_event_subscribe struct_msghdr_msg_flags" sctp_protocol_select="network" securetransport_conflict="openssl gnutls libtls mbedtls" +shared_protocol_deps="mmap unistd_h" srtp_protocol_select="rtp_protocol srtp" tcp_protocol_select="network" tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls" diff --git a/doc/protocols.texi b/doc/protocols.texi index 350c52a465..a434f2eb90 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1567,6 +1567,31 @@ If set to any value, listen for an incoming connection. Outgoing connection is d Set the maximum number of streams. By default no limit is set. @end table +@section shared + +Thread-safe, persistent, cross-process cache wrapper for input streams. Caches +the input stream to a given path. This is similar to @ref{cache}, but with a +persistent cache on disk that may be shared between multiple processes or +demuxers, even concurrently. + +The accepted options are: +@table @option + +@item shared_path +Path to the shared cache base filename. This option is required. + +@item block_shift +Shift factor (log2) of the block size used for internal reads/writes. Defaults +to 15, i.e. 32KB blocks. This value must agree between mulitple processes, +otherwise an error is raised. + +@end table + +URL Syntax is +@example +shared:@var{URL} +@end example + @section srt Haivision Secure Reliable Transport Protocol via libsrt. diff --git a/libavformat/Makefile b/libavformat/Makefile index 5fd3f7252a..be13e19ce9 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -710,6 +710,7 @@ OBJS-$(CONFIG_RTMPTE_PROTOCOL) += rtmpproto.o rtmpdigest.o rtmppkt.o OBJS-$(CONFIG_RTMPTS_PROTOCOL) += rtmpproto.o rtmpdigest.o rtmppkt.o OBJS-$(CONFIG_RTP_PROTOCOL) += rtpproto.o ip.o OBJS-$(CONFIG_SCTP_PROTOCOL) += sctp.o +OBJS-$(CONFIG_SHARED_PROTOCOL) += shared.o OBJS-$(CONFIG_SRTP_PROTOCOL) += srtpproto.o srtp.o OBJS-$(CONFIG_SUBFILE_PROTOCOL) += subfile.o OBJS-$(CONFIG_TEE_PROTOCOL) += teeproto.o tee_common.o diff --git a/libavformat/protocols.c b/libavformat/protocols.c index d394454d41..021fe614bb 100644 --- a/libavformat/protocols.c +++ b/libavformat/protocols.c @@ -56,6 +56,7 @@ extern const URLProtocol ff_rtmpt_protocol; extern const URLProtocol ff_rtmpte_protocol; extern const URLProtocol ff_rtmpts_protocol; extern const URLProtocol ff_rtp_protocol; +extern const URLProtocol ff_shared_protocol; extern const URLProtocol ff_sctp_protocol; extern const URLProtocol ff_srtp_protocol; extern const URLProtocol ff_subfile_protocol; diff --git a/libavformat/shared.c b/libavformat/shared.c new file mode 100644 index 0000000000..79d6ab6e00 --- /dev/null +++ b/libavformat/shared.c @@ -0,0 +1,467 @@ +/* + * Shared file cache protocol. + * Copyright (c) 2026 Niklas Haas + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Based on cache.c by Michael Niedermayer + */ + +#include "libavutil/avassert.h" +#include "libavutil/avstring.h" +#include "libavutil/file_open.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "url.h" + +#include <errno.h> +#include <fcntl.h> +#include <stdatomic.h> +#include <sys/file.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <unistd.h> + +#define HEADER_MAGIC MKTAG('F', 'S', 'H', '$') +#define HEADER_VERSION 1 + +typedef struct Spacemap { + atomic_uint header_magic; + atomic_ushort version; + atomic_ushort block_shift; + atomic_intptr_t filesize; /* byte offset of true EOF, or 0 if unknown */ + + atomic_uchar blocks[]; +} Spacemap; + +/* Set to value iff the current value is unset (zero) */ +#define DEF_SET_ONCE(ctype, atype) \ + static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \ + { \ + ctype prev = 0; \ + av_assert1(value != 0); \ + if (atomic_compare_exchange_strong_explicit( \ + ptr, &prev, value, memory_order_acquire, memory_order_relaxed)) \ + return 1; \ + else if (prev == value) \ + return 0; \ + else \ + return AVERROR(EINVAL); \ + } + +DEF_SET_ONCE(unsigned int, uint) +DEF_SET_ONCE(unsigned short, ushort) +DEF_SET_ONCE(intptr_t, intptr_t) + +typedef struct SharedContext { + AVClass *class; + URLContext *inner; + int64_t inner_pos; + + /* options */ + char *filename; + int block_shift; + + /* misc state */ + int64_t pos; ///< corrent logical position + uint8_t *tmp_buf; + int block_size; + + /* cache file */ + char *cache_path; + int fd; + + /* space map */ + Spacemap *spacemap; + char *map_path; + off_t map_size; + int mapfd; + + /* statistics */ + int64_t nb_hit; + int64_t nb_miss; +} SharedContext; + +static int shared_close(URLContext *h) +{ + SharedContext *s = h->priv_data; + + ffurl_close(s->inner); + if (s->spacemap) + munmap(s->spacemap, s->map_size); + if (s->fd != -1) + close(s->fd); + if (s->mapfd != -1) + close(s->mapfd); + av_free(s->cache_path); + av_free(s->map_path); + av_free(s->tmp_buf); + + av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n", + s->nb_hit, s->nb_miss); + return 0; +} + +static int spacemap_init(URLContext *h); + +static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options) +{ + SharedContext *s = h->priv_data; + int ret; + + if (!s->filename || !s->filename[0]) { + av_log(h, AV_LOG_ERROR, "Missing path for shared cache!\n"); + return AVERROR(EINVAL); + } + + s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */ + s->block_size = 1 << s->block_shift; + s->tmp_buf = av_malloc(s->block_size); + if (!s->tmp_buf) { + ret = AVERROR(ENOMEM); + goto fail; + } + + s->cache_path = av_asprintf("%s.cache", s->filename); + s->map_path = av_asprintf("%s.spacemap", s->filename); + if (!s->cache_path || !s->map_path) { + ret = AVERROR(ENOMEM); + goto fail; + } + + s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660); + s->mapfd = avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660); + if (!s->fd || !s->mapfd) { + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "Failed to open shared cache file: %s\n", av_err2str(ret)); + goto fail; + } + + ret = spacemap_init(h); + if (ret < 0) + goto fail; + + /* Open underlying protocol */ + av_strstart(arg, "shared:", &arg); + ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback, + options, h->protocol_whitelist, h->protocol_blacklist, h); + + if (ret < 0) + goto fail; + + h->max_packet_size = s->block_size; + h->min_packet_size = s->block_size; + +fail: + if (ret < 0) + shared_close(h); + return ret; +} + +static int spacemap_grow(URLContext *h, int64_t block) +{ + SharedContext *s = h->priv_data; + int64_t num_blocks = block + 1; + size_t map_bytes = sizeof(Spacemap) + ((num_blocks + 7) >> 3); + map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size); + if (map_bytes <= s->map_size) + return 0; + + /* Lock the spacemap to ensure no other process is currently resizing it */ + int ret = flock(s->mapfd, LOCK_EX); + if (ret < 0) + goto fail; + + /* Get current size in case another process already grew the spacemap */ + struct stat st; + ret = fstat(s->mapfd, &st); + if (ret < 0) + goto fail; + + if (map_bytes > st.st_size) { + ret = ftruncate(s->mapfd, map_bytes); + if (ret < 0) + goto fail; + st.st_size = map_bytes; + } + + if (s->spacemap) + munmap(s->spacemap, s->map_size); + s->map_size = st.st_size; + s->spacemap = mmap(NULL, s->map_size, PROT_READ | PROT_WRITE, MAP_SHARED, s->mapfd, 0); + if (!s->spacemap) { + s->map_size = 0; + goto fail; + } + + flock(s->mapfd, LOCK_UN); + + /* Report new size after successful (re)map */ + num_blocks = ((s->map_size - sizeof(Spacemap)) << 3); + av_log(h, AV_LOG_DEBUG, "Resized space map to %zu bytes, new capacity: " + "%"PRId64" blocks = %zu MB\n", (size_t) s->map_size, num_blocks, + (size_t) (num_blocks * s->block_size) >> 20); + return 0; + +fail: + flock(s->mapfd, LOCK_UN); + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret)); + return ret; +} + +static int spacemap_init(URLContext *h) +{ + SharedContext *s = h->priv_data; + int ret; + + /* Grow the spacemap to at least hold one block; this ensures the header + * space is allocated as well */ + ret = spacemap_grow(h, 0); + if (ret < 0) + return ret; + + if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 || + (ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0 || + (ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift)) < 0) + { + av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n"); + av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d, block_shift: %d\n", + HEADER_MAGIC, HEADER_VERSION, s->block_shift); + av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d, block_shift: %d\n", + atomic_load(&s->spacemap->header_magic), + atomic_load(&s->spacemap->version), + atomic_load(&s->spacemap->block_shift)); + return ret; + } + + if (ret) + av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n"); + return 0; +} + +static int64_t get_filesize(URLContext *h) +{ + SharedContext *s = h->priv_data; + return atomic_load_explicit(&s->spacemap->filesize, memory_order_acquire); +} + +static int set_filesize(URLContext *h, int64_t new_size) +{ + SharedContext *s = h->priv_data; + int ret; + + ret = set_once_intptr_t(&s->spacemap->filesize, new_size); + if (ret < 0) { + av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: " + "%"PRId64", got: %"PRId64"!\n", new_size, + atomic_load(&s->spacemap->filesize)); + return ret; + } + + return 0; +} + + +static int64_t block_id(SharedContext *s, int64_t pos) +{ + return pos >> s->block_shift; +} + +static int block_offset(SharedContext *s, int64_t pos) +{ + return pos & (s->block_size - 1); +} + +static int block_is_cached(SharedContext *s, int64_t block) +{ + atomic_uchar *const ptr = &s->spacemap->blocks[block >> 3]; + return atomic_load_explicit(ptr, memory_order_acquire) & (1 << (block & 7)); +} + +static void block_mark_cached(SharedContext *s, int64_t block) +{ + atomic_uchar *const ptr = &s->spacemap->blocks[block >> 3]; + atomic_fetch_or_explicit(ptr, 1 << (block & 7), memory_order_release); +} + +static int shared_read(URLContext *h, unsigned char *buf, int size) +{ + SharedContext *s = h->priv_data; + int ret; + + if (size <= 0) + return 0; + + int64_t filesize = get_filesize(h); + if (filesize) { /* limit read request to true filesize if known */ + if (s->pos + size >= filesize) + size = filesize - s->pos; + if (size <= 0) + return AVERROR_EOF; + } + + const int64_t block = block_id(s, s->pos); + const int64_t offset = block_offset(s, s->pos); + ret = spacemap_grow(h, block); + if (ret < 0) + return ret; + + if (block_is_cached(s, block)) { + const int64_t cached = s->block_size - offset; + ret = pread(s->fd, buf, FFMIN(cached, size), s->pos); + if (ret < 0) { + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret)); + return ret; + } + + s->nb_hit++; + s->pos += ret; + return ret; + } + + /* Cache miss, fetch this block from underlying protocol */ + s->nb_miss++; + + int64_t block_pos = block * s->block_size; + if (s->inner_pos != block_pos) { + block_pos = ffurl_seek(s->inner, block_pos, SEEK_SET); + if (block_pos < 0) { + av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n", av_err2str(ret)); + return ret; + } + + s->inner_pos = block_pos; + } + + /* Try and fetch the entire block; reuse the output buffer if possible */ + const int block_size = filesize ? FFMIN(filesize - block_pos, s->block_size) : s->block_size; + uint8_t *const tmp = (size >= block_size && !offset) ? buf : s->tmp_buf; + int bytes_read = 0; + while (bytes_read < block_size) { + ret = ffurl_read(s->inner, tmp, block_size - bytes_read); + if (!ret || ret == AVERROR_EOF) + break; + else if (ret < 0) + return ret; + + bytes_read += ret; + s->inner_pos += ret; + } + + if (bytes_read < block_size) { + /* Learned location of true EOF, update filesize */ + ret = set_filesize(h, block_pos + bytes_read); + if (ret < 0) + return ret; + } + + if (bytes_read > 0) { + ret = pwrite(s->fd, tmp, bytes_read, block_pos); + if (ret < 0) { + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n", av_err2str(ret)); + return ret; + } + block_mark_cached(s, block); + } else { + return AVERROR_EOF; + } + + const int wanted = FFMIN(bytes_read - offset, size); + av_assert0(wanted >= 0); + if (tmp != buf) + memcpy(buf, &s->tmp_buf[offset], wanted); + s->pos += wanted; + return wanted; +} + +static int64_t shared_seek(URLContext *h, int64_t pos, int whence) +{ + SharedContext *s = h->priv_data; + const int64_t filesize = get_filesize(h); + + if (whence == SEEK_SET) { + return s->pos = pos; + } else if (whence == SEEK_CUR) { + return s->pos += pos; + } else if (whence == SEEK_END) { + if (filesize) + return s->pos = filesize + pos; + /* Defer to underlying protocol if filesize is unknown */ + int64_t res = ffurl_seek(s->inner, pos, whence); + if (res < 0) + return res; + set_filesize(h, res - pos); /* Opportunistically update known filesize */ + return s->pos = s->inner_pos = res; + } else if (whence == AVSEEK_SIZE) { + if (filesize) + return filesize; + int64_t res = ffurl_seek(s->inner, pos, whence); + if (res < 0) + return res; + set_filesize(h, res); + return res; + } else { + return -1; + } +} + +static int shared_get_file_handle(URLContext *h) +{ + SharedContext *s = h->priv_data; + return ffurl_get_file_handle(s->inner); +} + +static int shared_get_short_seek(URLContext *h) +{ + SharedContext *s = h->priv_data; + int ret = ffurl_get_short_seek(s->inner); + if (ret < 0) + return ret; + return FFMAX(ret, s->block_size); +} + +#define OFFSET(x) offsetof(SharedContext, x) +#define D AV_OPT_FLAG_DECODING_PARAM + +static const AVOption options[] = { + { "shared_path", "Set (base) path for shared file cache", OFFSET(filename), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D }, + { "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D }, + {0}, +}; + +static const AVClass shared_context_class = { + .class_name = "shared", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const URLProtocol ff_shared_protocol = { + .name = "shared", + .url_open2 = shared_open, + .url_read = shared_read, + .url_seek = shared_seek, + .url_close = shared_close, + .url_get_file_handle = shared_get_file_handle, + .url_get_short_seek = shared_get_short_seek, + .priv_data_size = sizeof(SharedContext), + .priv_data_class = &shared_context_class, +}; -- 2.52.0

1 0