PR #23642 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23642 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23642.patch From 193f371efed20d31ccf7d9c4df2db743b0af86c9 Mon Sep 17 00:00:00 2001 From: Niklas Haas <git@haasn.dev> Date: Mon, 29 Jun 2026 17:35:13 +0200 Subject: [PATCH 1/2] avformat/shared: increase CRC size to 32-bits This increases the amount of corrupted data we can correctly detect as corrupted from (on average) 2 GiB to 128 TiB, at the default 32 KiB block size. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <git@haasn.dev> --- libavformat/shared.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 7c8b17350c..0c99867796 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -79,7 +79,7 @@ static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri) } #define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$') -#define HEADER_VERSION 2 +#define HEADER_VERSION 3 enum BlockState { /* Reserved block state values */ @@ -93,9 +93,9 @@ enum BlockState { */ }; -static uint16_t get_block_crc(const uint8_t *block, size_t block_size) +static uint32_t get_block_crc(const uint8_t *block, size_t block_size) { - uint16_t crc = av_crc(av_crc_get_table(AV_CRC_16_ANSI), 0, block, block_size); + uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_ANSI), 0, block, block_size); switch (crc) { case BLOCK_NONE: case BLOCK_FAILED: @@ -107,7 +107,7 @@ static uint16_t get_block_crc(const uint8_t *block, size_t block_size) } typedef struct Block { - atomic_ushort state; /* enum BlockState */ + atomic_uint state; /* enum BlockState */ } Block; typedef struct Spacemap { @@ -601,7 +601,7 @@ static int shared_read(URLContext *h, unsigned char *buf, int size) return ret; Block *const block = &s->spacemap->blocks[block_id]; - unsigned short state = atomic_load_explicit(&block->state, memory_order_acquire); + unsigned state = atomic_load_explicit(&block->state, memory_order_acquire); int64_t pending_since = 0; int verify_read = 0, is_race = 0; @@ -622,10 +622,10 @@ retry: } } - uint16_t crc = get_block_crc(tmp, block_size); + uint32_t crc = get_block_crc(tmp, block_size); if (crc != state) { av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at " - "offset 0x%"PRIx64": expected CRC: 0x%04X, got: 0x%04X\n", + "offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n", block_id, block_pos, state, crc); return AVERROR(EIO); } @@ -786,9 +786,9 @@ retry: memory_order_relaxed, memory_order_relaxed); } else { - uint16_t crc = get_block_crc(tmp, bytes_read); + uint32_t crc = get_block_crc(tmp, bytes_read); av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at " - "offset 0x%"PRIx64", CRC 0x%04X\n", bytes_read, block_id, + "offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id, block_pos, crc); atomic_store_explicit(&block->state, crc, memory_order_release); } -- 2.52.0 From 46ed278fad7cd527081651a9c9cd39202ae51620 Mon Sep 17 00:00:00 2001 From: Niklas Haas <git@haasn.dev> Date: Mon, 29 Jun 2026 17:56:24 +0200 Subject: [PATCH 2/2] avformat/shared: add -retry_corrupt option Similar to -retry_failed, this retries blocks whose only crime is failing the CRC self-check. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <git@haasn.dev> --- doc/protocols.texi | 5 +++++ libavformat/shared.c | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 5f8cc8094d..d51c12d6f8 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1621,6 +1621,11 @@ If true (the default), transient read errors from the underlying input stream are ignored and retried again. If false, any blocks that previously failed being read from will be treated as permanently inaccessible. +@item retry_corrupt +If true (the default), blocks whose contents fail the CRC integrity check are +re-fetched from the underlying input stream, overwriting the corrupt cached +data. If false, cache corruption is treated as a fatal read error. + @end table URL Syntax is diff --git a/libavformat/shared.c b/libavformat/shared.c index 0c99867796..96b0d8080b 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -152,6 +152,7 @@ typedef struct SharedContext { int read_only; int64_t timeout; int retry_errors; + int retry_corrupt; int verify; /* misc state */ @@ -627,6 +628,8 @@ retry: av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at " "offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n", block_id, block_pos, state, crc); + if (s->retry_corrupt) + goto read_block; return AVERROR(EIO); } @@ -643,9 +646,11 @@ retry: return size; case BLOCK_FAILED: - if (!s->retry_errors) - return AVERROR(EIO); - av_fallthrough; + if (s->retry_errors) + goto read_block; + return AVERROR(EIO); + +read_block: case BLOCK_NONE: if (s->read_only) break; /* don't mark block as pending */ @@ -874,6 +879,7 @@ static const AVOption options[] = { { "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D }, { "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D }, { "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, + { "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, {0}, }; -- 2.52.0