[FFmpeg-trac] #9791(swscale:new): yuv420_bgr32_ssse3 writes out of buffer boundary during sws_scale() call
FFmpeg
trac at avcodec.org
Tue May 24 01:12:09 EEST 2022
#9791: yuv420_bgr32_ssse3 writes out of buffer boundary during sws_scale() call
-------------------------------------+----------------------------------
Reporter: seanwangwdc | Type: defect
Status: new | Priority: normal
Component: swscale | Version: 4.4.1
Keywords: | Blocked By:
Blocking: | Reproduced by developer: 0
Analyzed by developer: 0 |
-------------------------------------+----------------------------------
Summary of the bug:
yuv420_bgr32_ssse3 scaling writes out of bound during sws_scale() function
call.
observations:
1. Some experiments indicated that there is no out of bound writes
observed when using the old mmx optimized implementation instead.
2. this issue occurs on windows, macOS, and linux builds.
sample code:
-- BEGIN --
{{{
#include <stdlib.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/log.h>
#include <libswscale/swscale.h>
void logFromSWS(char* str) {
char buf[1024];
memset(buf, 0, 1024);
sprintf(buf, "logfromsws: %s\n", str);
printf(buf);
}
int main(int argc, char** argv) {
av_log_set_level(AV_LOG_QUIET);
av_register_all();
//sws_register_logging(logFromSWS);
// new video
AVFormatContext* format_ctx = avformat_alloc_context();
if ( format_ctx == NULL ) {
printf("failed to allocate context\n");
return 1;
}
int rc = avformat_open_input(&format_ctx, argv[1], NULL, NULL);
if ( rc != 0 ) {
printf("failed to open input\n");
return 1;
}
if (format_ctx->nb_streams == 0) {
printf("no video stream found\n");
return 1;
}
struct AVStream* videoStream = NULL;
int videoStreamIndex = -1;
for ( int i = 0; i < format_ctx->nb_streams; i++ ) {
struct AVStream* v = format_ctx->streams[i];
if ( v == NULL ) {
continue;
}
if ( v->codec->codec_type == AVMEDIA_TYPE_VIDEO ) {
videoStream = v;
videoStreamIndex = i;
printf("streams=%d, vid index=%d\n", format_ctx->nb_streams, i);
break;
}
}
if ( videoStreamIndex == -1 ) {
printf("failed to find video stream\n");
return 1;
}
int width = videoStream->codec->width;
int height = videoStream->codec->height;
printf("width=%d, height=%d\n", width, height);
// extractFrameAt
AVCodecContext* video_codec_context = videoStream->codec;
if ( video_codec_context == NULL ) {
printf("no video codec context found\n");
return 1;
}
AVCodec* video_codec =
avcodec_find_decoder(video_codec_context->codec_id);
if ( video_codec == NULL ) {
printf("failed to find decoder\n");
return 1;
}
rc = avcodec_open2(video_codec_context, video_codec, NULL);
if ( rc != 0 ) {
printf("failed to open video");
return 1;
}
// seek?
// read next frame
AVFrame* frame = av_frame_alloc();
if ( frame == NULL ) {
printf("failed to allocate frame\n");
return 1;
}
int frame_finished;
int iteration = 0;
while(1) {
printf("read frame iter=%d\n", iteration++);
struct AVPacket packet;
rc = av_read_frame(format_ctx, &packet);
if ( rc != 0 ) {
printf("failed to read frame\n");
return 1;
}
if ( packet.stream_index != videoStreamIndex ) {
av_packet_unref(&packet);
continue;
}
rc = avcodec_decode_video2(video_codec_context, frame,
&frame_finished, &packet);
if ( rc < 0 ) {
printf("failed to decode video %s\n", av_err2str(rc));
return 1;
}
av_packet_unref(&packet);
if ( frame_finished != 0 ) {
break;
}
}
// image from frame
struct SwsContext* scaling_context = sws_getContext(width, height, (enum
AVPixelFormat)frame->format, width, height, AV_PIX_FMT_RGBA, SWS_BILINEAR,
NULL, NULL, NULL);
if ( scaling_context == NULL ) {
printf("failed to get scaling context\n");
return 1;
}
//uint8_t** src_slice = &(frame->data[0]);
//int* src_stride = &(frame->linesize[0]);
AVFrame *frame_rgb;
//frame_rgb = avcodec_alloc_frame();
frame_rgb = av_frame_alloc();
uint8_t* buffer_rgb;
int nbytes = avpicture_get_size(AV_PIX_FMT_RGBA, width, height);
size_t baseAllocSize = nbytes * sizeof(uint8_t);
size_t allocSize = baseAllocSize + 32;
if ( baseAllocSize != (width * height * 4 )) {
printf("invalid allocation size\n");
return 1;
}
buffer_rgb = (uint8_t*)av_malloc(allocSize);
memset(buffer_rgb, 0, nbytes);
avpicture_fill((AVPicture *)frame_rgb, buffer_rgb, AV_PIX_FMT_RGBA,
width, height);
printf("buffer size=%d, height=%d, linesize=%d\n", nbytes, height,
frame_rgb->linesize[0]);
if ( frame_rgb->linesize[0] != baseAllocSize / height ) {
printf("unexpected stride size\n");
return 1;
}
for ( int i = baseAllocSize; i < allocSize; i++ ) {
if ( buffer_rgb[i] != 0 ) {
printf("buffer index %d is not zero\n", i);
return 1;
}
}
rc = sws_scale(scaling_context, frame->data, frame->linesize, 0, height,
frame_rgb->data, frame_rgb->linesize);
if ( rc <= 0 ) {
printf("failed to scale %d\n", rc);
return 1;
}
for ( int i = baseAllocSize; i < allocSize; i++ ) {
if ( buffer_rgb[i] != 0 ) {
printf("buffer index %d is not zero\n", i);
}
}
printf("succeeded %d\n", rc);
return 0;
}
}}}
-- END --
Output:
-- BEGIN --
streams=2, vid index=0
width=1080, height=1280
read frame iter=0
buffer size=5529600, height=1280, linesize=4320
buffer index 5529603 is not zero
buffer index 5529607 is not zero
buffer index 5529611 is not zero
buffer index 5529615 is not zero
buffer index 5529619 is not zero
buffer index 5529623 is not zero
buffer index 5529627 is not zero
buffer index 5529631 is not zero
succeeded 1280
-- END --
The output indicated that yuv420_bgr32_ssse3 ended up writing another 32
bytes past the buffer boundary.
How to reproduce:
The following steps have been reproduced on 4.4.1 and 4.4.2.
1. change line 140 of the sample code to size_t allocSize =
baseAllocSize);
2. build and execute in valgrind with argv[1] being the input file
3. observe output from within valgrind:
valgrind ./video_extract_linux /userhome/Downloads/ASF.asf
==141== Memcheck, a memory error detector
==141== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==141== Using Valgrind-3.14.0 and LibVEX; rerun with -h for copyright info
==141== Command: ./video_extract_linux /userhome/Downloads/ASF.asf
==141==
streams=2, vid index=0
width=1080, height=1280
read frame iter=0
buffer size=5529600, height=1280, linesize=4320
==141== Invalid write of size 8
==141== at 0x608BB33: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x608A38A: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x605E45F: sws_scale (in
/deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x10986A: main (in /userhome/projects/video_extract_linux)
==141== Address 0x71bf080 is 0 bytes after a block of size 5,529,600
alloc'd
==141== at 0x4837EC3: memalign (vg_replace_malloc.c:898)
==141== by 0x4837FF0: posix_memalign (vg_replace_malloc.c:1062)
==141== by 0x5B2CCFE: av_malloc (in
/deps/amd64/lib/libavutil.so.56.70.100)
==141== by 0x109754: main (in /userhome/projects/video_extract_linux)
==141==
==141== Invalid write of size 8
==141== at 0x608BB38: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x608A38A: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x605E45F: sws_scale (in
/deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x10986A: main (in /userhome/projects/video_extract_linux)
==141== Address 0x71bf090 is 16 bytes after a block of size 5,529,600
alloc'd
==141== at 0x4837EC3: memalign (vg_replace_malloc.c:898)
==141== by 0x4837FF0: posix_memalign (vg_replace_malloc.c:1062)
==141== by 0x5B2CCFE: av_malloc (in
/deps/amd64/lib/libavutil.so.56.70.100)
==141== by 0x109754: main (in /userhome/projects/video_extract_linux)
==141==
succeeded 1280
==141==
==141== HEAP SUMMARY:
==141== in use at exit: 10,183,639 bytes in 193 blocks
==141== total heap usage: 233 allocs, 40 frees, 10,462,744 bytes
allocated
==141==
==141== LEAK SUMMARY:
==141== definitely lost: 55,584 bytes in 4 blocks
==141== indirectly lost: 10,128,055 bytes in 189 blocks
==141== possibly lost: 0 bytes in 0 blocks
==141== still reachable: 0 bytes in 0 blocks
==141== suppressed: 0 bytes in 0 blocks
==141== Rerun with --leak-check=full to see details of leaked memory
==141==
==141== For counts of detected and suppressed errors, rerun with: -v
==141== ERROR SUMMARY: 4 errors from 2 contexts (suppressed: 0 from 0)
--
Ticket URL: <https://trac.ffmpeg.org/ticket/9791>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker
More information about the FFmpeg-trac
mailing list