[PATCH 1/3] avcodec/dxtory: unbreak decoding after 6e1a167c556
get_unary() takes at minimum only 1 bit. Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libavcodec/dxtory.c b/libavcodec/dxtory.c index bc19f27951..3cd95afe80 100644 --- a/libavcodec/dxtory.c +++ b/libavcodec/dxtory.c @@ -395,7 +395,7 @@ static int dx2_decode_slice_5x5(GetBitContext *gb, AVFrame *frame, int stride = frame->linesize[0]; uint8_t *dst = frame->data[0] + stride * line; - for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) { + for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) { for (x = 0; x < width; x++) { b = decode_sym_565(gb, lru[0], 5); g = decode_sym_565(gb, lru[1], is_565 ? 6 : 5); @@ -462,7 +462,7 @@ static int dx2_decode_slice_rgb(GetBitContext *gb, AVFrame *frame, int stride = frame->linesize[0]; uint8_t *dst = frame->data[0] + stride * line; - for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) { + for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) { for (x = 0; x < width; x++) { dst[x * 3 + 0] = decode_sym(gb, lru[0]); dst[x * 3 + 1] = decode_sym(gb, lru[1]); @@ -508,7 +508,7 @@ static int dx2_decode_slice_410(GetBitContext *gb, AVFrame *frame, uint8_t *U = frame->data[1] + (ustride >> 2) * line; uint8_t *V = frame->data[2] + (vstride >> 2) * line; - for (y = 0; y < left - 3 && get_bits_left(gb) > 9 * width; y += 4) { + for (y = 0; y < left - 3 && get_bits_left(gb) >= 4 * width; y += 4) { for (x = 0; x < width; x += 4) { for (j = 0; j < 4; j++) for (i = 0; i < 4; i++) @@ -553,7 +553,7 @@ static int dx2_decode_slice_420(GetBitContext *gb, AVFrame *frame, uint8_t *V = frame->data[2] + (vstride >> 1) * line; - for (y = 0; y < left - 1 && get_bits_left(gb) > 6 * width; y += 2) { + for (y = 0; y < left - 1 && get_bits_left(gb) >= 3 * width; y += 2) { for (x = 0; x < width; x += 2) { Y[x + 0 + 0 * ystride] = decode_sym(gb, lru[0]); Y[x + 1 + 0 * ystride] = decode_sym(gb, lru[0]); @@ -597,7 +597,7 @@ static int dx2_decode_slice_444(GetBitContext *gb, AVFrame *frame, uint8_t *U = frame->data[1] + ustride * line; uint8_t *V = frame->data[2] + vstride * line; - for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) { + for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) { for (x = 0; x < width; x++) { Y[x] = decode_sym(gb, lru[0]); U[x] = decode_sym(gb, lru[1]) ^ 0x80; -- 2.17.1
Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 191 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 175 insertions(+), 16 deletions(-) diff --git a/libavcodec/dxtory.c b/libavcodec/dxtory.c index 3cd95afe80..123833741b 100644 --- a/libavcodec/dxtory.c +++ b/libavcodec/dxtory.c @@ -32,6 +32,25 @@ #include "internal.h" #include "unary.h" +static int64_t get_raw_size(enum AVPixelFormat fmt, int width, int height) +{ + switch (fmt) { + case AV_PIX_FMT_RGB555LE: + case AV_PIX_FMT_RGB565LE: + return width * height * 2LL; + case AV_PIX_FMT_RGB24: + case AV_PIX_FMT_BGR24: + case AV_PIX_FMT_YUV444P: + return width * height * 3LL; + case AV_PIX_FMT_YUV420P: + return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 1) * AV_CEIL_RSHIFT(height, 1); + case AV_PIX_FMT_YUV410P: + return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 2) * AV_CEIL_RSHIFT(height, 2); + } + + return 0; +} + static void do_vflip(AVCodecContext *avctx, AVFrame *pic, int vflip) { if (!vflip) @@ -51,17 +70,17 @@ static void do_vflip(AVCodecContext *avctx, AVFrame *pic, int vflip) case AV_PIX_FMT_YUV410P: pic->data[0] += (avctx->height - 1) * pic->linesize[0]; pic->linesize[0] = -pic->linesize[0]; - pic->data[1] += ((avctx->height >> 2) - 1) * pic->linesize[1]; + pic->data[1] += (AV_CEIL_RSHIFT(avctx->height, 2) - 1) * pic->linesize[1]; pic->linesize[1] = -pic->linesize[1]; - pic->data[2] += ((avctx->height >> 2) - 1) * pic->linesize[2]; + pic->data[2] += (AV_CEIL_RSHIFT(avctx->height, 2) - 1) * pic->linesize[2]; pic->linesize[2] = -pic->linesize[2]; break; case AV_PIX_FMT_YUV420P: pic->data[0] += (avctx->height - 1) * pic->linesize[0]; pic->linesize[0] = -pic->linesize[0]; - pic->data[1] += ((avctx->height >> 1) - 1) * pic->linesize[1]; + pic->data[1] += (AV_CEIL_RSHIFT(avctx->height, 1) - 1) * pic->linesize[1]; pic->linesize[1] = -pic->linesize[1]; - pic->data[2] += ((avctx->height >> 1) - 1) * pic->linesize[2]; + pic->data[2] += (AV_CEIL_RSHIFT(avctx->height, 1) - 1) * pic->linesize[2]; pic->linesize[2] = -pic->linesize[2]; break; } @@ -75,7 +94,7 @@ static int dxtory_decode_v1_rgb(AVCodecContext *avctx, AVFrame *pic, uint8_t *dst; int ret; - if (src_size < avctx->width * avctx->height * (int64_t)bpp) { + if (src_size < get_raw_size(id, avctx->width, avctx->height)) { av_log(avctx, AV_LOG_ERROR, "packet too small\n"); return AVERROR_INVALIDDATA; } @@ -104,9 +123,11 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic, { int h, w; uint8_t *Y1, *Y2, *Y3, *Y4, *U, *V; + int height, width, hmargin, vmargin; + int huvborder; int ret; - if (src_size < FFALIGN(avctx->width, 4) * FFALIGN(avctx->height, 4) * 9LL / 8) { + if (src_size < get_raw_size(AV_PIX_FMT_YUV410P, avctx->width, avctx->height)) { av_log(avctx, AV_LOG_ERROR, "packet too small\n"); return AVERROR_INVALIDDATA; } @@ -117,14 +138,20 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic, do_vflip(avctx, pic, vflipped); + height = avctx->height & ~3; + width = avctx->width & ~3; + hmargin = avctx->width - width; + vmargin = avctx->height - height; + huvborder = AV_CEIL_RSHIFT(avctx->width, 2) - 1; + Y1 = pic->data[0]; Y2 = pic->data[0] + pic->linesize[0]; Y3 = pic->data[0] + pic->linesize[0] * 2; Y4 = pic->data[0] + pic->linesize[0] * 3; U = pic->data[1]; V = pic->data[2]; - for (h = 0; h < avctx->height; h += 4) { - for (w = 0; w < avctx->width; w += 4) { + for (h = 0; h < height; h += 4) { + for (w = 0; w < width; w += 4) { AV_COPY32U(Y1 + w, src); AV_COPY32U(Y2 + w, src + 4); AV_COPY32U(Y3 + w, src + 8); @@ -133,6 +160,18 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic, V[w >> 2] = src[17] + 0x80; src += 18; } + if (hmargin) { + for (w = 0; w < hmargin; w++) { + Y1[width + w] = src[w]; + Y2[width + w] = src[w + hmargin * 1]; + Y3[width + w] = src[w + hmargin * 2]; + Y4[width + w] = src[w + hmargin * 3]; + } + src += 4 * hmargin; + U[huvborder] = src[0] + 0x80; + V[huvborder] = src[1] + 0x80; + src += 2; + } Y1 += pic->linesize[0] << 2; Y2 += pic->linesize[0] << 2; Y3 += pic->linesize[0] << 2; @@ -141,6 +180,33 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic, V += pic->linesize[2]; } + if (vmargin) { + for (w = 0; w < width; w += 4) { + AV_COPY32U(Y1 + w, src); + if (vmargin > 1) + AV_COPY32U(Y2 + w, src + 4); + if (vmargin > 2) + AV_COPY32U(Y3 + w, src + 8); + src += 4 * vmargin; + U[w >> 2] = src[0] + 0x80; + V[w >> 2] = src[1] + 0x80; + src += 2; + } + if (hmargin) { + for (w = 0; w < hmargin; w++) { + AV_COPY32U(Y1 + w, src); + if (vmargin > 1) + AV_COPY32U(Y2 + w, src + 4); + if (vmargin > 2) + AV_COPY32U(Y3 + w, src + 8); + src += 4 * vmargin; + } + U[huvborder] = src[0] + 0x80; + V[huvborder] = src[1] + 0x80; + src += 2; + } + } + do_vflip(avctx, pic, vflipped); return 0; @@ -152,9 +218,11 @@ static int dxtory_decode_v1_420(AVCodecContext *avctx, AVFrame *pic, { int h, w; uint8_t *Y1, *Y2, *U, *V; + int height, width, hmargin, vmargin; + int huvborder; int ret; - if (src_size < FFALIGN(avctx->width, 2) * FFALIGN(avctx->height, 2) * 3LL / 2) { + if (src_size < get_raw_size(AV_PIX_FMT_YUV420P, avctx->width, avctx->height)) { av_log(avctx, AV_LOG_ERROR, "packet too small\n"); return AVERROR_INVALIDDATA; } @@ -165,24 +233,52 @@ static int dxtory_decode_v1_420(AVCodecContext *avctx, AVFrame *pic, do_vflip(avctx, pic, vflipped); + height = avctx->height & ~1; + width = avctx->width & ~1; + hmargin = avctx->width - width; + vmargin = avctx->height - height; + huvborder = AV_CEIL_RSHIFT(avctx->width, 1) - 1; + Y1 = pic->data[0]; Y2 = pic->data[0] + pic->linesize[0]; U = pic->data[1]; V = pic->data[2]; - for (h = 0; h < avctx->height; h += 2) { - for (w = 0; w < avctx->width; w += 2) { + for (h = 0; h < height; h += 2) { + for (w = 0; w < width; w += 2) { AV_COPY16(Y1 + w, src); AV_COPY16(Y2 + w, src + 2); U[w >> 1] = src[4] + 0x80; V[w >> 1] = src[5] + 0x80; src += 6; } + if (hmargin) { + Y1[width + 1] = src[0]; + Y2[width + 1] = src[1]; + U[huvborder] = src[2] + 0x80; + V[huvborder] = src[3] + 0x80; + src += 4; + } Y1 += pic->linesize[0] << 1; Y2 += pic->linesize[0] << 1; U += pic->linesize[1]; V += pic->linesize[2]; } + if (vmargin) { + for (w = 0; w < width; w += 2) { + AV_COPY16U(Y1 + w, src); + U[w >> 1] = src[0] + 0x80; + V[w >> 1] = src[1] + 0x80; + src += 4; + } + if (hmargin) { + Y1[w] = src[0]; + U[huvborder] = src[1] + 0x80; + V[huvborder] = src[2] + 0x80; + src += 3; + } + } + do_vflip(avctx, pic, vflipped); return 0; @@ -196,7 +292,7 @@ static int dxtory_decode_v1_444(AVCodecContext *avctx, AVFrame *pic, uint8_t *Y, *U, *V; int ret; - if (src_size < avctx->width * avctx->height * 3LL) { + if (src_size < get_raw_size(AV_PIX_FMT_YUV444P, avctx->width, avctx->height)) { av_log(avctx, AV_LOG_ERROR, "packet too small\n"); return AVERROR_INVALIDDATA; } @@ -508,20 +604,56 @@ static int dx2_decode_slice_410(GetBitContext *gb, AVFrame *frame, uint8_t *U = frame->data[1] + (ustride >> 2) * line; uint8_t *V = frame->data[2] + (vstride >> 2) * line; - for (y = 0; y < left - 3 && get_bits_left(gb) >= 4 * width; y += 4) { - for (x = 0; x < width; x += 4) { + int h, w, hmargin, vmargin; + int huvborder; + + h = frame->height & ~3; + w = frame->width & ~3; + hmargin = frame->width - w; + vmargin = frame->height - h; + huvborder = AV_CEIL_RSHIFT(frame->width, 2) - 1; + + for (y = 0; y < left - 3 && get_bits_left(gb) >= 18 * w / 4 + hmargin * 4 + (!!hmargin * 2); y += 4) { + for (x = 0; x < w; x += 4) { for (j = 0; j < 4; j++) for (i = 0; i < 4; i++) Y[x + i + j * ystride] = decode_sym(gb, lru[0]); U[x >> 2] = decode_sym(gb, lru[1]) ^ 0x80; V[x >> 2] = decode_sym(gb, lru[2]) ^ 0x80; } + if (hmargin) { + for (j = 0; j < 4; j++) + for (i = 0; i < hmargin; i++) + Y[x + i + j * ystride] = decode_sym(gb, lru[0]); + U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80; + V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80; + } Y += ystride << 2; U += ustride; V += vstride; } + if (vmargin && y + vmargin == left) { + for (x = 0; x < width; x += 4) { + for (j = 0; j < vmargin; j++) + for (i = 0; i < 4; i++) + Y[x + i + j * ystride] = decode_sym(gb, lru[0]); + U[x >> 2] = decode_sym(gb, lru[1]) ^ 0x80; + V[x >> 2] = decode_sym(gb, lru[2]) ^ 0x80; + } + if (hmargin) { + for (j = 0; j < vmargin; j++) { + for (i = 0; i < hmargin; i++) + Y[x + i + j * ystride] = decode_sym(gb, lru[0]); + } + U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80; + V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80; + } + + y += vmargin; + } + return y; } @@ -552,9 +684,17 @@ static int dx2_decode_slice_420(GetBitContext *gb, AVFrame *frame, uint8_t *U = frame->data[1] + (ustride >> 1) * line; uint8_t *V = frame->data[2] + (vstride >> 1) * line; + int h, w, hmargin, vmargin; + int huvborder; - for (y = 0; y < left - 1 && get_bits_left(gb) >= 3 * width; y += 2) { - for (x = 0; x < width; x += 2) { + h = frame->height & ~1; + w = frame->width & ~1; + hmargin = frame->width - w; + vmargin = frame->height - h; + huvborder = AV_CEIL_RSHIFT(frame->width, 1) - 1; + + for (y = 0; y < left - 1 && get_bits_left(gb) >= 3 * w + hmargin * 4; y += 2) { + for (x = 0; x < w; x += 2) { Y[x + 0 + 0 * ystride] = decode_sym(gb, lru[0]); Y[x + 1 + 0 * ystride] = decode_sym(gb, lru[0]); Y[x + 0 + 1 * ystride] = decode_sym(gb, lru[0]); @@ -562,12 +702,31 @@ static int dx2_decode_slice_420(GetBitContext *gb, AVFrame *frame, U[x >> 1] = decode_sym(gb, lru[1]) ^ 0x80; V[x >> 1] = decode_sym(gb, lru[2]) ^ 0x80; } + if (hmargin) { + Y[x + 0 * ystride] = decode_sym(gb, lru[0]); + Y[x + 1 * ystride] = decode_sym(gb, lru[0]); + U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80; + V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80; + } Y += ystride << 1; U += ustride; V += vstride; } + if (vmargin) { + for (x = 0; x < width; x += 2) { + Y[x + 0] = decode_sym(gb, lru[0]); + U[x >> 1] = decode_sym(gb, lru[1]) ^ 0x80; + V[x >> 1] = decode_sym(gb, lru[2]) ^ 0x80; + } + if (hmargin) { + Y[x] = decode_sym(gb, lru[0]); + U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80; + V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80; + } + } + return y; } -- 2.17.1
Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/dxtory.c b/libavcodec/dxtory.c index 123833741b..e239abe233 100644 --- a/libavcodec/dxtory.c +++ b/libavcodec/dxtory.c @@ -62,6 +62,8 @@ static void do_vflip(AVCodecContext *avctx, AVFrame *pic, int vflip) pic->linesize[1] = -pic->linesize[1]; pic->data[2] += (avctx->height - 1) * pic->linesize[2]; pic->linesize[2] = -pic->linesize[2]; + case AV_PIX_FMT_RGB555LE: + case AV_PIX_FMT_RGB565LE: case AV_PIX_FMT_BGR24: case AV_PIX_FMT_RGB24: pic->data[0] += (avctx->height - 1) * pic->linesize[0]; -- 2.17.1
On Fri, Sep 04, 2020 at 12:17:04PM +0200, Paul B Mahol wrote:
get_unary() takes at minimum only 1 bit.
Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
should be ok thanks [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB When the tyrant has disposed of foreign enemies by conquest or treaty, and there is nothing more to fear from them, then he is always stirring up some war or other, in order that the people may require a leader. -- Plato
On 9/4/20, Michael Niedermayer <michael@niedermayer.cc> wrote:
On Fri, Sep 04, 2020 at 12:17:04PM +0200, Paul B Mahol wrote:
get_unary() takes at minimum only 1 bit.
Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
should be ok
There is more correct approach in latest version of set in 2nd patch. (It calculates bits more completely for 420 and 410 case) I probably should merge 1st and 2nd patch of that set. What do you think?
thanks
[...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
When the tyrant has disposed of foreign enemies by conquest or treaty, and there is nothing more to fear from them, then he is always stirring up some war or other, in order that the people may require a leader. -- Plato
On Fri, Sep 04, 2020 at 10:03:26PM +0200, Paul B Mahol wrote:
On 9/4/20, Michael Niedermayer <michael@niedermayer.cc> wrote:
On Fri, Sep 04, 2020 at 12:17:04PM +0200, Paul B Mahol wrote:
get_unary() takes at minimum only 1 bit.
Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavcodec/dxtory.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
should be ok
There is more correct approach in latest version of set in 2nd patch. (It calculates bits more completely for 420 and 410 case) I probably should merge 1st and 2nd patch of that set. What do you think?
do what you prefer thx [...] -- Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB If you drop bombs on a foreign country and kill a hundred thousand innocent people, expect your government to call the consequence "unprovoked inhuman terrorist attacks" and use it to justify dropping more bombs and killing more people. The technology changed, the idea is old.
participants (2)
-
Michael Niedermayer -
Paul B Mahol