Vitor Sessak <vitor1001 at gmail.com> writes:

On 01/29/2009 09:47 AM, Kostya wrote:

...

Again, for the record and not for review.

I just took very old patch and extended it to decode palettized LucasArts FMVs from the games like Full Throttle, Curse of Monkey Island, The Dig, etc.

I would be glad if somebody pick and finish it (there are minor glitches during decoding and fade in/fade out scenes are rendered as madly flipping frames instead; oh, and the code style is rather bad).

Git-friendly patch attached. Hope that patchwork will catch it up.

-Vitor

From 94c41ff457f032d1327b67868f39e9e804eecb87 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov <kostya.shishkov at gmail.com> Date: Sun, 23 Jan 2011 15:36:23 +0100 Subject: [PATCH] Add LucasArts SMUSH demuxer and decoder.

--- libavcodec/Makefile | 1 + libavcodec/allcodecs.c | 1 + libavcodec/avcodec.h | 1 + libavcodec/sanm.c | 1381 ++++++++++++++++++++++++++++++++++++++++++++++ libavformat/Makefile | 1 + libavformat/allformats.c | 1 + libavformat/sanm.c | 395 +++++++++++++ 7 files changed, 1781 insertions(+), 0 deletions(-) create mode 100644 libavcodec/sanm.c create mode 100644 libavformat/sanm.c

diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 9abedfc..cf782a6 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -185,6 +185,7 @@ OBJS-$(CONFIG_RV20_DECODER) += rv10.o h263.o mpeg12data.o mpegvideo.o OBJS-$(CONFIG_RV20_ENCODER) += rv10.o mpegvideo_enc.o motion_est.o ratecontrol.o h263.o mpeg12data.o mpegvideo.o error_resilience.o OBJS-$(CONFIG_RV30_DECODER) += rv30.o rv34.o h264pred.o rv30dsp.o OBJS-$(CONFIG_RV40_DECODER) += rv40.o rv34.o h264pred.o rv40dsp.o +OBJS-$(CONFIG_SANM_DECODER) += sanm.o OBJS-$(CONFIG_SGI_DECODER) += sgidec.o OBJS-$(CONFIG_SGI_ENCODER) += sgienc.o rle.o OBJS-$(CONFIG_SHORTEN_DECODER) += shorten.o diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index f369c0a..8da8077 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -137,6 +137,7 @@ void avcodec_register_all(void) REGISTER_ENCDEC (RV20, rv20); REGISTER_DECODER (RV30, rv30); REGISTER_DECODER (RV40, rv40); + REGISTER_DECODER (SANM, sanm); REGISTER_ENCDEC (SGI, sgi); REGISTER_DECODER (SMACKER, smacker); REGISTER_DECODER (SMC, smc); diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 6aa41b5..135b2bc 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -190,6 +190,7 @@ enum CodecID { CODEC_ID_MOTIONPIXELS, CODEC_ID_TGV, CODEC_ID_TGQ, + CODEC_ID_SANM,

/* various PCM "codecs" */ CODEC_ID_PCM_S16LE= 0x10000, diff --git a/libavcodec/sanm.c b/libavcodec/sanm.c new file mode 100644 index 0000000..f01ab11 --- /dev/null +++ b/libavcodec/sanm.c @@ -0,0 +1,1381 @@ +/* + * LucasArts Smush v2 decoder + * Copyright (c) 2006 Cyril Zorin + * <firstname dot lastname at gmail dot com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +/** + * @file sanm.c + * LucasArts Smush v2 decoder + */ + +/* + * Based on http://wiki.multimedia.cx/index.php?title=SANM + */ + +#include "avcodec.h" +#include "bytestream.h" + +#define GLYPH_COORD_VECT_SIZE 16 +static const int8_t glyph4_x[GLYPH_COORD_VECT_SIZE] = { 0, 1, 2, 3, 3, 3, 3, 2, 1, 0, 0, 0, 1, 2, 2, 1 }; +static const int8_t glyph4_y[GLYPH_COORD_VECT_SIZE] = { 0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 2, 1, 1, 1, 2, 2 }; +static const int8_t glyph8_x[GLYPH_COORD_VECT_SIZE] = { 0, 2, 5, 7, 7, 7, 7, 7, 7, 5, 2, 0, 0, 0, 0, 0 }; +static const int8_t glyph8_y[GLYPH_COORD_VECT_SIZE] = { 0, 0, 0, 0, 1, 3, 4, 6, 7, 7, 7, 7, 6, 4, 3, 1 }; + +static const int8_t motion_vectors[256][2] = +{ + {0, 0}, {-1, -43}, {6, -43}, {-9, -42}, {13, -41}, + {-16, -40}, {19, -39}, {-23, -36}, {26, -34}, {-2, -33}, + {4, -33}, {-29, -32}, {-9, -32}, {11, -31}, {-16, -29}, + {32, -29}, {18, -28}, {-34, -26}, {-22, -25}, {-1, -25},

[...]

+};

This table could be made prettier. For your c&p pleasure: { 0, 0}, { -1, -43}, { 6, -43}, { -9, -42}, { 13, -41}, {-16, -40}, { 19, -39}, {-23, -36}, { 26, -34}, { -2, -33}, { 4, -33}, {-29, -32}, { -9, -32}, { 11, -31}, {-16, -29}, { 32, -29}, { 18, -28}, {-34, -26}, {-22, -25}, { -1, -25}, { 3, -25}, { -7, -24}, { 8, -24}, { 24, -23}, { 36, -23}, {-12, -22}, { 13, -21}, {-38, -20}, { 0, -20}, {-27, -19}, { -4, -19}, { 4, -19}, {-17, -18}, { -8, -17}, { 8, -17}, { 18, -17}, { 28, -17}, { 39, -17}, {-12, -15}, { 12, -15}, {-21, -14}, { -1, -14}, { 1, -14}, {-41, -13}, { -5, -13}, { 5, -13}, { 21, -13}, {-31, -12}, {-15, -11}, { -8, -11}, { 8, -11}, { 15, -11}, { -2, -10}, { 1, -10}, { 31, -10}, {-23, -9}, {-11, -9}, { -5, -9}, { 4, -9}, { 11, -9}, { 42, -9}, { 6, -8}, { 24, -8}, {-18, -7}, { -7, -7}, { -3, -7}, { -1, -7}, { 2, -7}, { 18, -7}, {-43, -6}, {-13, -6}, { -4, -6}, { 4, -6}, { 8, -6}, {-33, -5}, { -9, -5}, { -2, -5}, { 0, -5}, { 2, -5}, { 5, -5}, { 13, -5}, {-25, -4}, { -6, -4}, { -3, -4}, { 3, -4}, { 9, -4}, {-19, -3}, { -7, -3}, { -4, -3}, { -2, -3}, { -1, -3}, { 0, -3}, { 1, -3}, { 2, -3}, { 4, -3}, { 6, -3}, { 33, -3}, {-14, -2}, {-10, -2}, { -5, -2}, { -3, -2}, { -2, -2}, { -1, -2}, { 0, -2}, { 1, -2}, { 2, -2}, { 3, -2}, { 5, -2}, { 7, -2}, { 14, -2}, { 19, -2}, { 25, -2}, { 43, -2}, { -7, -1}, { -3, -1}, { -2, -1}, { -1, -1}, { 0, -1}, { 1, -1}, { 2, -1}, { 3, -1}, { 10, -1}, { -5, 0}, { -3, 0}, { -2, 0}, { -1, 0}, { 1, 0}, { 2, 0}, { 3, 0}, { 5, 0}, { 7, 0}, {-10, 1}, { -7, 1}, { -3, 1}, { -2, 1}, { -1, 1}, { 0, 1}, { 1, 1}, { 2, 1}, { 3, 1}, {-43, 2}, {-25, 2}, {-19, 2}, {-14, 2}, { -5, 2}, { -3, 2}, { -2, 2}, { -1, 2}, { 0, 2}, { 1, 2}, { 2, 2}, { 3, 2}, { 5, 2}, { 7, 2}, { 10, 2}, { 14, 2}, {-33, 3}, { -6, 3}, { -4, 3}, { -2, 3}, { -1, 3}, { 0, 3}, { 1, 3}, { 2, 3}, { 4, 3}, { 19, 3}, { -9, 4}, { -3, 4}, { 3, 4}, { 7, 4}, { 25, 4}, {-13, 5}, { -5, 5}, { -2, 5}, { 0, 5}, { 2, 5}, { 5, 5}, { 9, 5}, { 33, 5}, { -8, 6}, { -4, 6}, { 4, 6}, { 13, 6}, { 43, 6}, {-18, 7}, { -2, 7}, { 0, 7}, { 2, 7}, { 7, 7}, { 18, 7}, {-24, 8}, { -6, 8}, {-42, 9}, {-11, 9}, { -4, 9}, { 5, 9}, { 11, 9}, { 23, 9}, {-31, 10}, { -1, 10}, { 2, 10}, {-15, 11}, { -8, 11}, { 8, 11}, { 15, 11}, { 31, 12}, {-21, 13}, { -5, 13}, { 5, 13}, { 41, 13}, { -1, 14}, { 1, 14}, { 21, 14}, {-12, 15}, { 12, 15}, {-39, 17}, {-28, 17}, {-18, 17}, { -8, 17}, { 8, 17}, { 17, 18}, { -4, 19}, { 0, 19}, { 4, 19}, { 27, 19}, { 38, 20}, {-13, 21}, { 12, 22}, {-36, 23}, {-24, 23}, { -8, 24}, { 7, 24}, { -3, 25}, { 1, 25}, { 22, 25}, { 34, 26}, {-18, 28}, {-32, 29}, { 16, 29}, {-11, 31}, { 9, 32}, { 29, 32}, { -4, 33}, { 2, 33}, {-26, 34}, { 23, 36}, {-19, 39}, { 16, 40}, {-13, 41}, { 9, 42}, { -6, 43}, { 1, 43}, { 0, 0}, { 0, 0}, { 0, 0},

+#define NGLYPHS 256 +static int8_t p4x4glyphs[NGLYPHS][16]; +static int8_t p8x8glyphs[NGLYPHS][64]; + +typedef struct sanm_ctx +{ + AVCodecContext *avctx; + const uint8_t* psrc; + + int version, subversion; + uint32_t pal[256]; + int16_t delta_pal[768]; + + int pitch; + int width, height; + int aligned_width, aligned_height; + int prev_seq; + + AVFrame frame, *output; + uint16_t* pdb0, * pdb1, * pdb2; + int rotate_code; + + long npixels, buf_size; + + uint16_t* pcodebook; + uint16_t* psmall_codebook; +} sanm_ctx;

I prefer keeping stars attached to the name they apply to rather than the type name. The latter is confusing.

+typedef struct sanm_frame_header +{ + int seq_num, codec, rotate_code, rle_output_size; + + uint16_t bg_color; + uint32_t width, height; + + const uint8_t* pvstream; +} sanm_frame_header; + +typedef enum glyph_edge +{ + left_edge, + top_edge, + right_edge, + bottom_edge, + no_edge +} glyph_edge; + +typedef enum glyph_dir +{ + dir_left = 0, + dir_up, + dir_right, + dir_down +} glyph_dir;

Uppercase enum values is common practice. We also don't tend to typedef enums, but that's getting into nitpicking territory.

+static uint16_t* point_at(uint16_t* pbuf, int x, int y, int width) +{ + return &pbuf[y * width + x]; +} + +static glyph_edge which_edge(int x, int y, int edge_size) +{ + glyph_edge edge; + const int edge_max = edge_size - 1; + + if (!y) + { + edge = bottom_edge; + } + else if (edge_max == y) + { + edge = top_edge; + } + else if (!x) + { + edge = left_edge; + } + else if (edge_max == x) + { + edge = right_edge; + } + else + { + edge = no_edge; + } + + return edge; +}

Brace placement is inconsistent with usual ffmpeg style.

+static glyph_dir which_direction(glyph_edge edge0, glyph_edge edge1) +{ + glyph_dir dir = -1; + + if ((left_edge == edge0 && right_edge == edge1) || + (left_edge == edge1 && right_edge == edge0) || + (bottom_edge == edge0 && top_edge != edge1) || + (bottom_edge == edge1 && top_edge != edge0)) + { + dir = dir_up; + } + else if ((top_edge == edge0 && bottom_edge != edge1) || + (top_edge == edge1 && bottom_edge != edge0)) + { + dir = dir_down; + } + else if ((left_edge == edge0 && right_edge != edge1) || + (left_edge == edge1 && right_edge != edge0)) + { + dir = dir_left; + } + else if ((top_edge == edge0 && bottom_edge == edge1) || + (top_edge == edge1 && bottom_edge == edge0) || + (right_edge == edge0 && left_edge != edge1) || + (right_edge == edge1 && left_edge != edge0)) + { + dir = dir_right; + } + + return dir; +}

Shift, or, and a switch or table lookup seems simpler.

+static void interp_point(int8_t* ppoint, int x0, int y0, int x1, int y1, int ipoint, int npoints) +{ + if (npoints) + { + ppoint[0] = (x0 * ipoint + x1 * (npoints - ipoint) + npoints / 2) / npoints; + ppoint[1] = (y0 * ipoint + y1 * (npoints - ipoint) + npoints / 2) / npoints; + } + else + { + ppoint[0] = x0; + ppoint[1] = y0; + } +} + +static void make_glyphs(int8_t* pglyphs, const int8_t* pxvector, const int8_t* pyvector, const int side_length) +{ + const int glyph_size = side_length * side_length; + int8_t* pglyph = pglyphs; + + int i, j; + for (i = 0; i != GLYPH_COORD_VECT_SIZE; ++i) + { + int x0 = pxvector[i], y0 = pyvector[i]; + glyph_edge edge0 = which_edge(x0, y0, side_length); + + for (j = 0; j != GLYPH_COORD_VECT_SIZE; ++j, pglyph += glyph_size) + { + int x1 = pxvector[j], y1 = pyvector[j]; + glyph_edge edge1 = which_edge(x1, y1, side_length); + glyph_dir dir = which_direction(edge0, edge1); + int ipoint, npoints = FFMAX(FFABS(x1 - x0), FFABS(y1 - y0)); + + memset(pglyph, 0, glyph_size * sizeof(pglyph[0])); + for (ipoint = 0; ipoint <= npoints; ++ipoint) + { + int8_t point[2]; + int irow, icol; + + interp_point(point, x0, y0, x1, y1, ipoint, npoints); + + switch (dir) + { + case dir_up: + for (irow = point[1]; irow >= 0; --irow) + { + pglyph[point[0] + irow*side_length] = 1; + } + break; + + case dir_down: + for (irow = point[1]; irow < side_length; ++irow) + { + pglyph[point[0] + irow*side_length] = 1; + } + break; + + case dir_left: + for (icol = point[0]; icol >= 0; --icol) + { + pglyph[icol + point[1]*side_length] = 1; + } + break; + + case dir_right: + for (icol = point[0]; icol < side_length; ++icol) + { + pglyph[icol + point[1]*side_length] = 1; + } + break; + } + } + } + } +} + +static void init_sizes(sanm_ctx* pctx, int width, int height) +{ + pctx->width = width; + pctx->height = height; + pctx->npixels = width * height; + + pctx->aligned_width = (width + 7) & ~7; + pctx->aligned_height = (height + 7) & ~7;

FFALIGN()

+ pctx->buf_size = pctx->aligned_width * pctx->aligned_height * sizeof(pctx->pdb0[0]); + pctx->pitch = width; +} + +static void init_buffers(sanm_ctx* pctx) +{ + pctx->pdb0 = av_realloc(pctx->pdb0, pctx->buf_size); + memset(pctx->pdb0, 0, pctx->buf_size); + + pctx->pdb1 = av_realloc(pctx->pdb1, pctx->buf_size); + memset(pctx->pdb1, 0, pctx->buf_size); + + pctx->pdb2 = av_realloc(pctx->pdb2, pctx->buf_size); + memset(pctx->pdb2, 0, pctx->buf_size); +}

Unchecked mallocs.

+static void destroy_buffers(sanm_ctx* pctx) +{ + av_free(pctx->pdb0); + av_free(pctx->pdb1); + av_free(pctx->pdb2); +} + +static int decode_init(AVCodecContext* pav_ctx) +{ + sanm_ctx* pctx = pav_ctx->priv_data; + + pctx->avctx = pav_ctx; + /*if (avcodec_check_dimensions(pav_ctx, pav_ctx->height, pav_ctx->width) < 0) + { + return -1; + }*/

This commented out bit should be either fixed or removed.

+ pctx->version = !pav_ctx->extradata_size; + + pav_ctx->has_b_frames = 0; + pav_ctx->pix_fmt = pctx->version ? PIX_FMT_RGB565 : PIX_FMT_PAL8; + + init_sizes(pctx, pav_ctx->width, pav_ctx->height); + init_buffers(pctx); + pctx->output = &pctx->frame; + pctx->output->data[0] = 0; + pctx->output->quality = 1;

+ make_glyphs(&p4x4glyphs[0][0], glyph4_x, glyph4_y, 4); + make_glyphs(&p8x8glyphs[0][0], glyph8_x, glyph8_y, 8); + + if(!pctx->version){ + int i; + pctx->subversion = AV_RL16(pav_ctx->extradata); + for(i = 0; i < 256; i++) + pctx->pal[i] = AV_RL32(pav_ctx->extradata + 2 + i*4); + } + + return 0; +} + +static int decode_end(AVCodecContext* pav_ctx) +{ + sanm_ctx* pctx = pav_ctx->priv_data; + int i; + + destroy_buffers(pctx); + + if (pctx->frame.data[0]){ + pav_ctx->release_buffer(pav_ctx, &pctx->frame); + pctx->frame.data[0] = 0; + } + + return 0; +} + +static int read_frame_header(sanm_ctx* pctx, const uint8_t* pinput, sanm_frame_header* pheader) +{ + pinput += 8; // skip pad + + pheader->width = AV_RL32(pinput); pinput += 4; + pheader->height = AV_RL32(pinput); pinput += 4;

Use bytestream_get_le32() and friends here and below.

+ if (pheader->width != pctx->width || pheader->height != pctx->height) + { + av_log(0, AV_LOG_ERROR, "sanm decoder: variable size frames are not implemented. video may be garbled until next keyframe.\n"); + return -1; + } + + pheader->seq_num = AV_RL16(pinput); pinput += 2; + pheader->codec = *pinput; pinput += 1; + pheader->rotate_code = *pinput; pinput += 1; + + pinput += 4; // skip pad + + pctx->psmall_codebook = (uint16_t*) pinput; pinput += 8; + pheader->bg_color = AV_RL16(pinput); pinput += 2; + + pinput += 2; // skip pad + + pheader->rle_output_size = AV_RL32(pinput); pinput += 4; + pctx->pcodebook = (uint16_t*) pinput; pinput += 512; // 256 entries in codebook + + pinput += 8; // skip pad + + pheader->pvstream = pinput; + + return 0; +} + +static void fill_db(uint16_t* pbuf, int buf_size, uint16_t color)

db?

+{ + while (buf_size--) + { + *pbuf++ = color; + } +}

Fill functions for various sizes is something we should implement in a common location. Just an observation.

+static void swap(uint16_t** ppleft, uint16_t** ppright) +{ + uint16_t* ptemp = *ppleft; + *ppleft = *ppright; + *ppright = ptemp; +}

FFSWAP()

+static void rotate_bufs(sanm_ctx* pctx, int rotate_code) +{ + if (1 == rotate_code) + { + swap(&pctx->pdb0, &pctx->pdb2); + } + else if (2 == rotate_code) + { + swap(&pctx->pdb1, &pctx->pdb2); + swap(&pctx->pdb2, &pctx->pdb0); + } +} + +static void codec0(sanm_ctx* pctx, const uint8_t* pinput) +{ + uint16_t* pdb0 = pctx->pdb0; + + int x, y; + for (y = 0; y != pctx->height; ++y) + { + for (x = 0; x != pctx->width; ++x) + { + *point_at(pdb0, x, y, pctx->pitch) = AV_RL16(pinput); pinput += 2; + } + } +} + +static void codec6(sanm_ctx* pctx, const uint8_t* pinput) +{ + int npixels = pctx->npixels; + uint16_t* pdb0 = pctx->pdb0; + uint16_t* pcodebook = pctx->pcodebook; + + int index; + while (npixels--) + { + index = *pinput++; + *pdb0++ = AV_RL16(&pcodebook[index]); + } +} + +static void copy_block(uint16_t* pdest, uint16_t* psrc, int block_size, int pitch) +{ + int y; + for (y = 0; y != block_size; ++y, pdest += pitch, psrc += pitch) + { + memcpy(pdest, psrc, block_size * sizeof(pdest[0])); + } +} + +static void fill_block(uint16_t* pdest, uint16_t color, int block_size, int pitch) +{ + int x, y; + pitch -= block_size; + for (y = 0; y != block_size; ++y, pdest += pitch) + { + for (x = 0; x != block_size; ++x) + { + *pdest++ = color; + } + } +}

These two functions also look rather generic. Perhaps candidates for sharing.

+static void draw_glyph(uint16_t* pdest, int index, uint16_t fg_color, uint16_t bg_color, int block_size, int pitch) +{ + int8_t* pglyph; + uint16_t colors[2] = { fg_color, bg_color }; + int x, y; + + if (index > NGLYPHS) + { + av_log(0, AV_LOG_ERROR, "sanm decoder: ignoring nonexistent glyph #%u.\n", index); + return; + } + + pglyph = (8 == block_size) ? p8x8glyphs[index] : p4x4glyphs[index]; + pitch -= block_size; + + for (y = 0; y != block_size; ++y, pdest += pitch) + { + for (x = 0; x != block_size; ++x) + { + *pdest++ = colors[*pglyph++];

What happens if input contains something other than 0 and 1?

+ } + } +} + +static void opcode_0xf7(sanm_ctx* pctx, int cx, int cy, int block_size, int pitch) +{ + if (2 == block_size) + { + uint16_t* pcodebook = pctx->pcodebook; + uint16_t* pdest = point_at(pctx->pdb0, cx, cy, pctx->pitch); + uint32_t indices = AV_RL32(pctx->psrc); pctx->psrc += 4; + + pdest[0] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8; + pdest[1] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8; + pdest[pitch] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8; + pdest[pitch + 1] = AV_RL16(&pcodebook[indices & 0xff]); + } + else + { + uint16_t fgcolor, bgcolor; + + int index = *pctx->psrc++; + uint16_t color_indices = AV_RL16(pctx->psrc); pctx->psrc += 2; + fgcolor = AV_RL16(&pctx->pcodebook[color_indices >> 8]); + bgcolor = AV_RL16(&pctx->pcodebook[color_indices & 0xff]); + + draw_glyph(point_at(pctx->pdb0, cx, cy, pctx->pitch), index, fgcolor, bgcolor, block_size, pitch); + } +} + +static void opcode_0xf8(sanm_ctx* pctx, int cx, int cy, int block_size, int pitch) +{ + if (2 == block_size) + { + uint16_t* pdest = point_at(pctx->pdb0, cx, cy, pctx->pitch); + pdest[0] = AV_RL16(pctx->psrc); pctx->psrc += 2; + pdest[1] = AV_RL16(pctx->psrc); pctx->psrc += 2; + pdest[pitch] = AV_RL16(pctx->psrc); pctx->psrc += 2; + pdest[pitch + 1] = AV_RL16(pctx->psrc); pctx->psrc += 2; + } + else + { + uint16_t fgcolor, bgcolor; + + int index = *pctx->psrc++; + uint32_t colors = AV_RL32(pctx->psrc); pctx->psrc += 4; + fgcolor = colors >> 16; + bgcolor = colors & 0xffff; + + draw_glyph(point_at(pctx->pdb0, cx, cy, pctx->pitch), index, fgcolor, bgcolor, block_size, pitch); + } +} + +static int good_mvec(sanm_ctx* pctx, int cx, int cy, int mx, int my, int block_size) +{ + int good = point_at(pctx->pdb2, cx + mx + block_size - 1, cy + my + block_size - 1, pctx->pitch) < (&pctx->pdb2[pctx->buf_size / 2]);

This sanity check looks rather incomplete.

+ if (!good) + { + av_log(0, AV_LOG_ERROR, "sanm decoder: ignoring segfault-inducing motion vector (%i, %i)->(%u, %u), block size = %u.\n", + cx + mx, cy + my, cx, cy, block_size); + } + + return good; +} + +static void codec2level(sanm_ctx* pctx, int cx, int cy, int block_size) +{ + int16_t mx, my, index; + uint16_t color; + + int opcode = *pctx->psrc++; + + switch (opcode) + { + default: + mx = motion_vectors[opcode][0]; + my = motion_vectors[opcode][1]; + + if (good_mvec(pctx, cx, cy, mx, my, block_size)) + { + copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), + point_at(pctx->pdb2, cx + mx, cy + my, pctx->pitch), + block_size, pctx->pitch); + } + break; + + case 0xf5: + index = AV_RL16(pctx->psrc); pctx->psrc += 2; + + mx = index % pctx->width; + my = index / pctx->width; + + if (good_mvec(pctx, cx, cy, mx, my, block_size)) + { + copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), + point_at(pctx->pdb2, cx + mx, cy + my, pctx->pitch), + block_size, pctx->pitch); + } + break; + + case 0xf6: + copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), + point_at(pctx->pdb1, cx, cy, pctx->pitch), + block_size, pctx->pitch); + break; + + case 0xf7: + opcode_0xf7(pctx, cx, cy, block_size, pctx->pitch); + break; + + case 0xf8: + opcode_0xf8(pctx, cx, cy, block_size, pctx->pitch); + break; + + case 0xf9: + case 0xfa: + case 0xfb: + case 0xfc: + color = AV_RL16(&pctx->psmall_codebook[opcode - 0xf9]); + fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch); + break; + + case 0xfd: + index = *pctx->psrc++; + color = AV_RL16(&pctx->pcodebook[index]); + fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch); + break; + + case 0xfe: + color = AV_RL16(pctx->psrc); pctx->psrc += 2; + fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch); + break; + + case 0xff: + if (2 == block_size) + { + opcode_0xf8(pctx, cx, cy, block_size, pctx->pitch); + } + else + { + block_size >>= 1; + codec2level(pctx, cx , cy , block_size); + codec2level(pctx, cx + block_size, cy , block_size); + codec2level(pctx, cx , cy + block_size, block_size); + codec2level(pctx, cx + block_size, cy + block_size, block_size); + } + break; + } +} + +static void rle_decode(uint8_t* pdest, const uint8_t* pinput, const int out_size) +{ + int opcode, color, run_len, remaining = out_size; + + assert(out_size > 0); + + while (remaining) + { + opcode = *pinput++; + run_len = (opcode >> 1) + 1; + assert(run_len <= remaining); + + if (opcode & 1) // rle strip + { + color = *pinput++; + memset(pdest, color, run_len); + } + else + { + memcpy(pdest, pinput, run_len); + pinput += run_len; + } + + pdest += run_len; + remaining -= run_len; + } +} + +static void codec5(sanm_ctx* pctx, const uint8_t* pinput, const int decoded_size) +{ + uint8_t* pdest = (uint8_t*) pctx->pdb0; + int npixels = pctx->npixels; + uint16_t* pdb0 = pctx->pdb0; + + assert(!(decode_size & 1)); + + rle_decode(pdest, pinput, decoded_size); + + while (npixels--) + { + *pdb0 = AV_RL16(pdb0); + ++pdb0; + } +}

Yet another candidate for a shared bswap16_buf().

+static void codec2(sanm_ctx* pctx, const uint8_t* pinput) +{ + const int width = pctx->aligned_width, height = pctx->aligned_height; + int cx, cy; + + pctx->psrc = pinput; + for (cy = 0; cy != height; cy += 8) + { + for (cx = 0; cx != width; cx += 8) + { + codec2level(pctx, cx, cy, 8); + } + } +} + +static void codec8(sanm_ctx* pctx, const uint8_t* pinput, const int decoded_size) +{ + uint16_t* pdest = pctx->pdb0; + uint8_t* pindices = av_malloc(decoded_size);

Unchecked malloc.

+ uint8_t* pidx_cursor = pindices; + long npixels = pctx->npixels; + + rle_decode(pindices, pinput, decoded_size); + + while (npixels--) + { + int index = *pidx_cursor++; + *pdest++ = AV_RL16(&pctx->pcodebook[index]); + } + + av_free(pindices); +}

I have a feeling this could be done without the temp buffer.

+static void copy_output(sanm_ctx* pctx, sanm_frame_header* pheader) +{ + uint8_t* poutput; + const uint8_t* pinput = (uint8_t*) pctx->pdb0; + + int height = pctx->height; + long inpitch = pctx->pitch * (pheader ? sizeof(pctx->pdb0[0]) : 1); + long outpitch; + + if (pctx->avctx->get_buffer(pctx->avctx, pctx->output)) + { + av_log(pctx->avctx, AV_LOG_ERROR, "sanm decoder: get_buffer() failed.\n"); + return; + } + poutput = pctx->output->data[0]; + outpitch = pctx->output->linesize[0]; + while (height--) + { + memcpy(poutput, pinput, inpitch); + pinput += inpitch; + poutput += outpitch; + } +}

Why not decode directly into the get_buffer() buffers?

+static void codec1(sanm_ctx* pctx, const uint8_t* pinput, int top, int left, int width, int height) +{ + uint8_t *dst = ((uint8_t*)pctx->pdb0) + left + top * pctx->pitch; + const uint8_t* end; + int i, j, len, flag, code, val, pos; + for(i = 0; i < height; i++){ + pos = 0; + len = bytestream_get_le16(&pinput); + end = pinput + len; + while(pinput < end){ + code = bytestream_get_byte(&pinput); + flag = code & 1; + code = (code >> 1) + 1; + if(flag){ + val = bytestream_get_byte(&pinput); + if(val) + memset(dst + pos, val, code); + pos += code; + }else{ + for(j = 0; j < code; j++){ + val = bytestream_get_byte(&pinput); + if(val) + dst[pos] = val; + pos++; + } + }

This is missing output buffer bounds checks.

+ } + dst += pctx->pitch; + } + pctx->rotate_code = 0; +} +

[...]

+static void codec37(sanm_ctx* pctx, const uint8_t* src, int top, int left, int width, int height) +{ + int stride = pctx->pitch; + uint8_t *dst; + uint8_t *prev; + uint8_t *buf; + int compr = src[0]; + int mvoff = src[1]; + int seq = src[2]; + int i, j, k, t; + int flags = src[12]; + int skip_run = 0; + + pctx->rotate_code = 0; + if(((seq & 1) || !(flags & 1)) && (compr >= 3)) + rotate_bufs(pctx, 1); + dst = ((uint8_t*)pctx->pdb0) + left + top * stride; + prev = pctx->pdb2; + src += 16; + switch(compr){ + case 0: + for(i = 0; i < height; i++){ + memcpy(dst, src, width); + src += width; + dst += stride; + } + memset(pctx->pdb1, 0, pctx->height * stride); + memset(pctx->pdb2, 0, pctx->height * stride); + break; + case 1: + av_log(NULL,0,"Codec 37 compr 1: TODO\n");

av_log_missing_feature() here and elsewhere.

+ break; + case 2: + t = width * height; + buf = av_malloc(t+16); + rle_decode(buf, src, t); + src = buf; + for(j = 0; j < height; j++){ + for(i = 0; i < width; i++) + memcpy(dst, src, width); + dst += stride; + src += width; + } + memset(pctx->pdb1, 0, pctx->height * stride); + memset(pctx->pdb2, 0, pctx->height * stride); + av_free(buf); + break; + case 3: + case 4: + if(flags & 4){ + for(j = 0; j < height; j += 4){ + for(i = 0; i < width; i += 4){ + int code; + if(skip_run){ + skip_run--; + for(k = 0; k < 4; k++) + memcpy(dst + i + k*stride, prev + i + k*stride, 4); + continue; + } + code = bytestream_get_byte(&src); + switch(code){ + case 0xFF: + for(k = 0; k < 4; k++) + bytestream_get_buffer(&src, dst + i + k*stride, 4); + break; + case 0xFE: + for(k = 0; k < 4; k++) + memset(dst + i + k*stride, bytestream_get_byte(&src), 4); + break; + case 0xFD: + t = bytestream_get_byte(&src); + for(k = 0; k < 4; k++) + memset(dst + i + k*stride, t, 4); + break; + default: + if(compr == 4 && !code){ + skip_run = bytestream_get_byte(&src) + 1; + i -= 4; + }else{ + int mx, my; + mx = c37_mv[(mvoff*255 + code)*2]; + my = c37_mv[(mvoff*255 + code)*2+1]; + for(k = 0; k < 4; k++) + memcpy(dst + i + k*stride, prev + i + mx + (k + my)*stride, 4); + } + } + } + dst += stride * 4; + prev += stride * 4; + } + }else{ + for(j = 0; j < height; j += 4){ + for(i = 0; i < width; i += 4){ + int code; + if(skip_run){ + skip_run--; + for(k = 0; k < 4; k++) + memcpy(dst + i + k*stride, prev + i + k*stride, 4); + continue; + } + code = bytestream_get_byte(&src); + if(code == 0xFF){ + for(k = 0; k < 4; k++) + bytestream_get_buffer(&src, dst + i + k*stride, 4); + }else if(compr == 4 && !code){ + skip_run = bytestream_get_byte(&src) + 1; + i -= 4; + }else{ + int mx, my; + mx = c37_mv[(mvoff*255 + code)*2]; + my = c37_mv[(mvoff*255 + code)*2+1]; + if(my + j >= 0) + for(k = 0; k < 4; k++) + memcpy(dst + i + k*stride, prev + i + mx + (k + my)*stride, 4); + } + } + dst += stride * 4; + prev += stride * 4; + } + } + break; + } +} + +static void process_block(uint8_t *dst, uint8_t *prev1, uint8_t *prev2, int stride, const uint8_t **src, const uint8_t *tbl, int size) +{ + int code; + int k, t; + + code = bytestream_get_byte(src); + if(code >= 0xF8){ + switch(code){ + case 0xFF: + if(size == 2){ + dst[0] = bytestream_get_byte(src); + dst[1] = bytestream_get_byte(src); + dst[0+stride] = bytestream_get_byte(src); + dst[1+stride] = bytestream_get_byte(src); + }else{ + size >>= 1; + process_block(dst, prev1, prev2, stride, src, tbl, size); + process_block(dst+size, prev1+size, prev2+size, stride, src, tbl, size); + dst += size * stride; + prev1 += size * stride; + prev2 += size * stride; + process_block(dst, prev1, prev2, stride, src, tbl, size); + process_block(dst+size, prev1+size, prev2+size, stride, src, tbl, size); + } + break; + case 0xFE: + t = bytestream_get_byte(src); + for(k = 0; k < size; k++) + memset(dst + k*stride, t, size); + break; + case 0xFD: + { + int code = bytestream_get_byte(src); + int8_t *pglyph = (size == 8) ? p8x8glyphs[code] : p4x4glyphs[code]; + + for(k = 0; k < size; k++) + for(t = 0; t < size; t++) + dst[t + k*stride] = src[0][!(*pglyph++)]; + *src += 2; + } + break; + case 0xFC: + for(k = 0; k < size; k++) + memcpy(dst + k*stride, prev1 + k*stride, size); + break; + default: + t = tbl[code & 7]; + for(k = 0; k < size; k++) + memset(dst + k*stride, t, size); + } + }else{ + int mx = motion_vectors[code][0]; + int my = motion_vectors[code][1]; + for(k = 0; k < size; k++) + memcpy(dst + k*stride, prev2 + mx + (my+k)*stride, size); + } +} + +static void codec47(sanm_ctx* pctx, const uint8_t* pinput, int top, int left, int width, int height) +{ + int compr = pinput[2]; + const uint8_t *src = pinput + 26; + int stride = pctx->pitch; + uint8_t *dst = pctx->pdb0 + left + top * stride; + uint8_t *prev1 = pctx->pdb1; + uint8_t *prev2 = pctx->pdb2; + int i, j, t; + uint8_t *buf; + int seq = AV_RL16(pinput); + if(pinput[4] & 1) + src += 0x8080; + if(!seq) + pctx->prev_seq = -1; + switch(compr){ + case 0: + for(j = 0; j < height; j++){ + for(i = 0; i < width; i++) + memcpy(dst, src, width); + dst += stride; + src += width; + } + break; + case 1: + av_log(NULL,0,"Codec 47 compr 1: TODO\n"); + break; + case 2: + if(seq == pctx->prev_seq + 1){ + for(j = 0; j < height; j += 8){ + for(i = 0; i < width; i += 8){ + process_block(dst + i, prev1 + i, prev2 + i, stride, &src, pinput + 8, 8); + } + dst += stride * 8; + prev1 += stride * 8; + prev2 += stride * 8; + } + } + break; + case 3: + memcpy(pctx->pdb0, pctx->pdb2, pctx->pitch * pctx->height); + break; + case 4: + memcpy(pctx->pdb0, pctx->pdb1, pctx->pitch * pctx->height); + break; + case 5: + t = AV_RL32(pinput + 14); + buf = av_malloc(t+16);

Unchecked malloc and with size provided by input to boot.

+ rle_decode(buf, src, t); + src = buf; + for(j = 0; j < height; j++){ + for(i = 0; i < width; i++) + memcpy(dst, src, width); + dst += stride; + src += width; + } + av_free(buf); + break; + } + if(seq == pctx->prev_seq + 1) + pctx->rotate_code = pinput[3]; + else + pctx->rotate_code = 0; + pctx->prev_seq = seq; +} + + +static void process_npal(sanm_ctx* pctx, const uint8_t* pinput) +{ + int i; + for(i = 0; i < 256; i++) + pctx->pal[i] = bytestream_get_be24(&pinput); +}

I'm sure I've seen this pattern somewhere before.

+static void process_xpal(sanm_ctx* pctx, const uint8_t* pinput, int size) +{ + int i, j; + + if(size == 6){ + for(i = 0; i < 256; i++){ + uint8_t tmp[4]; + for(j = 0; j < 3; j++){ + int t = (pctx->pal[i] >> (16 - j*8)) & 0xFF; + tmp[2-j] = av_clip_uint8((t*129 + pctx->delta_pal[i*3+j]) / 128); + } + pctx->pal[i] = AV_RL24(tmp); + } + }else{ + pinput += 4; + for(i = 0; i < 768; i++) + pctx->delta_pal[i] = bytestream_get_le16(&pinput); + for(i = 0; i < 256; i++) + pctx->pal[i] = bytestream_get_be24(&pinput); + } +} + +static void process_fobj(sanm_ctx* pctx, const uint8_t* pinput) +{ + int i; + int codec, top, left, w, h; + codec = bytestream_get_le16(&pinput); + top = bytestream_get_le16(&pinput); + left = bytestream_get_le16(&pinput); + w = bytestream_get_le16(&pinput); + h = bytestream_get_le16(&pinput); + if(pctx->width < left + w || pctx->height < top + h){ + pctx->avctx->width = left + w; + pctx->avctx->height = top + h; + init_sizes(pctx, left + w, top + h); + init_buffers(pctx); + } + pinput += 4; + switch(codec){ + case 1: + codec1(pctx, pinput, top, left, w, h); + break; + case 37: + codec37(pctx, pinput, top, left, w, h); + break; + case 47: + codec47(pctx, pinput, top, left, w, h); + break; + default: + av_log(NULL,0,"Unknown codec %d\n",codec); + } +} + +static int dispatch_codec1(sanm_ctx* pctx, const uint8_t* pinput, const int isize) +{ + const uint8_t *frame_end = pinput + isize; + while(pinput < frame_end - 8){ + int sig = AV_RB32(pinput); + int size = AV_RB32(pinput+4); + pinput += 8; + if(size < 0 || pinput + size > frame_end){ + av_log(NULL,0,"Incorrect size (%d|%X, %d)\n",size,size,frame_end-pinput); + break; + } + switch(sig){ + case MKBETAG('N', 'P', 'A', 'L'): + process_npal(pctx, pinput); + break; + case MKBETAG('F', 'O', 'B', 'J'): + process_fobj(pctx, pinput); + break; + case MKBETAG('X', 'P', 'A', 'L'): + process_xpal(pctx, pinput, size); + break; + } + + pinput += size; + if(size & 1) + pinput++; + } + copy_output(pctx, NULL); + rotate_bufs(pctx, pctx->rotate_code); + memcpy(pctx->output->data[1], pctx->pal, 1024); + return 0; +} + +static int dispatch_codec(sanm_ctx* pctx, const uint8_t* pinput) +{ + sanm_frame_header header; + if (read_frame_header(pctx, pinput, &header)) + { + return -1; + } + + if ((pctx->output->key_frame = !header.seq_num)) + { + pctx->output->pict_type = FF_I_TYPE; + fill_db(pctx->pdb1, pctx->npixels, header.bg_color); + fill_db(pctx->pdb2, pctx->npixels, header.bg_color); + } + else + { + pctx->output->pict_type = FF_P_TYPE; + } + + switch (header.codec) + { + case 0: + codec0(pctx, header.pvstream); + break; + + case 2: + codec2(pctx, header.pvstream); + break; + + case 3: + memcpy(pctx->pdb0, pctx->pdb2, pctx->buf_size); + break; + + case 4: + memcpy(pctx->pdb0, pctx->pdb1, pctx->buf_size); + break; + + case 5: + codec5(pctx, header.pvstream, header.rle_output_size); + break; + + case 6: + codec6(pctx, header.pvstream); + break; + + case 8: + codec8(pctx, header.pvstream, pctx->npixels); + break; + + default: + av_log(0, AV_LOG_ERROR, "sanm decoder: subcodec %u is not implemented. video may be garbled until next keyframe.\n", header.codec); + break; + }

Perhaps a table of function pointers would be better.

+ copy_output(pctx, &header); + rotate_bufs(pctx, header.rotate_code); + + return 0; +} + +static int decode_frame(AVCodecContext* pav_ctx, void* poutput, int* poutput_size, uint8_t* pinput, int input_size) +{ + sanm_ctx* pctx = pav_ctx->priv_data; + + if (pctx->output->data[0]) + { + pav_ctx->release_buffer(pav_ctx, pctx->output); + } + + if (!pctx->version && dispatch_codec1(pctx, pinput, input_size)) + { + return -1; + } + + if (pctx->version && dispatch_codec(pctx, pinput)) + { + return -1; + } + + *poutput_size = sizeof(AVFrame); + *((AVFrame*) poutput) = *pctx->output; + + return input_size; +} + +AVCodec sanm_decoder = +{ + "sanm", + CODEC_TYPE_VIDEO, + CODEC_ID_SANM, + sizeof(sanm_ctx), + decode_init, + 0, + decode_end, + decode_frame +};

This decoder is lacking error/sanity checks in many places. I suspect a fuzz attack would make it fall over rather quickly. Needs more work. Leaving the demuxer for later. -- M?ns Rullg?rd mans at mansr.com

On Sun, Jan 23, 2011 at 03:50:04PM +0100, Vitor Sessak wrote:

Git-friendly patch attached. Hope that patchwork will catch it up.

--- /dev/null +++ b/libavcodec/sanm.c @@ -0,0 +1,1381 @@ +/* + * LucasArts Smush v2 decoder + * Copyright (c) 2006 Cyril Zorin + * <firstname dot lastname at gmail dot com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */

This is not our standard license header.

+/** + * @file sanm.c + * LucasArts Smush v2 decoder + */

Filenames are unnecessary and evil with @file directives.

+#include "avcodec.h" +#include "bytestream.h"

More than these are necessary I think, at the very least a few system headers like stdint.h.

+typedef struct sanm_ctx +{ + AVCodecContext *avctx; + const uint8_t* psrc; + + int version, subversion; + uint32_t pal[256]; + int16_t delta_pal[768]; + + int pitch; + int width, height; + int aligned_width, aligned_height; + int prev_seq; + + AVFrame frame, *output; + uint16_t* pdb0, * pdb1, * pdb2; + int rotate_code; + + long npixels, buf_size; + + uint16_t* pcodebook; + uint16_t* psmall_codebook; +} sanm_ctx;

inconsistent * placement This whole file could use some K&R love. I'd also skip all those typedefs. same for the demuxer

--- /dev/null +++ b/libavformat/sanm.c @@ -0,0 +1,395 @@ +/* + * LucasArts Smush v2 demuxer + * Copyright (c) 2006 Cyril Zorin + * <firstname dot lastname at gmail dot com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + * + */

see above

+/** + * @file sanm.c + * LucasArts Smush v2 demuxer + */

ditto