[FFmpeg-devel] [PATCH]Avoid an assertion failure in ff_init_vlc_sparse

Carl Eugen Hoyos cehoyos at ag.or.at
Fri Jul 12 21:13:43 CEST 2013


Hi!

Attached patch fixes an assertion failure on oom while initializing tables, 
an example is in ticket #2724.

Please comment, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/bitstream.c b/libavcodec/bitstream.c
index bbd9491..70801a5 100644
--- a/libavcodec/bitstream.c
+++ b/libavcodec/bitstream.c
@@ -282,7 +282,8 @@ int ff_init_vlc_sparse(VLC *vlc, int nb_bits, int nb_codes,
                                  codes, codes_wrap, codes_size,
                                  symbols, symbols_wrap, symbols_size,
                                  flags & ~INIT_VLC_USE_NEW_STATIC);
-        av_assert0(ret >= 0);
+        if (ret < 0)
+            return ret;
         av_assert0(dyn_vlc.table_size <= vlc->table_allocated);
         if (dyn_vlc.table_size < vlc->table_allocated)
             av_log(NULL, AV_LOG_ERROR, "needed %d had %d\n", dyn_vlc.table_size, vlc->table_allocated);
diff --git a/libavcodec/faxcompr.c b/libavcodec/faxcompr.c
index 900851b..9fc6651 100644
--- a/libavcodec/faxcompr.c
+++ b/libavcodec/faxcompr.c
@@ -96,30 +96,33 @@ static const uint8_t ccitt_group3_2d_lens[11] = {
 
 static VLC ccitt_vlc[2], ccitt_group3_2d_vlc;
 
-av_cold void ff_ccitt_unpack_init(void)
+av_cold int ff_ccitt_unpack_init(void)
 {
     static VLC_TYPE code_table1[528][2];
     static VLC_TYPE code_table2[648][2];
-    int i;
+    int i, ret;
     static int initialized = 0;
 
     if (initialized)
-        return;
+        return 0;
     ccitt_vlc[0].table = code_table1;
     ccitt_vlc[0].table_allocated = 528;
     ccitt_vlc[1].table = code_table2;
     ccitt_vlc[1].table_allocated = 648;
     for (i = 0; i < 2; i++) {
-        ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
+        ret = ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
                            ccitt_codes_lens[i], 1, 1,
                            ccitt_codes_bits[i], 1, 1,
                            ccitt_syms, 2, 2,
                            INIT_VLC_USE_NEW_STATIC);
+        if (ret < 0)
+            return ret;
     }
     INIT_VLC_STATIC(&ccitt_group3_2d_vlc, 9, 11,
                     ccitt_group3_2d_lens, 1, 1,
                     ccitt_group3_2d_bits, 1, 1, 512);
     initialized = 1;
+    return 0;
 }
 
 
diff --git a/libavcodec/faxcompr.h b/libavcodec/faxcompr.h
index 53d1168..18f96dd 100644
--- a/libavcodec/faxcompr.h
+++ b/libavcodec/faxcompr.h
@@ -33,7 +33,7 @@
 /**
  * initialize upacker code
  */
-void ff_ccitt_unpack_init(void);
+int ff_ccitt_unpack_init(void);
 
 /**
  * unpack data compressed with CCITT Group 3 1/2-D or Group 4 method
diff --git a/libavcodec/mpc8.c b/libavcodec/mpc8.c
index 4d6ca89..239de61 100644
--- a/libavcodec/mpc8.c
+++ b/libavcodec/mpc8.c
@@ -96,7 +96,7 @@ static const uint16_t vlc_offsets[13] = {
 
 static av_cold int mpc8_decode_init(AVCodecContext * avctx)
 {
-    int i;
+    int i, ret;
     MPCContext *c = avctx->priv_data;
     GetBitContext gb;
     static int vlc_initialized = 0;
@@ -188,16 +188,20 @@ static av_cold int mpc8_decode_init(AVCodecContext * avctx)
 
     q3_vlc[0].table = q3_0_table;
     q3_vlc[0].table_allocated = 512;
-    ff_init_vlc_sparse(&q3_vlc[0], MPC8_Q3_BITS, MPC8_Q3_SIZE,
+    ret = ff_init_vlc_sparse(&q3_vlc[0], MPC8_Q3_BITS, MPC8_Q3_SIZE,
              mpc8_q3_bits,  1, 1,
              mpc8_q3_codes, 1, 1,
              mpc8_q3_syms,  1, 1, INIT_VLC_USE_NEW_STATIC);
+    if (ret < 0)
+        return ret;
     q3_vlc[1].table = q3_1_table;
     q3_vlc[1].table_allocated = 516;
-    ff_init_vlc_sparse(&q3_vlc[1], MPC8_Q4_BITS, MPC8_Q4_SIZE,
+    ret = ff_init_vlc_sparse(&q3_vlc[1], MPC8_Q4_BITS, MPC8_Q4_SIZE,
              mpc8_q4_bits,  1, 1,
              mpc8_q4_codes, 1, 1,
              mpc8_q4_syms,  1, 1, INIT_VLC_USE_NEW_STATIC);
+    if (ret < 0)
+        return ret;
 
     for(i = 0; i < 2; i++){
         res_vlc[i].table = &codes_table[vlc_offsets[0+i]];
diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
index 9b47d2e..f3fae7a 100644
--- a/libavcodec/rv34.c
+++ b/libavcodec/rv34.c
@@ -103,10 +103,10 @@ static VLC_TYPE table_data[117592][2];
  * @param insyms symbols for input codes (NULL for default ones)
  * @param num    VLC table number (for static initialization)
  */
-static void rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t *insyms,
+static int rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t *insyms,
                          const int num)
 {
-    int i;
+    int i, ret;
     int counts[17] = {0}, codes[17];
     uint16_t cw[MAX_VLC_SIZE], syms[MAX_VLC_SIZE];
     uint8_t bits2[MAX_VLC_SIZE];
@@ -130,46 +130,74 @@ static void rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t
 
     vlc->table = &table_data[table_offs[num]];
     vlc->table_allocated = table_offs[num + 1] - table_offs[num];
-    ff_init_vlc_sparse(vlc, FFMIN(maxbits, 9), realsize,
+    ret = ff_init_vlc_sparse(vlc, FFMIN(maxbits, 9), realsize,
                        bits2, 1, 1,
                        cw,    2, 2,
                        syms,  2, 2, INIT_VLC_USE_NEW_STATIC);
+    if (ret < 0)
+        return ret;
+    return 0;
 }
 
 /**
  * Initialize all tables.
  */
-static av_cold void rv34_init_tables(void)
+static av_cold int rv34_init_tables(void)
 {
-    int i, j, k;
+    int i, j, k, ret;
 
     for(i = 0; i < NUM_INTRA_TABLES; i++){
         for(j = 0; j < 2; j++){
-            rv34_gen_vlc(rv34_table_intra_cbppat   [i][j], CBPPAT_VLC_SIZE,   &intra_vlcs[i].cbppattern[j],     NULL, 19*i + 0 + j);
-            rv34_gen_vlc(rv34_table_intra_secondpat[i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].second_pattern[j], NULL, 19*i + 2 + j);
-            rv34_gen_vlc(rv34_table_intra_thirdpat [i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].third_pattern[j],  NULL, 19*i + 4 + j);
+            ret = rv34_gen_vlc(rv34_table_intra_cbppat   [i][j], CBPPAT_VLC_SIZE,   &intra_vlcs[i].cbppattern[j],     NULL, 19*i + 0 + j);
+            if (ret < 0)
+                return ret;
+            ret = rv34_gen_vlc(rv34_table_intra_secondpat[i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].second_pattern[j], NULL, 19*i + 2 + j);
+            if (ret < 0)
+                return ret;
+            ret = rv34_gen_vlc(rv34_table_intra_thirdpat [i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].third_pattern[j],  NULL, 19*i + 4 + j);
+            if (ret < 0)
+                return ret;
             for(k = 0; k < 4; k++){
-                rv34_gen_vlc(rv34_table_intra_cbp[i][j+k*2],  CBP_VLC_SIZE,   &intra_vlcs[i].cbp[j][k],         rv34_cbp_code, 19*i + 6 + j*4 + k);
+                ret = rv34_gen_vlc(rv34_table_intra_cbp[i][j+k*2],  CBP_VLC_SIZE,   &intra_vlcs[i].cbp[j][k],         rv34_cbp_code, 19*i + 6 + j*4 + k);
+                if (ret < 0)
+                    return ret;
             }
         }
         for(j = 0; j < 4; j++){
-            rv34_gen_vlc(rv34_table_intra_firstpat[i][j], FIRSTBLK_VLC_SIZE, &intra_vlcs[i].first_pattern[j], NULL, 19*i + 14 + j);
+            ret = rv34_gen_vlc(rv34_table_intra_firstpat[i][j], FIRSTBLK_VLC_SIZE, &intra_vlcs[i].first_pattern[j], NULL, 19*i + 14 + j);
+            if (ret < 0)
+                return ret;
         }
-        rv34_gen_vlc(rv34_intra_coeff[i], COEFF_VLC_SIZE, &intra_vlcs[i].coefficient, NULL, 19*i + 18);
+        ret = rv34_gen_vlc(rv34_intra_coeff[i], COEFF_VLC_SIZE, &intra_vlcs[i].coefficient, NULL, 19*i + 18);
+        if (ret < 0)
+            return ret;
     }
 
     for(i = 0; i < NUM_INTER_TABLES; i++){
-        rv34_gen_vlc(rv34_inter_cbppat[i], CBPPAT_VLC_SIZE, &inter_vlcs[i].cbppattern[0], NULL, i*12 + 95);
+        ret = rv34_gen_vlc(rv34_inter_cbppat[i], CBPPAT_VLC_SIZE, &inter_vlcs[i].cbppattern[0], NULL, i*12 + 95);
+            if (ret < 0)
+                return ret;
         for(j = 0; j < 4; j++){
-            rv34_gen_vlc(rv34_inter_cbp[i][j], CBP_VLC_SIZE, &inter_vlcs[i].cbp[0][j], rv34_cbp_code, i*12 + 96 + j);
+            ret = rv34_gen_vlc(rv34_inter_cbp[i][j], CBP_VLC_SIZE, &inter_vlcs[i].cbp[0][j], rv34_cbp_code, i*12 + 96 + j);
+                if (ret < 0)
+                    return ret;
         }
         for(j = 0; j < 2; j++){
-            rv34_gen_vlc(rv34_table_inter_firstpat [i][j], FIRSTBLK_VLC_SIZE, &inter_vlcs[i].first_pattern[j],  NULL, i*12 + 100 + j);
-            rv34_gen_vlc(rv34_table_inter_secondpat[i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].second_pattern[j], NULL, i*12 + 102 + j);
-            rv34_gen_vlc(rv34_table_inter_thirdpat [i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].third_pattern[j],  NULL, i*12 + 104 + j);
+            ret = rv34_gen_vlc(rv34_table_inter_firstpat [i][j], FIRSTBLK_VLC_SIZE, &inter_vlcs[i].first_pattern[j],  NULL, i*12 + 100 + j);
+            if (ret < 0)
+                return ret;
+            ret = rv34_gen_vlc(rv34_table_inter_secondpat[i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].second_pattern[j], NULL, i*12 + 102 + j);
+            if (ret < 0)
+                return ret;
+            ret = rv34_gen_vlc(rv34_table_inter_thirdpat [i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].third_pattern[j],  NULL, i*12 + 104 + j);
+            if (ret < 0)
+                return ret;
         }
-        rv34_gen_vlc(rv34_inter_coeff[i], COEFF_VLC_SIZE, &inter_vlcs[i].coefficient, NULL, i*12 + 106);
+        ret = rv34_gen_vlc(rv34_inter_coeff[i], COEFF_VLC_SIZE, &inter_vlcs[i].coefficient, NULL, i*12 + 106);
+        if (ret < 0)
+            return ret;
     }
+    return 0;
 }
 
 /** @} */ // vlc group
@@ -1503,8 +1531,11 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx)
     if ((ret = rv34_decoder_alloc(r)) < 0)
         return ret;
 
-    if(!intra_vlcs[0].cbppattern[0].bits)
-        rv34_init_tables();
+    if(!intra_vlcs[0].cbppattern[0].bits) {
+        ret = rv34_init_tables();
+        if (ret < 0)
+            return ret;
+    }
 
     avctx->internal->allocate_progress = 1;
 
diff --git a/libavcodec/rv40.c b/libavcodec/rv40.c
index 17574ee..edf4f29 100644
--- a/libavcodec/rv40.c
+++ b/libavcodec/rv40.c
@@ -46,9 +46,9 @@ static const int16_t mode2_offs[] = {
 /**
  * Initialize all tables.
  */
-static av_cold void rv40_init_tables(void)
+static av_cold int rv40_init_tables(void)
 {
-    int i;
+    int i, ret;
     static VLC_TYPE aic_table[1 << AIC_TOP_BITS][2];
     static VLC_TYPE aic_mode1_table[AIC_MODE1_NUM << AIC_MODE1_BITS][2];
     static VLC_TYPE aic_mode2_table[11814][2];
@@ -79,19 +79,24 @@ static av_cold void rv40_init_tables(void)
     for(i = 0; i < NUM_PTYPE_VLCS; i++){
         ptype_vlc[i].table = &ptype_table[i << PTYPE_VLC_BITS];
         ptype_vlc[i].table_allocated = 1 << PTYPE_VLC_BITS;
-        ff_init_vlc_sparse(&ptype_vlc[i], PTYPE_VLC_BITS, PTYPE_VLC_SIZE,
+        ret = ff_init_vlc_sparse(&ptype_vlc[i], PTYPE_VLC_BITS, PTYPE_VLC_SIZE,
                             ptype_vlc_bits[i],  1, 1,
                             ptype_vlc_codes[i], 1, 1,
                             ptype_vlc_syms,     1, 1, INIT_VLC_USE_NEW_STATIC);
+        if (ret < 0)
+            return ret;
     }
     for(i = 0; i < NUM_BTYPE_VLCS; i++){
         btype_vlc[i].table = &btype_table[i << BTYPE_VLC_BITS];
         btype_vlc[i].table_allocated = 1 << BTYPE_VLC_BITS;
-        ff_init_vlc_sparse(&btype_vlc[i], BTYPE_VLC_BITS, BTYPE_VLC_SIZE,
+        ret = ff_init_vlc_sparse(&btype_vlc[i], BTYPE_VLC_BITS, BTYPE_VLC_SIZE,
                             btype_vlc_bits[i],  1, 1,
                             btype_vlc_codes[i], 1, 1,
                             btype_vlc_syms,     1, 1, INIT_VLC_USE_NEW_STATIC);
+        if (ret < 0)
+            return ret;
     }
+    return 0;
 }
 
 /**
@@ -554,7 +559,9 @@ static av_cold int rv40_decode_init(AVCodecContext *avctx)
     if (ret < 0)
         return ret;
     if(!aic_top_vlc.bits)
-        rv40_init_tables();
+        ret = rv40_init_tables();
+    if (ret < 0)
+        return ret;
     r->parse_slice_header = rv40_parse_slice_header;
     r->decode_intra_types = rv40_decode_intra_types;
     r->decode_mb_info     = rv40_decode_mb_info;
diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c
index fdfa8f2..754c84f 100644
--- a/libavcodec/tiff.c
+++ b/libavcodec/tiff.c
@@ -1259,14 +1259,15 @@ static int decode_frame(AVCodecContext *avctx,
 static av_cold int tiff_init(AVCodecContext *avctx)
 {
     TiffContext *s = avctx->priv_data;
+    int ret;
 
     s->width  = 0;
     s->height = 0;
     s->avctx  = avctx;
     ff_lzw_decode_open(&s->lzw);
-    ff_ccitt_unpack_init();
+    ret = ff_ccitt_unpack_init();
 
-    return 0;
+    return ret;
 }
 
 static av_cold int tiff_end(AVCodecContext *avctx)


More information about the ffmpeg-devel mailing list