[FFmpeg-cvslog] r22076 - in branches/0.5: . libavcodec/vorbis.c libavcodec/vp3.c

[siretart]

Author: siretart
Date: Fri Feb 26 15:32:27 2010
New Revision: 22076

Log:
fix the remaining ogv segfaults from issue 1240. 

First commit:

Make decode_init fail if the huffman tables are invalid and thus init_vlc fails.
Otherwise this will crash during decoding because the vlc tables are NULL.
Partially fixes ogv/smclock.ogv.1.101.ogv from issue 1240.

backport r19355 by reimar

Second commit:

Add extra validation checks to ff_vorbis_len2vlc.
They should not be necessary, but it seems like a reasonable precaution.

r19374 by reimar

Modified:
   branches/0.5/   (props changed)
   branches/0.5/libavcodec/vorbis.c
   branches/0.5/libavcodec/vp3.c

Modified: branches/0.5/libavcodec/vorbis.c
==============================================================================
--- branches/0.5/libavcodec/vorbis.c	Fri Feb 26 12:51:17 2010	(r22075)
+++ branches/0.5/libavcodec/vorbis.c	Fri Feb 26 15:32:27 2010	(r22076)
@@ -45,6 +45,9 @@ unsigned int ff_vorbis_nth_root(unsigned
 
 // Generate vlc codes from vorbis huffman code lengths
 
+// the two bits[p] > 32 checks should be redundant, all calling code should
+// already ensure that, but since it allows overwriting the stack it seems
+// reasonable to check redundantly.
 int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, uint_fast32_t num) {
     uint_fast32_t exit_at_level[33]={404,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
@@ -63,6 +66,7 @@ int ff_vorbis_len2vlc(uint8_t *bits, uin
     }
 
     codes[p]=0;
+    if (bits[p] > 32) return 1;
     for(i=0;i<bits[p];++i) {
         exit_at_level[i+1]=1<<i;
     }
@@ -79,6 +83,7 @@ int ff_vorbis_len2vlc(uint8_t *bits, uin
     ++p;
 
     for(;p<num;++p) {
+        if (bits[p] > 32) return 1;
         if (bits[p]==0) continue;
         // find corresponding exit(node which the tree can grow further from)
         for(i=bits[p];i>0;--i) {

Modified: branches/0.5/libavcodec/vp3.c
==============================================================================
--- branches/0.5/libavcodec/vp3.c	Fri Feb 26 12:51:17 2010	(r22075)
+++ branches/0.5/libavcodec/vp3.c	Fri Feb 26 15:32:27 2010	(r22076)
@@ -1744,29 +1744,34 @@ static av_cold int vp3_decode_init(AVCod
         for (i = 0; i < 16; i++) {
 
             /* DC histograms */
-            init_vlc(&s->dc_vlc[i], 5, 32,
+            if (init_vlc(&s->dc_vlc[i], 5, 32,
                 &s->huffman_table[i][0][1], 4, 2,
-                &s->huffman_table[i][0][0], 4, 2, 0);
+                &s->huffman_table[i][0][0], 4, 2, 0) < 0)
+                goto vlc_fail;
 
             /* group 1 AC histograms */
-            init_vlc(&s->ac_vlc_1[i], 5, 32,
+            if (init_vlc(&s->ac_vlc_1[i], 5, 32,
                 &s->huffman_table[i+16][0][1], 4, 2,
-                &s->huffman_table[i+16][0][0], 4, 2, 0);
+                &s->huffman_table[i+16][0][0], 4, 2, 0) < 0)
+                goto vlc_fail;
 
             /* group 2 AC histograms */
-            init_vlc(&s->ac_vlc_2[i], 5, 32,
+            if (init_vlc(&s->ac_vlc_2[i], 5, 32,
                 &s->huffman_table[i+16*2][0][1], 4, 2,
-                &s->huffman_table[i+16*2][0][0], 4, 2, 0);
+                &s->huffman_table[i+16*2][0][0], 4, 2, 0) < 0)
+                goto vlc_fail;
 
             /* group 3 AC histograms */
-            init_vlc(&s->ac_vlc_3[i], 5, 32,
+            if (init_vlc(&s->ac_vlc_3[i], 5, 32,
                 &s->huffman_table[i+16*3][0][1], 4, 2,
-                &s->huffman_table[i+16*3][0][0], 4, 2, 0);
+                &s->huffman_table[i+16*3][0][0], 4, 2, 0) < 0)
+                goto vlc_fail;
 
             /* group 4 AC histograms */
-            init_vlc(&s->ac_vlc_4[i], 5, 32,
+            if (init_vlc(&s->ac_vlc_4[i], 5, 32,
                 &s->huffman_table[i+16*4][0][1], 4, 2,
-                &s->huffman_table[i+16*4][0][0], 4, 2, 0);
+                &s->huffman_table[i+16*4][0][0], 4, 2, 0) < 0)
+                goto vlc_fail;
         }
     }
 
@@ -1805,6 +1810,10 @@ static av_cold int vp3_decode_init(AVCod
     }
 
     return 0;
+
+vlc_fail:
+    av_log(avctx, AV_LOG_FATAL, "Invalid huffman table\n");
+    return -1;
 }
 
 /*