[FFmpeg-cvslog] avcodec/motion_est: Fix mv_penalty table size

Michael Niedermayer git at videolan.org
Fri Jan 15 17:39:18 CET 2016


ffmpeg | branch: release/2.6 | Michael Niedermayer <michael at niedermayer.cc> | Tue Jan  5 14:41:04 2016 +0100| [f3cf94fa47d197ed33c00012a0a827bf88431e6d] | committer: Michael Niedermayer

avcodec/motion_est: Fix mv_penalty table size

Fixes out of array read

Found-by: Tyson Smith <twsmith at mozilla.com>
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 5b4da8a38a5ed211df9504c85ce401c30af86b97)

Conflicts:

	libavcodec/motion_est.h

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f3cf94fa47d197ed33c00012a0a827bf88431e6d
---

 libavcodec/ituh263enc.c    |    6 +++---
 libavcodec/motion_est.c    |   16 ++++++++--------
 libavcodec/mpeg12enc.c     |    6 +++---
 libavcodec/mpegvideo.h     |    3 ++-
 libavcodec/mpegvideo_enc.c |    2 +-
 libavcodec/snowenc.c       |    2 +-
 6 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/libavcodec/ituh263enc.c b/libavcodec/ituh263enc.c
index 7d4d9b0..6ce634d 100644
--- a/libavcodec/ituh263enc.c
+++ b/libavcodec/ituh263enc.c
@@ -43,7 +43,7 @@
 /**
  * Table of number of bits a motion vector component needs.
  */
-static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
+static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1];
 
 /**
  * Minimal fcode that a motion vector component would need.
@@ -676,7 +676,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
     int mv;
 
     for(f_code=1; f_code<=MAX_FCODE; f_code++){
-        for(mv=-MAX_MV; mv<=MAX_MV; mv++){
+        for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){
             int len;
 
             if(mv==0) len= ff_mvtab[0][1];
@@ -697,7 +697,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
                 }
             }
 
-            mv_penalty[f_code][mv+MAX_MV]= len;
+            mv_penalty[f_code][mv+MAX_DMV]= len;
         }
     }
 
diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c
index a0a5965..d161043 100644
--- a/libavcodec/motion_est.c
+++ b/libavcodec/motion_est.c
@@ -906,7 +906,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
     c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
     c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
-    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
+    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
 
     get_limits(s, 16*mb_x, 16*mb_y);
     c->skip=0;
@@ -1082,7 +1082,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s,
     av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
 
     c->pre_penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp);
-    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
+    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
 
     get_limits(s, 16*mb_x, 16*mb_y);
     c->skip=0;
@@ -1131,7 +1131,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
     const int shift= 1+s->quarter_sample;
     const int mot_stride = s->mb_stride;
     const int mot_xy = mb_y*mot_stride + mb_x;
-    uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV;
+    uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV;
     int mv_scale;
 
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
@@ -1205,8 +1205,8 @@ static inline int check_bidir_mv(MpegEncContext * s,
     //FIXME better f_code prediction (max mv & distance)
     //FIXME pointers
     MotionEstContext * const c= &s->me;
-    uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame
-    uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame
+    uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame
+    uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame
     int stride= c->stride;
     uint8_t *dest_y = c->scratchpad;
     uint8_t *ptr;
@@ -1419,7 +1419,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
     int mx, my, xmin, xmax, ymin, ymax;
     int16_t (*mv_table)[2]= s->b_direct_mv_table;
 
-    c->current_mv_penalty= c->mv_penalty[1] + MAX_MV;
+    c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV;
     ymin= xmin=(-32)>>shift;
     ymax= xmax=   31>>shift;
 
@@ -1555,11 +1555,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
     if(s->flags & CODEC_FLAG_INTERLACED_ME){
 //FIXME mb type penalty
         c->skip=0;
-        c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
+        c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
         fimin= interlaced_search(s, 0,
                                  s->b_field_mv_table[0], s->b_field_select_table[0],
                                  s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0);
-        c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV;
+        c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV;
         bimin= interlaced_search(s, 2,
                                  s->b_field_mv_table[1], s->b_field_select_table[1],
                                  s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0);
diff --git a/libavcodec/mpeg12enc.c b/libavcodec/mpeg12enc.c
index 9795b7f..451463c 100644
--- a/libavcodec/mpeg12enc.c
+++ b/libavcodec/mpeg12enc.c
@@ -52,7 +52,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = {
     0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 };
 
-static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
+static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
 static uint8_t fcode_tab[MAX_MV * 2 + 1];
 
 static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2];
@@ -1051,7 +1051,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
         }
 
         for (f_code = 1; f_code <= MAX_FCODE; f_code++)
-            for (mv = -MAX_MV; mv <= MAX_MV; mv++) {
+            for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) {
                 int len;
 
                 if (mv == 0) {
@@ -1074,7 +1074,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
                               2 + bit_size;
                 }
 
-                mv_penalty[f_code][mv + MAX_MV] = len;
+                mv_penalty[f_code][mv + MAX_DMV] = len;
             }
 
 
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index e26bd99..7dd5ad7 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -66,6 +66,7 @@ enum OutputFormat {
 
 #define MAX_FCODE 7
 #define MAX_MV 4096
+#define MAX_DMV (2*MAX_MV)
 
 #define MAX_THREADS 32
 #define MAX_PICTURE_COUNT 36
@@ -198,7 +199,7 @@ typedef struct MotionEstContext{
     op_pixels_func (*hpel_avg)[4];
     qpel_mc_func (*qpel_put)[16];
     qpel_mc_func (*qpel_avg)[16];
-    uint8_t (*mv_penalty)[MAX_MV*2+1];  ///< amount of bits needed to encode a MV
+    uint8_t (*mv_penalty)[MAX_DMV*2+1];  ///< amount of bits needed to encode a MV
     uint8_t *current_mv_penalty;
     int (*sub_motion_search)(struct MpegEncContext * s,
                                   int *mx_ptr, int *my_ptr, int dmin,
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 43ccfed..76f78bd 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -70,7 +70,7 @@ static int sse_mb(MpegEncContext *s);
 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
 
-static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
+static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
 
 const AVOption ff_mpv_generic_options[] = {
diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c
index 544efbf..957fc9b 100644
--- a/libavcodec/snowenc.c
+++ b/libavcodec/snowenc.c
@@ -286,7 +286,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
     c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
     c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
-    c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV;
+    c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV;
 
     c->xmin = - x*block_w - 16+3;
     c->ymin = - y*block_w - 16+3;



More information about the ffmpeg-cvslog mailing list