[FFmpeg-cvslog] AAC encoder: fix signed integer overflow

Claudio Freire git at videolan.org
Wed Mar 30 15:04:48 CEST 2016


ffmpeg | branch: master | Claudio Freire <klaussfreire at gmail.com> | Wed Mar 30 10:05:03 2016 -0300| [c883da6bf4610ee37848687346f543dfb73de011] | committer: Claudio Freire

AAC encoder: fix signed integer overflow

Clamp scalefactors by coef2minsf to avoid undefined behavior
caused by signed integer overflow. It also avoids clipping of
coefficients so it should avoid artifacts as well, on very
rare corner cases.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c883da6bf4610ee37848687346f543dfb73de011
---

 libavcodec/aaccoder_twoloop.h |    9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libavcodec/aaccoder_twoloop.h b/libavcodec/aaccoder_twoloop.h
index 397a4db..4747c79 100644
--- a/libavcodec/aaccoder_twoloop.h
+++ b/libavcodec/aaccoder_twoloop.h
@@ -77,7 +77,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
     int toomanybits, toofewbits;
     char nzs[128];
     uint8_t nextband[128];
-    int maxsf[128];
+    int maxsf[128], minsf[128];
     float dists[128] = { 0 }, qenergies[128] = { 0 }, uplims[128], euplims[128], energies[128];
     float maxvals[128], spread_thr_r[128];
     float min_spread_thr_r, max_spread_thr_r;
@@ -294,11 +294,14 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
     abs_pow34_v(s->scoefs, sce->coeffs, 1024);
     ff_quantize_band_cost_cache_init(s);
 
+    for (i = 0; i < sizeof(minsf) / sizeof(minsf[0]); ++i)
+        minsf[i] = 0;
     for (w = 0; w < sce->ics.num_windows; w += sce->ics.group_len[w]) {
         start = w*128;
         for (g = 0;  g < sce->ics.num_swb; g++) {
             const float *scaled = s->scoefs + start;
             maxvals[w*16+g] = find_max_val(sce->ics.group_len[w], sce->ics.swb_sizes[g], scaled);
+            minsf[w*16+g] = coef2minsf(maxvals[w*16+g]);
             start += sce->ics.swb_sizes[g];
         }
     }
@@ -425,7 +428,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
                 recomprd = 1;
                 for (i = 0; i < 128; i++) {
                     if (sce->sf_idx[i] > SCALE_ONE_POS) {
-                        int new_sf = FFMAX(SCALE_ONE_POS, sce->sf_idx[i] - qstep);
+                        int new_sf = FFMAX3(minsf[i], SCALE_ONE_POS, sce->sf_idx[i] - qstep);
                         if (new_sf != sce->sf_idx[i]) {
                             sce->sf_idx[i] = new_sf;
                             changed = 1;
@@ -595,7 +598,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
                     int cmb = find_min_book(maxvals[w*16+g], sce->sf_idx[w*16+g]);
                     int mindeltasf = FFMAX(0, prev - SCALE_MAX_DIFF);
                     int maxdeltasf = FFMIN(SCALE_MAX_POS - SCALE_DIV_512, prev + SCALE_MAX_DIFF);
-                    if ((!cmb || dists[w*16+g] > uplims[w*16+g]) && sce->sf_idx[w*16+g] > mindeltasf) {
+                    if ((!cmb || dists[w*16+g] > uplims[w*16+g]) && sce->sf_idx[w*16+g] > FFMAX(mindeltasf, minsf[w*16+g])) {
                         /* Try to make sure there is some energy in every nonzero band
                          * NOTE: This algorithm must be forcibly imbalanced, pushing harder
                          *  on holes or more distorted bands at first, otherwise there's



More information about the ffmpeg-cvslog mailing list