[PATCH] motion_est: fixed signed_integer_overflow when computing score
--- libavcodec/motion_est.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index d17ffe42b4..95978d95dd 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -1571,7 +1571,7 @@ void ff_estimate_b_frame_motion(MpegEncContext * s, type= CANDIDATE_MB_TYPE_BACKWARD_I; } - score= ((unsigned)(score*score + 128*256))>>16; + score= ((uint64_t)(score)*(uint64_t)(score) + 128*256)>>16; c->mc_mb_var_sum_temp += score; s->mc_mb_var[mb_y*s->mb_stride + mb_x] = score; //FIXME use SSE } -- 2.38.1.493.g58b659f92b-goog
On 11/9/2022 7:31 PM, Thierry Foucu wrote:
--- libavcodec/motion_est.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index d17ffe42b4..95978d95dd 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -1571,7 +1571,7 @@ void ff_estimate_b_frame_motion(MpegEncContext * s, type= CANDIDATE_MB_TYPE_BACKWARD_I; }
- score= ((unsigned)(score*score + 128*256))>>16; + score= ((uint64_t)(score)*(uint64_t)(score) + 128*256)>>16;
Is it not enough just doing score= ((unsigned)score*score + 128*256)>>16; If not, then use uint64_t, but like the above, you should only need to cast score once.
c->mc_mb_var_sum_temp += score; s->mc_mb_var[mb_y*s->mb_stride + mb_x] = score; //FIXME use SSE }
participants (2)
-
James Almer -
Thierry Foucu