generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

partial_idct_test,InitInput: fix rollover in mult

Browse Source 
promote coeff to signed 64-bit to avoid exceeding integer bounds when
squaring the value

Change-Id: If77bef6bc0a6a4c39ca3013e5e2ddb426a1c6e1f
This commit is contained in:
James Zern 2017-05-24 15:25:44 +02:00
parent 36f1b183e4
commit 566f6d75bd
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/partial_idct_test.cc
View File

@ -133,7 +133,7 @@ class PartialIDctTest : public ::testing::TestWithParam<PartialInvTxfmParam> {
for (int j = 0; j < last_nonzero_; ++j) {
tran_low_t coeff = static_cast<tran_low_t>(
sqrt(1.0 * max_energy_leftover) * (rnd_.Rand16() - 32768) / 65536);
max_energy_leftover -= coeff * coeff;
max_energy_leftover -= static_cast<int64_t>(coeff) * coeff;
if (max_energy_leftover < 0) {
max_energy_leftover = 0;
coeff = 0;