generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fdct4x4_test: fix unsigned overflow

Browse Source 
the difference between src and dst will be signed, the error will be
unsigned.
quiets -fsanitize=integer:
unsigned integer overflow: 4294967295 * 4294967295

Change-Id: I502fd707823c4faaa7f587c9cc0312f057e04904
This commit is contained in:
James Zern 2016-06-08 17:29:02 -07:00
parent 77ffea92c5
commit 06c6e4cbf6
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
test/fdct4x4_test.cc
View File

@ -141,11 +141,11 @@ class Trans4x4TestBase {
for (int j = 0; j < kNumCoeffs; ++j) { for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_VP9_HIGHBITDEPTH #if CONFIG_VP9_HIGHBITDEPTH
const uint32_t diff = const int diff =
bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else #else
ASSERT_EQ(VPX_BITS_8, bit_depth_); ASSERT_EQ(VPX_BITS_8, bit_depth_);
const uint32_t diff = dst[j] - src[j]; const int diff = dst[j] - src[j];
#endif #endif
const uint32_t error = diff * diff; const uint32_t error = diff * diff;
if (max_error < error) if (max_error < error)
@ -258,10 +258,10 @@ class Trans4x4TestBase {
for (int j = 0; j < kNumCoeffs; ++j) { for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_VP9_HIGHBITDEPTH #if CONFIG_VP9_HIGHBITDEPTH
const uint32_t diff = const int diff =
bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else #else
const uint32_t diff = dst[j] - src[j]; const int diff = dst[j] - src[j];
#endif #endif
const uint32_t error = diff * diff; const uint32_t error = diff * diff;
EXPECT_GE(static_cast<uint32_t>(limit), error) EXPECT_GE(static_cast<uint32_t>(limit), error)