From 76a7dc39e57ae192f521779c81c6e0b8c5bd4e85 Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Fri, 25 Sep 2015 14:34:02 +0200 Subject: [PATCH] rescaler: add some SSE2 code The rounding and arithmetic is not the same as previously, to prevent overflow cases for large upscale factors. We still rely on 32b x 32b -> 64b multiplies. Raised the fixed-point precision to 32b so that we have some nice shifts from epi64 to epi32. Changed rescaler_t type to 'uint32_t' in order to squeeze in all the precision required. The MIPS code has been disabled because it's now out-of-sync. Will be fixed in a subsequent CL when the dust settles. ~30-35% faster Change-Id: I32e4ddc00933f1b1aa3463403086199fd5dad07b --- Android.mk | 1 + Makefile.vc | 1 + makefile.unix | 1 + src/dsp/Makefile.am | 1 + src/dsp/rescaler.c | 43 ++++-- src/dsp/rescaler_mips32.c | 31 +++-- src/dsp/rescaler_mips_dsp_r2.c | 30 +++-- src/dsp/rescaler_sse2.c | 232 +++++++++++++++++++++++++++++++++ src/utils/rescaler.c | 15 ++- src/utils/rescaler.h | 9 +- 10 files changed, 320 insertions(+), 44 deletions(-) create mode 100644 src/dsp/rescaler_sse2.c diff --git a/Android.mk b/Android.mk index d2cd9980..99510133 100644 --- a/Android.mk +++ b/Android.mk @@ -62,6 +62,7 @@ dsp_dec_srcs := \ src/dsp/rescaler.c \ src/dsp/rescaler_mips32.c \ src/dsp/rescaler_mips_dsp_r2.c \ + src/dsp/rescaler_sse2.c \ src/dsp/upsampling.c \ src/dsp/upsampling_mips_dsp_r2.c \ src/dsp/upsampling_neon.$(NEON) \ diff --git a/Makefile.vc b/Makefile.vc index 911474db..fdeaf1c7 100644 --- a/Makefile.vc +++ b/Makefile.vc @@ -207,6 +207,7 @@ DSP_DEC_OBJS = \ $(DIROBJ)\dsp\rescaler.obj \ $(DIROBJ)\dsp\rescaler_mips32.obj \ $(DIROBJ)\dsp\rescaler_mips_dsp_r2.obj \ + $(DIROBJ)\dsp\rescaler_sse2.obj \ $(DIROBJ)\dsp\upsampling.obj \ $(DIROBJ)\dsp\upsampling_mips_dsp_r2.obj \ $(DIROBJ)\dsp\upsampling_neon.obj \ diff --git a/makefile.unix b/makefile.unix index 8570e411..48df98dd 100644 --- a/makefile.unix +++ b/makefile.unix @@ -149,6 +149,7 @@ DSP_DEC_OBJS = \ src/dsp/rescaler.o \ src/dsp/rescaler_mips32.o \ src/dsp/rescaler_mips_dsp_r2.o \ + src/dsp/rescaler_sse2.o \ src/dsp/upsampling.o \ src/dsp/upsampling_mips_dsp_r2.o \ src/dsp/upsampling_neon.o \ diff --git a/src/dsp/Makefile.am b/src/dsp/Makefile.am index 0981336b..769373d6 100644 --- a/src/dsp/Makefile.am +++ b/src/dsp/Makefile.am @@ -69,6 +69,7 @@ libwebpdspdecode_sse2_la_SOURCES += alpha_processing_sse2.c libwebpdspdecode_sse2_la_SOURCES += dec_sse2.c libwebpdspdecode_sse2_la_SOURCES += filters_sse2.c libwebpdspdecode_sse2_la_SOURCES += lossless_sse2.c +libwebpdspdecode_sse2_la_SOURCES += rescaler_sse2.c libwebpdspdecode_sse2_la_SOURCES += upsampling_sse2.c libwebpdspdecode_sse2_la_SOURCES += yuv_sse2.c libwebpdspdecode_sse2_la_SOURCES += yuv_tables_sse2.h diff --git a/src/dsp/rescaler.c b/src/dsp/rescaler.c index 76a214ae..7bdae1a7 100644 --- a/src/dsp/rescaler.c +++ b/src/dsp/rescaler.c @@ -100,18 +100,24 @@ void WebPRescalerExportRowExpandC(WebPRescaler* const wrk) { assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(wrk->y_expand); + assert(wrk->y_sub != 0); if (wrk->y_accum == 0) { for (x_out = 0; x_out < x_out_max; ++x_out) { - const int v = (int)MULT_FIX(frow[x_out], wrk->fy_scale); - dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; } } else { - const int64_t A = wrk->y_sub + wrk->y_accum; - const int64_t B = -wrk->y_accum; + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = WEBP_RESCALER_ONE - B; for (x_out = 0; x_out < x_out_max; ++x_out) { - const int64_t I = A * frow[x_out] + B * irow[x_out]; - const int v = (int)MULT_FIX(I, wrk->fxy_scale); - dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; } } } @@ -122,21 +128,28 @@ void WebPRescalerExportRowShrinkC(WebPRescaler* const wrk) { rescaler_t* const irow = wrk->irow; const int x_out_max = wrk->dst_width * wrk->num_channels; const rescaler_t* const frow = wrk->frow; - const int yscale = wrk->fy_scale * (-wrk->y_accum); + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(!wrk->y_expand); if (yscale) { for (x_out = 0; x_out < x_out_max; ++x_out) { - const int frac = (int)MULT_FIX(frow[x_out], yscale); + const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale); const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); - dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; + assert(v >= 0 && v <= 255); + dst[x_out] = v; irow[x_out] = frac; // new fractional start } - } else { + } else if (wrk->fxy_scale) { for (x_out = 0; x_out < x_out_max; ++x_out) { const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale); - dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } else { // very special case for src = dst = 1x1 + for (x_out = 0; x_out < x_out_max; ++x_out) { + dst[x_out] = irow[x_out]; irow[x_out] = 0; } } @@ -179,6 +192,7 @@ WebPRescalerImportRowFunc WebPRescalerImportRowShrink; WebPRescalerExportRowFunc WebPRescalerExportRowExpand; WebPRescalerExportRowFunc WebPRescalerExportRowShrink; +extern void WebPRescalerDspInitSSE2(void); extern void WebPRescalerDspInitMIPS32(void); extern void WebPRescalerDspInitMIPSdspR2(void); @@ -194,6 +208,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInit(void) { WebPRescalerExportRowShrink = WebPRescalerExportRowShrinkC; if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPRescalerDspInitSSE2(); + } +#endif #if defined(WEBP_USE_MIPS32) if (VP8GetCPUInfo(kMIPS32)) { WebPRescalerDspInitMIPS32(); diff --git a/src/dsp/rescaler_mips32.c b/src/dsp/rescaler_mips32.c index f474b4ab..04ce6ad4 100644 --- a/src/dsp/rescaler_mips32.c +++ b/src/dsp/rescaler_mips32.c @@ -31,7 +31,7 @@ static void ImportRowShrink(WebPRescaler* const wrk, const uint8_t* src) { for (channel = 0; channel < x_stride; ++channel) { const uint8_t* src1 = src + channel; - int* frow = wrk->frow + channel; + rescaler_t* frow = wrk->frow + channel; int temp1, temp2, temp3; int base, frac, sum; int accum, accum1; @@ -90,7 +90,7 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) { for (channel = 0; channel < x_stride; ++channel) { const uint8_t* src1 = src + channel; - int* frow = wrk->frow + channel; + rescaler_t* frow = wrk->frow + channel; int temp1, temp2, temp3, temp4; int frac; int accum; @@ -138,18 +138,15 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) { } static void ExportRowShrink(WebPRescaler* const wrk) { + const int x_out_max = wrk->dst_width * wrk->num_channels; + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(!wrk->y_expand); - // if wrk->fxy_scale can fit into 32 bits use optimized code, - // otherwise use C code - if ((wrk->fxy_scale >> 32) == 0) { - uint8_t* dst = wrk->dst; - rescaler_t* irow = wrk->irow; + if (wrk->fxy_scale != 0) { const rescaler_t* frow = wrk->frow; const int yscale = wrk->fy_scale * (-wrk->y_accum); - const int x_out_max = wrk->dst_width * wrk->num_channels; - int temp0, temp1, temp3, temp4, temp5, temp6, temp7, loop_end; const int temp2 = (int)(wrk->fxy_scale); const int temp8 = x_out_max << 2; @@ -192,8 +189,12 @@ static void ExportRowShrink(WebPRescaler* const wrk) { : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp8]"r"(temp8) : "memory", "hi", "lo" ); - } else { - WebPRescalerExportRowShrinkC(wrk); + } else { // very special case for src = dst = 1x1 + int x_out; + for (x_out = 0; x_out < x_out_max; ++x_out) { + dst[x_out] = irow[x_out]; + irow[x_out] = 0; + } } } @@ -205,9 +206,17 @@ static void ExportRowShrink(WebPRescaler* const wrk) { extern void WebPRescalerDspInitMIPS32(void); WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPS32(void) { +#if 0 + // The assembly code is currently out-of-sync wrt the C-implementation. + // Disabled for now. WebPRescalerImportRowExpand = ImportRowExpand; WebPRescalerImportRowShrink = ImportRowShrink; WebPRescalerExportRowShrink = ExportRowShrink; +#else + (void)ImportRowExpand; + (void)ImportRowShrink; + (void)ExportRowShrink; +#endif } #else // !WEBP_USE_MIPS32 diff --git a/src/dsp/rescaler_mips_dsp_r2.c b/src/dsp/rescaler_mips_dsp_r2.c index 267ae4be..17708fd4 100644 --- a/src/dsp/rescaler_mips_dsp_r2.c +++ b/src/dsp/rescaler_mips_dsp_r2.c @@ -30,7 +30,7 @@ static void ImportRowShrink(WebPRescaler* const wrk, const uint8_t* src) { assert(!WebPRescalerInputDone(wrk)); for (channel = 0; channel < x_stride; ++channel) { - int* frow = wrk->frow + channel; + rescaler_t* frow = wrk->frow + channel; const uint8_t* src1 = src + channel; int temp3; int base, frac, sum; @@ -84,7 +84,7 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) { assert(!WebPRescalerInputDone(wrk)); for (channel = 0; channel < x_stride; ++channel) { - int* frow = wrk->frow + channel; + rescaler_t* frow = wrk->frow + channel; const uint8_t* src1 = src + channel; int temp1, temp2, temp3, temp4; int frac; @@ -133,17 +133,15 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) { } static void ExportRowShrink(WebPRescaler* const wrk) { + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(!wrk->y_expand); - // if wrk->fxy_scale can fit into 32 bits use optimized code, - // otherwise use C code - if ((wrk->fxy_scale >> 32) == 0) { - uint8_t* dst = wrk->dst; - rescaler_t* irow = wrk->irow; + if (wrk->fxy_scale) { const rescaler_t* frow = wrk->frow; const int yscale = wrk->fy_scale * (-wrk->y_accum); - const int x_out_max = wrk->dst_width * wrk->num_channels; int temp0, temp1, temp3, temp4, temp5, temp6, temp7; const int temp2 = (int)wrk->fxy_scale; @@ -212,8 +210,12 @@ static void ExportRowShrink(WebPRescaler* const wrk) { [rest]"r"(rest) : "memory", "hi", "lo" ); - } else { - WebPRescalerExportRowShrinkC(wrk); + } else { // very special case for src = dst = 1x1 + int x_out; + for (x_out = 0; x_out < x_out_max; ++x_out) { + dst[x_out] = irow[x_out]; + irow[x_out] = 0; + } } } @@ -225,9 +227,17 @@ static void ExportRowShrink(WebPRescaler* const wrk) { extern void WebPRescalerDspInitMIPSdspR2(void); WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPSdspR2(void) { +#if 0 + // The assembly code is currently out-of-sync wrt the C-implementation. + // Disabled for now. WebPRescalerImportRowExpand = ImportRowExpand; WebPRescalerImportRowShrink = ImportRowShrink; WebPRescalerExportRowShrink = ExportRowShrink; +#else + (void)ImportRowExpand; + (void)ImportRowShrink; + (void)ExportRowShrink; +#endif } #else // !WEBP_USE_MIPS_DSP_R2 diff --git a/src/dsp/rescaler_sse2.c b/src/dsp/rescaler_sse2.c new file mode 100644 index 00000000..a0534cab --- /dev/null +++ b/src/dsp/rescaler_sse2.c @@ -0,0 +1,232 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 Rescaling functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include + +#include +#include "../utils/rescaler.h" + +//------------------------------------------------------------------------------ +// Implementations of critical functions ImportRow / ExportRow + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +//------------------------------------------------------------------------------ +// Row export + +// load *src as epi64, multiply by mult and store result in [out0 ... out3] +static WEBP_INLINE void LoadDispatchAndMult(const rescaler_t* const src, + const __m128i* const mult, + __m128i* const out0, + __m128i* const out1, + __m128i* const out2, + __m128i* const out3) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0)); + const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4)); + const __m128i A2 = _mm_srli_epi64(A0, 32); + const __m128i A3 = _mm_srli_epi64(A1, 32); + if (mult != NULL) { + *out0 = _mm_mul_epu32(A0, *mult); + *out1 = _mm_mul_epu32(A1, *mult); + *out2 = _mm_mul_epu32(A2, *mult); + *out3 = _mm_mul_epu32(A3, *mult); + } else { + *out0 = A0; + *out1 = A1; + *out2 = A2; + *out3 = A3; + } +} + +static WEBP_INLINE void ProcessRow(const __m128i* const A0, + const __m128i* const A1, + const __m128i* const A2, + const __m128i* const A3, + const __m128i* const mult, + uint8_t* const dst) { + const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER); + const __m128i mask = _mm_set_epi64x(0xffffffff00000000ull, + 0xffffffff00000000ull); + const __m128i B0 = _mm_mul_epu32(*A0, *mult); + const __m128i B1 = _mm_mul_epu32(*A1, *mult); + const __m128i B2 = _mm_mul_epu32(*A2, *mult); + const __m128i B3 = _mm_mul_epu32(*A3, *mult); + const __m128i C0 = _mm_add_epi64(B0, rounder); + const __m128i C1 = _mm_add_epi64(B1, rounder); + const __m128i C2 = _mm_add_epi64(B2, rounder); + const __m128i C3 = _mm_add_epi64(B3, rounder); + const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); + const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); + const __m128i D2 = _mm_and_si128(C2, mask); + const __m128i D3 = _mm_and_si128(C3, mask); + const __m128i E0 = _mm_or_si128(D0, D2); + const __m128i E1 = _mm_or_si128(D1, D3); + const __m128i F = _mm_packs_epi32(E0, E1); + const __m128i G = _mm_packus_epi16(F, F); + _mm_storel_epi64((__m128i*)dst, G); +} + +static void RescalerExportRowExpandSSE2(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + const __m128i mult = _mm_set_epi64x(wrk->fy_scale, wrk->fy_scale); + + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0); + assert(wrk->y_expand); + if (wrk->y_accum == 0) { + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3; + LoadDispatchAndMult(frow + x_out, NULL, &A0, &A1, &A2, &A3); + ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = WEBP_RESCALER_ONE - B; + const __m128i mA = _mm_set_epi64x(A, A); + const __m128i mB = _mm_set_epi64x(B, B); + const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3, B0, B1, B2, B3; + LoadDispatchAndMult(frow + x_out, &mA, &A0, &A1, &A2, &A3); + LoadDispatchAndMult(irow + x_out, &mB, &B0, &B1, &B2, &B3); + { + const __m128i C0 = _mm_add_epi64(A0, B0); + const __m128i C1 = _mm_add_epi64(A1, B1); + const __m128i C2 = _mm_add_epi64(A2, B2); + const __m128i C3 = _mm_add_epi64(A3, B3); + const __m128i D0 = _mm_add_epi64(C0, rounder); + const __m128i D1 = _mm_add_epi64(C1, rounder); + const __m128i D2 = _mm_add_epi64(C2, rounder); + const __m128i D3 = _mm_add_epi64(C3, rounder); + const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX); + const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX); + const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX); + const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX); + ProcessRow(&E0, &E1, &E2, &E3, &mult, dst + x_out); + } + } + for (; x_out < x_out_max; ++x_out) { + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +static void RescalerExportRowShrinkSSE2(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + if (yscale) { + const int scale_xy = wrk->fxy_scale; + const __m128i mult_xy = _mm_set_epi64x(scale_xy, scale_xy); + const __m128i mult_y = _mm_set_epi64x(yscale, yscale); + const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3, B0, B1, B2, B3; + LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); + LoadDispatchAndMult(frow + x_out, &mult_y, &B0, &B1, &B2, &B3); + { + const __m128i C0 = _mm_add_epi64(B0, rounder); + const __m128i C1 = _mm_add_epi64(B1, rounder); + const __m128i C2 = _mm_add_epi64(B2, rounder); + const __m128i C3 = _mm_add_epi64(B3, rounder); + const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); // = frac + const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); + const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX); + const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX); + const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac + const __m128i E1 = _mm_sub_epi64(A1, D1); + const __m128i E2 = _mm_sub_epi64(A2, D2); + const __m128i E3 = _mm_sub_epi64(A3, D3); + const __m128i F2 = _mm_slli_epi64(D2, 32); + const __m128i F3 = _mm_slli_epi64(D3, 32); + const __m128i G0 = _mm_or_si128(D0, F2); + const __m128i G1 = _mm_or_si128(D1, F3); + _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0); + _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1); + ProcessRow(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); + } + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = frac; // new fractional start + } + } else if (wrk->fxy_scale) { + const uint32_t scale = wrk->fxy_scale; + const __m128i mult = _mm_set_epi64x(scale, scale); + const __m128i zero = _mm_setzero_si128(); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3; + LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); + _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero); + _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero); + ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const int v = (int)MULT_FIX(irow[x_out], scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } else { // very special case for src = 1x1 + for (x_out = 0; x_out < x_out_max; ++x_out) { + dst[x_out] = irow[x_out]; + irow[x_out] = 0; + } + } +} + +#undef MULT_FIX +#undef ROUNDER + +//------------------------------------------------------------------------------ + +extern void WebPRescalerDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) { + WebPRescalerExportRowExpand = RescalerExportRowExpandSSE2; + WebPRescalerExportRowShrink = RescalerExportRowShrinkSSE2; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/src/utils/rescaler.c b/src/utils/rescaler.c index 6b5b56d3..41a0c1e2 100644 --- a/src/utils/rescaler.c +++ b/src/utils/rescaler.c @@ -41,19 +41,20 @@ void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height, wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add; wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub; if (!wrk->x_expand) { // fx_scale is not used otherwise - wrk->fx_scale = WEBP_RESCALER_ONE / wrk->x_sub; + wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub); } // vertical scaling parameters wrk->y_add = wrk->y_expand ? y_add - 1 : y_add; - wrk->y_sub = wrk->y_expand ? y_sub - 1: y_sub; + wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub; wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add; if (!wrk->y_expand) { - wrk->fy_scale = WEBP_RESCALER_ONE / wrk->y_sub; - wrk->fxy_scale = ((uint64_t)dst_height << WEBP_RESCALER_RFIX) - / (wrk->x_add * wrk->y_add); + // note the very special case where x_add = y_add = 1 cannot be represented. + // We special-case fxy_scale = 0 in this case, in ExportRowShrink + wrk->fxy_scale = WEBP_RESCALER_FRAC(dst_height, wrk->x_add * wrk->y_add); + wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub); } else { - wrk->fy_scale = WEBP_RESCALER_ONE / wrk->x_add; - wrk->fxy_scale = WEBP_RESCALER_ONE / (wrk->x_add * wrk->y_sub); + wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add); + // wrk->fxy_scale is unused here. } wrk->irow = work; wrk->frow = work + num_channels * dst_width; diff --git a/src/utils/rescaler.h b/src/utils/rescaler.h index 7787f9c7..61405875 100644 --- a/src/utils/rescaler.h +++ b/src/utils/rescaler.h @@ -20,11 +20,12 @@ extern "C" { #include "../webp/types.h" -#define WEBP_RESCALER_RFIX 30 // fixed-point precision for multiplies -#define WEBP_RESCALER_ONE (1u << WEBP_RESCALER_RFIX) +#define WEBP_RESCALER_RFIX 32 // fixed-point precision for multiplies +#define WEBP_RESCALER_ONE (1ull << WEBP_RESCALER_RFIX) +#define WEBP_RESCALER_FRAC(x, y) (((uint64_t)(x) << WEBP_RESCALER_RFIX) / (y)) // Structure used for on-the-fly rescaling -typedef int32_t rescaler_t; // type for side-buffer +typedef uint32_t rescaler_t; // type for side-buffer typedef struct WebPRescaler WebPRescaler; struct WebPRescaler { int x_expand; // true if we're expanding in the x direction @@ -32,7 +33,7 @@ struct WebPRescaler { int num_channels; // bytes to jump between pixels uint32_t fx_scale; // fixed-point scaling factors uint32_t fy_scale; // '' - uint64_t fxy_scale; // '' + uint32_t fxy_scale; // '' int y_accum; // vertical accumulator int y_add, y_sub; // vertical increments int x_add, x_sub; // horizontal increments