encoding SSE4.1 stub for StoreHistogram + Quantize + SSE_16xN
Visible speed-up, thanks to pshufb and pabsw and psignw use. had to tweak configure.ac to make "smmintri.h" presence correctly detected (we need to set the CPPFLAGS instead of the CFLAGS!) Change-Id: I2ab99e16a27a64fdf1f09b2b4e30a5e74ccca080
This commit is contained in:
Pascal Massimino
James Zern
parent
c64659e1b4
commit
94055503e3
@ -79,6 +79,7 @@ dsp_enc_srcs := \
|
||||
src/dsp/enc_mips_dsp_r2.c \
|
||||
src/dsp/enc_neon.$(NEON) \
|
||||
src/dsp/enc_sse2.c \
|
||||
src/dsp/enc_sse41.c \
|
||||
src/dsp/lossless_enc.c \
|
||||
src/dsp/lossless_enc_mips32.c \
|
||||
src/dsp/lossless_enc_mips_dsp_r2.c \
|
||||
|
||||
@ -229,6 +229,7 @@ DSP_ENC_OBJS = \
|
||||
$(DIROBJ)\dsp\enc_mips_dsp_r2.obj \
|
||||
$(DIROBJ)\dsp\enc_neon.obj \
|
||||
$(DIROBJ)\dsp\enc_sse2.obj \
|
||||
$(DIROBJ)\dsp\enc_sse41.obj \
|
||||
$(DIROBJ)\dsp\lossless_enc.obj \
|
||||
$(DIROBJ)\dsp\lossless_enc_mips32.obj \
|
||||
$(DIROBJ)\dsp\lossless_enc_mips_dsp_r2.obj \
|
||||
|
||||
@ -99,13 +99,13 @@ AC_SUBST([AVX2_FLAGS])
|
||||
|
||||
TEST_AND_ADD_CFLAGS([SSE41_FLAGS], [-msse4.1])
|
||||
AS_IF([test -n "$SSE41_FLAGS"], [
|
||||
SAVED_CFLAGS=$CFLAGS
|
||||
CFLAGS="$CFLAGS $SSE41_FLAGS"
|
||||
SAVED_CPPFLAGS=$CPPFLAGS
|
||||
CPPFLAGS="$CPPFLAGS $SSE41_FLAGS"
|
||||
AC_CHECK_HEADER([smmintrin.h],
|
||||
[AC_DEFINE(WEBP_HAVE_SSE41, [1],
|
||||
[Set to 1 if SSE4.1 is supported])],
|
||||
[SSE41_FLAGS=""])
|
||||
CFLAGS=$SAVED_CFLAGS])
|
||||
CPPFLAGS=$SAVED_CPPFLAGS])
|
||||
AC_SUBST([SSE41_FLAGS])
|
||||
|
||||
TEST_AND_ADD_CFLAGS([SSE2_FLAGS], [-msse2])
|
||||
|
||||
@ -159,6 +159,7 @@ DSP_ENC_OBJS = \
|
||||
src/dsp/enc_mips_dsp_r2.o \
|
||||
src/dsp/enc_neon.o \
|
||||
src/dsp/enc_sse2.o \
|
||||
src/dsp/enc_sse41.o \
|
||||
src/dsp/lossless_enc.o \
|
||||
src/dsp/lossless_enc_mips32.o \
|
||||
src/dsp/lossless_enc_mips_dsp_r2.o \
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
noinst_LTLIBRARIES = libwebpdsp.la libwebpdsp_avx2.la
|
||||
noinst_LTLIBRARIES += libwebpdsp_sse2.la libwebpdspdecode_sse2.la
|
||||
noinst_LTLIBRARIES += libwebpdspdecode_sse41.la
|
||||
noinst_LTLIBRARIES += libwebpdsp_sse41.la libwebpdspdecode_sse41.la
|
||||
|
||||
if BUILD_LIBWEBPDECODER
|
||||
noinst_LTLIBRARIES += libwebpdspdecode.la
|
||||
@ -83,6 +83,12 @@ libwebpdsp_sse2_la_CPPFLAGS = $(libwebpdsp_la_CPPFLAGS)
|
||||
libwebpdsp_sse2_la_CFLAGS = $(AM_CFLAGS) $(SSE2_FLAGS)
|
||||
libwebpdsp_sse2_la_LIBADD = libwebpdspdecode_sse2.la
|
||||
|
||||
libwebpdsp_sse41_la_SOURCES =
|
||||
libwebpdsp_sse41_la_SOURCES += enc_sse41.c
|
||||
libwebpdsp_sse41_la_CPPFLAGS = $(libwebpdsp_la_CPPFLAGS)
|
||||
libwebpdsp_sse41_la_CFLAGS = $(AM_CFLAGS) $(SSE41_FLAGS)
|
||||
libwebpdsp_sse41_la_LIBADD = libwebpdspdecode_sse41.la
|
||||
|
||||
libwebpdsp_la_SOURCES = $(COMMON_SOURCES) $(ENC_SOURCES)
|
||||
|
||||
noinst_HEADERS =
|
||||
@ -93,7 +99,7 @@ libwebpdsp_la_CPPFLAGS = $(USE_EXPERIMENTAL_CODE) $(USE_SWAP_16BIT_CSP)
|
||||
libwebpdsp_la_LDFLAGS = -lm
|
||||
libwebpdsp_la_LIBADD =
|
||||
libwebpdsp_la_LIBADD += libwebpdsp_avx2.la libwebpdsp_sse2.la
|
||||
libwebpdsp_la_LIBADD += libwebpdspdecode_sse41.la
|
||||
libwebpdsp_la_LIBADD += libwebpdsp_sse41.la
|
||||
|
||||
if BUILD_LIBWEBPDECODER
|
||||
libwebpdspdecode_la_SOURCES = $(COMMON_SOURCES)
|
||||
|
||||
@ -722,6 +722,7 @@ VP8BlockCopy VP8Copy4x4;
|
||||
VP8BlockCopy VP8Copy16x8;
|
||||
|
||||
extern void VP8EncDspInitSSE2(void);
|
||||
extern void VP8EncDspInitSSE41(void);
|
||||
extern void VP8EncDspInitAVX2(void);
|
||||
extern void VP8EncDspInitNEON(void);
|
||||
extern void VP8EncDspInitMIPS32(void);
|
||||
@ -761,6 +762,11 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) {
|
||||
#if defined(WEBP_USE_SSE2)
|
||||
if (VP8GetCPUInfo(kSSE2)) {
|
||||
VP8EncDspInitSSE2();
|
||||
#if defined(WEBP_USE_SSE41)
|
||||
if (VP8GetCPUInfo(kSSE4_1)) {
|
||||
VP8EncDspInitSSE41();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
#if defined(WEBP_USE_AVX2)
|
||||
|
||||
263
src/dsp/enc_sse41.c
Normal file
263
src/dsp/enc_sse41.c
Normal file
@ -0,0 +1,263 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license
|
||||
// that can be found in the COPYING file in the root of the source
|
||||
// tree. An additional intellectual property rights grant can be found
|
||||
// in the file PATENTS. All contributing project authors may
|
||||
// be found in the AUTHORS file in the root of the source tree.
|
||||
// -----------------------------------------------------------------------------
|
||||
//
|
||||
// SSE4 version of some encoding functions.
|
||||
//
|
||||
// Author: Skal (pascal.massimino@gmail.com)
|
||||
|
||||
#include "./dsp.h"
|
||||
|
||||
#if defined(WEBP_USE_SSE41)
|
||||
#include <smmintrin.h>
|
||||
|
||||
#include "../enc/vp8enci.h"
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Compute susceptibility based on DCT-coeff histograms.
|
||||
|
||||
static void CollectHistogram(const uint8_t* ref, const uint8_t* pred,
|
||||
int start_block, int end_block,
|
||||
VP8Histogram* const histo) {
|
||||
const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH);
|
||||
int j;
|
||||
int distribution[MAX_COEFF_THRESH + 1] = { 0 };
|
||||
for (j = start_block; j < end_block; ++j) {
|
||||
int16_t out[16];
|
||||
int k;
|
||||
|
||||
VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out);
|
||||
|
||||
// Convert coefficients to bin (within out[]).
|
||||
{
|
||||
// Load.
|
||||
const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]);
|
||||
const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]);
|
||||
// v = abs(out) >> 3
|
||||
const __m128i abs0 = _mm_abs_epi16(out0);
|
||||
const __m128i abs1 = _mm_abs_epi16(out1);
|
||||
const __m128i v0 = _mm_srai_epi16(abs0, 3);
|
||||
const __m128i v1 = _mm_srai_epi16(abs1, 3);
|
||||
// bin = min(v, MAX_COEFF_THRESH)
|
||||
const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh);
|
||||
const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh);
|
||||
// Store.
|
||||
_mm_storeu_si128((__m128i*)&out[0], bin0);
|
||||
_mm_storeu_si128((__m128i*)&out[8], bin1);
|
||||
}
|
||||
|
||||
// Convert coefficients to bin.
|
||||
for (k = 0; k < 16; ++k) {
|
||||
++distribution[out[k]];
|
||||
}
|
||||
}
|
||||
VP8SetHistogramData(distribution, histo);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Metric
|
||||
|
||||
static WEBP_INLINE __m128i SubtractAndAccumulate(const __m128i a,
|
||||
const __m128i b) {
|
||||
// take abs(a-b) in 8b
|
||||
const __m128i a_b = _mm_subs_epu8(a, b);
|
||||
const __m128i b_a = _mm_subs_epu8(b, a);
|
||||
const __m128i abs_a_b = _mm_or_si128(a_b, b_a);
|
||||
// zero-extend to 16b
|
||||
const __m128i C0 = _mm_cvtepu8_epi16(abs_a_b);
|
||||
const __m128i C1 = _mm_cvtepu8_epi16(_mm_srli_si128(abs_a_b, 8));
|
||||
// multiply with self
|
||||
const __m128i D0 = _mm_madd_epi16(C0, C0);
|
||||
const __m128i D1 = _mm_madd_epi16(C1, C1);
|
||||
// accumulate
|
||||
const __m128i sum = _mm_add_epi32(D0, D1);
|
||||
return sum;
|
||||
}
|
||||
|
||||
static int SSE_16xN(const uint8_t* a, const uint8_t* b, int num_pairs) {
|
||||
__m128i sum = _mm_setzero_si128();
|
||||
int32_t tmp[4];
|
||||
|
||||
while (num_pairs-- > 0) {
|
||||
const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[BPS * 0]);
|
||||
const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[BPS * 1]);
|
||||
const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[BPS * 0]);
|
||||
const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[BPS * 1]);
|
||||
const __m128i sum1 = SubtractAndAccumulate(a0, b0);
|
||||
const __m128i sum2 = SubtractAndAccumulate(a1, b1);
|
||||
const __m128i sum12 = _mm_add_epi32(sum1, sum2);
|
||||
sum = _mm_add_epi32(sum, sum12);
|
||||
a += 2 * BPS;
|
||||
b += 2 * BPS;
|
||||
}
|
||||
_mm_storeu_si128((__m128i*)tmp, sum);
|
||||
return (tmp[3] + tmp[2] + tmp[1] + tmp[0]);
|
||||
}
|
||||
|
||||
static int SSE16x16(const uint8_t* a, const uint8_t* b) {
|
||||
return SSE_16xN(a, b, 8);
|
||||
}
|
||||
|
||||
static int SSE16x8(const uint8_t* a, const uint8_t* b) {
|
||||
return SSE_16xN(a, b, 4);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Quantization
|
||||
//
|
||||
|
||||
// Generates a pshufb constant for shuffling 16b words.
|
||||
#define PSHUFB_CST(A,B,C,D,E,F,G,H) \
|
||||
_mm_set_epi8(2 * (H) + 1, 2 * (H) + 0, 2 * (G) + 1, 2 * (G) + 0, \
|
||||
2 * (F) + 1, 2 * (F) + 0, 2 * (E) + 1, 2 * (E) + 0, \
|
||||
2 * (D) + 1, 2 * (D) + 0, 2 * (C) + 1, 2 * (C) + 0, \
|
||||
2 * (B) + 1, 2 * (B) + 0, 2 * (A) + 1, 2 * (A) + 0)
|
||||
|
||||
static WEBP_INLINE int DoQuantizeBlock(int16_t in[16], int16_t out[16],
|
||||
const uint16_t* const sharpen,
|
||||
const VP8Matrix* const mtx) {
|
||||
const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL);
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
__m128i out0, out8;
|
||||
__m128i packed_out;
|
||||
|
||||
// Load all inputs.
|
||||
// TODO(cduvivier): Make variable declarations and allocations aligned so that
|
||||
// we can use _mm_load_si128 instead of _mm_loadu_si128.
|
||||
__m128i in0 = _mm_loadu_si128((__m128i*)&in[0]);
|
||||
__m128i in8 = _mm_loadu_si128((__m128i*)&in[8]);
|
||||
const __m128i iq0 = _mm_loadu_si128((const __m128i*)&mtx->iq_[0]);
|
||||
const __m128i iq8 = _mm_loadu_si128((const __m128i*)&mtx->iq_[8]);
|
||||
const __m128i q0 = _mm_loadu_si128((const __m128i*)&mtx->q_[0]);
|
||||
const __m128i q8 = _mm_loadu_si128((const __m128i*)&mtx->q_[8]);
|
||||
|
||||
// coeff = abs(in)
|
||||
__m128i coeff0 = _mm_abs_epi16(in0);
|
||||
__m128i coeff8 = _mm_abs_epi16(in8);
|
||||
|
||||
// coeff = abs(in) + sharpen
|
||||
if (sharpen != NULL) {
|
||||
const __m128i sharpen0 = _mm_loadu_si128((const __m128i*)&sharpen[0]);
|
||||
const __m128i sharpen8 = _mm_loadu_si128((const __m128i*)&sharpen[8]);
|
||||
coeff0 = _mm_add_epi16(coeff0, sharpen0);
|
||||
coeff8 = _mm_add_epi16(coeff8, sharpen8);
|
||||
}
|
||||
|
||||
// out = (coeff * iQ + B) >> QFIX
|
||||
{
|
||||
// doing calculations with 32b precision (QFIX=17)
|
||||
// out = (coeff * iQ)
|
||||
const __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0);
|
||||
const __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0);
|
||||
const __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8);
|
||||
const __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8);
|
||||
__m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H);
|
||||
__m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H);
|
||||
__m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H);
|
||||
__m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H);
|
||||
// out = (coeff * iQ + B)
|
||||
const __m128i bias_00 = _mm_loadu_si128((const __m128i*)&mtx->bias_[0]);
|
||||
const __m128i bias_04 = _mm_loadu_si128((const __m128i*)&mtx->bias_[4]);
|
||||
const __m128i bias_08 = _mm_loadu_si128((const __m128i*)&mtx->bias_[8]);
|
||||
const __m128i bias_12 = _mm_loadu_si128((const __m128i*)&mtx->bias_[12]);
|
||||
out_00 = _mm_add_epi32(out_00, bias_00);
|
||||
out_04 = _mm_add_epi32(out_04, bias_04);
|
||||
out_08 = _mm_add_epi32(out_08, bias_08);
|
||||
out_12 = _mm_add_epi32(out_12, bias_12);
|
||||
// out = QUANTDIV(coeff, iQ, B, QFIX)
|
||||
out_00 = _mm_srai_epi32(out_00, QFIX);
|
||||
out_04 = _mm_srai_epi32(out_04, QFIX);
|
||||
out_08 = _mm_srai_epi32(out_08, QFIX);
|
||||
out_12 = _mm_srai_epi32(out_12, QFIX);
|
||||
|
||||
// pack result as 16b
|
||||
out0 = _mm_packs_epi32(out_00, out_04);
|
||||
out8 = _mm_packs_epi32(out_08, out_12);
|
||||
|
||||
// if (coeff > 2047) coeff = 2047
|
||||
out0 = _mm_min_epi16(out0, max_coeff_2047);
|
||||
out8 = _mm_min_epi16(out8, max_coeff_2047);
|
||||
}
|
||||
|
||||
// put sign back
|
||||
out0 = _mm_sign_epi16(out0, in0);
|
||||
out8 = _mm_sign_epi16(out8, in8);
|
||||
|
||||
// in = out * Q
|
||||
in0 = _mm_mullo_epi16(out0, q0);
|
||||
in8 = _mm_mullo_epi16(out8, q8);
|
||||
|
||||
_mm_storeu_si128((__m128i*)&in[0], in0);
|
||||
_mm_storeu_si128((__m128i*)&in[8], in8);
|
||||
|
||||
// zigzag the output before storing it. The re-ordering is:
|
||||
// 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15
|
||||
// -> 0 1 4[8]5 2 3 6 | 9 12 13 10 [7]11 14 15
|
||||
// There's only two misplaced entries ([8] and [7]) that are crossing the
|
||||
// reg's boundaries.
|
||||
// We use pshufb instead of pshuflo/pshufhi.
|
||||
{
|
||||
const __m128i kCst_lo = PSHUFB_CST(0, 1, 4, -1, 5, 2, 3, 6);
|
||||
const __m128i kCst_7 = PSHUFB_CST(-1, -1, -1, -1, 7, -1, -1, -1);
|
||||
const __m128i tmp_lo = _mm_shuffle_epi8(out0, kCst_lo);
|
||||
const __m128i tmp_7 = _mm_shuffle_epi8(out0, kCst_7); // extract #7
|
||||
const __m128i kCst_hi = PSHUFB_CST(1, 4, 5, 2, -1, 3, 6, 7);
|
||||
const __m128i kCst_8 = PSHUFB_CST(-1, -1, -1, 0, -1, -1, -1, -1);
|
||||
const __m128i tmp_hi = _mm_shuffle_epi8(out8, kCst_hi);
|
||||
const __m128i tmp_8 = _mm_shuffle_epi8(out8, kCst_8); // extract #8
|
||||
const __m128i out_z0 = _mm_or_si128(tmp_lo, tmp_8);
|
||||
const __m128i out_z8 = _mm_or_si128(tmp_hi, tmp_7);
|
||||
_mm_storeu_si128((__m128i*)&out[0], out_z0);
|
||||
_mm_storeu_si128((__m128i*)&out[8], out_z8);
|
||||
packed_out = _mm_packs_epi16(out_z0, out_z8);
|
||||
}
|
||||
|
||||
// detect if all 'out' values are zeroes or not
|
||||
return (_mm_movemask_epi8(_mm_cmpeq_epi8(packed_out, zero)) != 0xffff);
|
||||
}
|
||||
|
||||
#undef PSHUFB_CST
|
||||
|
||||
static int QuantizeBlock(int16_t in[16], int16_t out[16],
|
||||
const VP8Matrix* const mtx) {
|
||||
return DoQuantizeBlock(in, out, &mtx->sharpen_[0], mtx);
|
||||
}
|
||||
|
||||
static int QuantizeBlockWHT(int16_t in[16], int16_t out[16],
|
||||
const VP8Matrix* const mtx) {
|
||||
return DoQuantizeBlock(in, out, NULL, mtx);
|
||||
}
|
||||
|
||||
static int Quantize2Blocks(int16_t in[32], int16_t out[32],
|
||||
const VP8Matrix* const mtx) {
|
||||
int nz;
|
||||
const uint16_t* const sharpen = &mtx->sharpen_[0];
|
||||
nz = DoQuantizeBlock(in + 0 * 16, out + 0 * 16, sharpen, mtx) << 0;
|
||||
nz |= DoQuantizeBlock(in + 1 * 16, out + 1 * 16, sharpen, mtx) << 1;
|
||||
return nz;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Entry point
|
||||
|
||||
extern void VP8EncDspInitSSE41(void);
|
||||
WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE41(void) {
|
||||
VP8CollectHistogram = CollectHistogram;
|
||||
VP8EncQuantizeBlock = QuantizeBlock;
|
||||
VP8EncQuantize2Blocks = Quantize2Blocks;
|
||||
VP8EncQuantizeBlockWHT = QuantizeBlockWHT;
|
||||
VP8SSE16x16 = SSE16x16;
|
||||
VP8SSE16x8 = SSE16x8;
|
||||
}
|
||||
|
||||
#else // !WEBP_USE_SSE41
|
||||
|
||||
extern void VP8EncDspInitSSE41(void);
|
||||
WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE41(void) {}
|
||||
|
||||
#endif // WEBP_USE_SSE41
|
||||
Loading…
Reference in New Issue
Block a user