generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge changes I783c5f4f,I365f8e53,I5dac0e98

Browse Source 
* changes:
  Clean vpx_idct16x16_256_add_sse2()
  Update vpx_idct{8x8,16x16,32x32}_1_add_sse2()
  Clean 32x32 full idct sse2 and ssse3 code
This commit is contained in:
Linfeng Zhang
2017-06-22 21:42:23 +00:00
committed by Gerrit Code Review
parent 92145006c3 466b667ff3
commit c5f9de573f
6 changed files with 560 additions and 1238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
test/dct_test.cc
View File

@@ -498,18 +498,16 @@ INSTANTIATE_TEST_CASE_P(
// TODO(johannkoenig): high bit depth fdct8x8.
INSTANTIATE_TEST_CASE_P(
SSSE3, TransDCT,
::testing::Values(make_tuple(&vpx_fdct32x32_c,
&vpx_idct32x32_1024_add_ssse3, 32, 0,
VPX_BITS_8),
::testing::Values(make_tuple(&vpx_fdct32x32_c, &vpx_idct32x32_1024_add_sse2,
32, 0, VPX_BITS_8),
make_tuple(&vpx_fdct8x8_c, &vpx_idct8x8_64_add_sse2, 8, 0,
VPX_BITS_8)));
#else
// vpx_fdct8x8_ssse3 is only available in 64 bit builds.
INSTANTIATE_TEST_CASE_P(
SSSE3, TransDCT,
::testing::Values(make_tuple(&vpx_fdct32x32_c,
&vpx_idct32x32_1024_add_ssse3, 32, 0,
VPX_BITS_8),
::testing::Values(make_tuple(&vpx_fdct32x32_c, &vpx_idct32x32_1024_add_sse2,
32, 0, VPX_BITS_8),
make_tuple(&vpx_fdct8x8_ssse3, &vpx_idct8x8_64_add_sse2,
8, 0, VPX_BITS_8)));
#endif // !ARCH_X86_64

2
test/partial_idct_test.cc
View File

@@ -738,8 +738,6 @@ INSTANTIATE_TEST_CASE_P(SSE2, PartialIDctTest,
#if HAVE_SSSE3 && !CONFIG_EMULATE_HARDWARE
const PartialInvTxfmParam ssse3_partial_idct_tests[] = {
make_tuple(&vpx_fdct32x32_c, &wrapper<vpx_idct32x32_1024_add_c>,
&wrapper<vpx_idct32x32_1024_add_ssse3>, TX_32X32, 1024, 8, 1),
make_tuple(&vpx_fdct32x32_c, &wrapper<vpx_idct32x32_135_add_c>,
&wrapper<vpx_idct32x32_135_add_ssse3>, TX_32X32, 135, 8, 1),
make_tuple(&vpx_fdct32x32_c, &wrapper<vpx_idct32x32_34_add_c>,

2
vpx_dsp/vpx_dsp_rtcd_defs.pl
View File

@@ -592,7 +592,7 @@ if (vpx_config("CONFIG_EMULATE_HARDWARE") ne "yes") {
$vpx_idct16x16_38_add_sse2=vpx_idct16x16_256_add_sse2;
specialize qw/vpx_idct16x16_10_add neon sse2/;
specialize qw/vpx_idct16x16_1_add neon sse2/;
specialize qw/vpx_idct32x32_1024_add neon sse2 ssse3/;
specialize qw/vpx_idct32x32_1024_add neon sse2/;
specialize qw/vpx_idct32x32_135_add neon sse2 ssse3/;
$vpx_idct32x32_135_add_sse2=vpx_idct32x32_1024_add_sse2;
specialize qw/vpx_idct32x32_34_add neon sse2 ssse3/;

1337
vpx_dsp/x86/inv_txfm_sse2.c
View File

File diff suppressed because it is too large Load Diff

72
vpx_dsp/x86/inv_txfm_sse2.h
View File

@@ -258,6 +258,78 @@ static INLINE void recon_and_store4x4_sse2(const __m128i *const in,
*(int *)(dest + stride * 3) = _mm_cvtsi128_si32(d[0]);
}
static INLINE void store_buffer_8x32(__m128i *in, uint8_t *dst, int stride) {
const __m128i final_rounding = _mm_set1_epi16(1 << 5);
int j = 0;
while (j < 32) {
in[j] = _mm_adds_epi16(in[j], final_rounding);
in[j + 1] = _mm_adds_epi16(in[j + 1], final_rounding);
in[j] = _mm_srai_epi16(in[j], 6);
in[j + 1] = _mm_srai_epi16(in[j + 1], 6);
recon_and_store(dst, in[j]);
dst += stride;
recon_and_store(dst, in[j + 1]);
dst += stride;
j += 2;
}
}
// Only do addition and subtraction butterfly, size = 16, 32
static INLINE void add_sub_butterfly(const __m128i *in, __m128i *out,
int size) {
int i = 0;
const int num = size >> 1;
const int bound = size - 1;
while (i < num) {
out[i] = _mm_add_epi16(in[i], in[bound - i]);
out[bound - i] = _mm_sub_epi16(in[i], in[bound - i]);
i++;
}
}
#define BUTTERFLY_PAIR(x0, x1, co0, co1) \
do { \
tmp0 = _mm_madd_epi16(x0, co0); \
tmp1 = _mm_madd_epi16(x1, co0); \
tmp2 = _mm_madd_epi16(x0, co1); \
tmp3 = _mm_madd_epi16(x1, co1); \
tmp0 = _mm_add_epi32(tmp0, rounding); \
tmp1 = _mm_add_epi32(tmp1, rounding); \
tmp2 = _mm_add_epi32(tmp2, rounding); \
tmp3 = _mm_add_epi32(tmp3, rounding); \
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
} while (0)
static INLINE void butterfly(const __m128i *x0, const __m128i *x1,
const __m128i *c0, const __m128i *c1, __m128i *y0,
__m128i *y1) {
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
u0 = _mm_unpacklo_epi16(*x0, *x1);
u1 = _mm_unpackhi_epi16(*x0, *x1);
BUTTERFLY_PAIR(u0, u1, *c0, *c1);
*y0 = _mm_packs_epi32(tmp0, tmp1);
*y1 = _mm_packs_epi32(tmp2, tmp3);
}
static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i *c0,
const __m128i *c1) {
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
u0 = _mm_unpacklo_epi16(*x0, *x1);
u1 = _mm_unpackhi_epi16(*x0, *x1);
BUTTERFLY_PAIR(u0, u1, *c0, *c1);
*x0 = _mm_packs_epi32(tmp0, tmp1);
*x1 = _mm_packs_epi32(tmp2, tmp3);
}
void idct4_sse2(__m128i *in);
void idct8_sse2(__m128i *in);
void idct16_sse2(__m128i *in0, __m128i *in1);

375
vpx_dsp/x86/inv_txfm_ssse3.c
View File

@@ -150,60 +150,6 @@ void vpx_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest,
write_buffer_8x8(in, dest, stride);
}
// Only do addition and subtraction butterfly, size = 16, 32
static INLINE void add_sub_butterfly(const __m128i *in, __m128i *out,
int size) {
int i = 0;
const int num = size >> 1;
const int bound = size - 1;
while (i < num) {
out[i] = _mm_add_epi16(in[i], in[bound - i]);
out[bound - i] = _mm_sub_epi16(in[i], in[bound - i]);
i++;
}
}
#define BUTTERFLY_PAIR(x0, x1, co0, co1) \
do { \
tmp0 = _mm_madd_epi16(x0, co0); \
tmp1 = _mm_madd_epi16(x1, co0); \
tmp2 = _mm_madd_epi16(x0, co1); \
tmp3 = _mm_madd_epi16(x1, co1); \
tmp0 = _mm_add_epi32(tmp0, rounding); \
tmp1 = _mm_add_epi32(tmp1, rounding); \
tmp2 = _mm_add_epi32(tmp2, rounding); \
tmp3 = _mm_add_epi32(tmp3, rounding); \
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
} while (0)
static INLINE void butterfly(const __m128i *x0, const __m128i *x1,
const __m128i *c0, const __m128i *c1, __m128i *y0,
__m128i *y1) {
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
u0 = _mm_unpacklo_epi16(*x0, *x1);
u1 = _mm_unpackhi_epi16(*x0, *x1);
BUTTERFLY_PAIR(u0, u1, *c0, *c1);
*y0 = _mm_packs_epi32(tmp0, tmp1);
*y1 = _mm_packs_epi32(tmp2, tmp3);
}
static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i *c0,
const __m128i *c1) {
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
u0 = _mm_unpacklo_epi16(*x0, *x1);
u1 = _mm_unpackhi_epi16(*x0, *x1);
BUTTERFLY_PAIR(u0, u1, *c0, *c1);
*x0 = _mm_packs_epi32(tmp0, tmp1);
*x1 = _mm_packs_epi32(tmp2, tmp3);
}
static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {
const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
@@ -715,24 +661,6 @@ static void idct32_8x32_135(__m128i *in /*in[32]*/) {
add_sub_butterfly(out, in, 32);
}
static INLINE void store_buffer_8x32(__m128i *in, uint8_t *dst, int stride) {
const __m128i final_rounding = _mm_set1_epi16(1 << 5);
int j = 0;
while (j < 32) {
in[j] = _mm_adds_epi16(in[j], final_rounding);
in[j + 1] = _mm_adds_epi16(in[j + 1], final_rounding);
in[j] = _mm_srai_epi16(in[j], 6);
in[j + 1] = _mm_srai_epi16(in[j + 1], 6);
recon_and_store(dst, in[j]);
dst += stride;
recon_and_store(dst, in[j + 1]);
dst += stride;
j += 2;
}
}
static INLINE void recon_and_store_ssse3(__m128i *in0, __m128i *in1,
uint8_t *dest, int stride) {
store_buffer_8x32(in0, dest, stride);
@@ -793,306 +721,3 @@ void vpx_idct32x32_135_add_ssse3(const tran_low_t *input, uint8_t *dest,
idct32_135(col0, col1);
recon_and_store_ssse3(col0, col1, dest + 16, stride);
}
// For each 8x32 block __m128i in[32],
// Input with index, 2, 6, 10, 14, 18, 22, 26, 30
// output pixels: 8-15 in __m128i in[32]
static void idct32_full_8x32_quarter_2(const __m128i *in /*in[32]*/,
__m128i *out /*out[16]*/) {
__m128i u8, u9, u10, u11, u12, u13, u14, u15; // stp2_
__m128i v8, v9, v10, v11, v12, v13, v14, v15; // stp1_
{
const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64);
const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64);
const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64);
const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64);
butterfly(&in[2], &in[30], &stg2_0, &stg2_1, &u8, &u15);
butterfly(&in[18], &in[14], &stg2_2, &stg2_3, &u9, &u14);
}
v8 = _mm_add_epi16(u8, u9);
v9 = _mm_sub_epi16(u8, u9);
v14 = _mm_sub_epi16(u15, u14);
v15 = _mm_add_epi16(u15, u14);
{
const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64);
const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64);
const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64);
const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64);
butterfly(&in[10], &in[22], &stg2_4, &stg2_5, &u10, &u13);
butterfly(&in[26], &in[6], &stg2_6, &stg2_7, &u11, &u12);
}
v10 = _mm_sub_epi16(u11, u10);
v11 = _mm_add_epi16(u11, u10);
v12 = _mm_add_epi16(u12, u13);
v13 = _mm_sub_epi16(u12, u13);
{
const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
butterfly_self(&v9, &v14, &stg4_4, &stg4_5);
butterfly_self(&v10, &v13, &stg4_6, &stg4_4);
}
out[0] = _mm_add_epi16(v8, v11);
out[1] = _mm_add_epi16(v9, v10);
out[6] = _mm_add_epi16(v14, v13);
out[7] = _mm_add_epi16(v15, v12);
out[2] = _mm_sub_epi16(v9, v10);
out[3] = _mm_sub_epi16(v8, v11);
out[4] = _mm_sub_epi16(v15, v12);
out[5] = _mm_sub_epi16(v14, v13);
{
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
butterfly_self(&out[2], &out[5], &stg6_0, &stg4_0);
butterfly_self(&out[3], &out[4], &stg6_0, &stg4_0);
}
}
// For each 8x32 block __m128i in[32],
// Input with index, 0, 4, 8, 12, 16, 20, 24, 28
// output pixels: 0-7 in __m128i in[32]
static void idct32_full_8x32_quarter_1(const __m128i *in /*in[32]*/,
__m128i *out /*out[8]*/) {
__m128i u0, u1, u2, u3, u4, u5, u6, u7; // stp1_
__m128i v0, v1, v2, v3, v4, v5, v6, v7; // stp2_
{
const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64);
const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64);
const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64);
const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64);
butterfly(&in[4], &in[28], &stg3_0, &stg3_1, &u4, &u7);
butterfly(&in[20], &in[12], &stg3_2, &stg3_3, &u5, &u6);
}
v4 = _mm_add_epi16(u4, u5);
v5 = _mm_sub_epi16(u4, u5);
v6 = _mm_sub_epi16(u7, u6);
v7 = _mm_add_epi16(u7, u6);
{
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64);
const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64);
butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6);
butterfly(&in[0], &in[16], &stg4_0, &stg4_1, &u0, &u1);
butterfly(&in[8], &in[24], &stg4_2, &stg4_3, &u2, &u3);
}
v0 = _mm_add_epi16(u0, u3);
v1 = _mm_add_epi16(u1, u2);
v2 = _mm_sub_epi16(u1, u2);
v3 = _mm_sub_epi16(u0, u3);
out[0] = _mm_add_epi16(v0, v7);
out[1] = _mm_add_epi16(v1, v6);
out[2] = _mm_add_epi16(v2, v5);
out[3] = _mm_add_epi16(v3, v4);
out[4] = _mm_sub_epi16(v3, v4);
out[5] = _mm_sub_epi16(v2, v5);
out[6] = _mm_sub_epi16(v1, v6);
out[7] = _mm_sub_epi16(v0, v7);
}
// For each 8x32 block __m128i in[32],
// Input with odd index,
// 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
// output pixels: 16-23, 24-31 in __m128i in[32]
// We avoid hide an offset, 16, inside this function. So we output 0-15 into
// array out[16]
static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/,
__m128i *out /*out[16]*/) {
__m128i v16, v17, v18, v19, v20, v21, v22, v23;
__m128i v24, v25, v26, v27, v28, v29, v30, v31;
__m128i u16, u17, u18, u19, u20, u21, u22, u23;
__m128i u24, u25, u26, u27, u28, u29, u30, u31;
{
const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64);
const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64);
const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64);
const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64);
const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64);
const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64);
const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64);
const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64);
const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64);
const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64);
const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64);
const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64);
const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64);
const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64);
const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64);
const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64);
butterfly(&in[1], &in[31], &stg1_0, &stg1_1, &u16, &u31);
butterfly(&in[17], &in[15], &stg1_2, &stg1_3, &u17, &u30);
butterfly(&in[9], &in[23], &stg1_4, &stg1_5, &u18, &u29);
butterfly(&in[25], &in[7], &stg1_6, &stg1_7, &u19, &u28);
butterfly(&in[5], &in[27], &stg1_8, &stg1_9, &u20, &u27);
butterfly(&in[21], &in[11], &stg1_10, &stg1_11, &u21, &u26);
butterfly(&in[13], &in[19], &stg1_12, &stg1_13, &u22, &u25);
butterfly(&in[29], &in[3], &stg1_14, &stg1_15, &u23, &u24);
}
v16 = _mm_add_epi16(u16, u17);
v17 = _mm_sub_epi16(u16, u17);
v18 = _mm_sub_epi16(u19, u18);
v19 = _mm_add_epi16(u19, u18);
v20 = _mm_add_epi16(u20, u21);
v21 = _mm_sub_epi16(u20, u21);
v22 = _mm_sub_epi16(u23, u22);
v23 = _mm_add_epi16(u23, u22);
v24 = _mm_add_epi16(u24, u25);
v25 = _mm_sub_epi16(u24, u25);
v26 = _mm_sub_epi16(u27, u26);
v27 = _mm_add_epi16(u27, u26);
v28 = _mm_add_epi16(u28, u29);
v29 = _mm_sub_epi16(u28, u29);
v30 = _mm_sub_epi16(u31, u30);
v31 = _mm_add_epi16(u31, u30);
{
const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
butterfly_self(&v17, &v30, &stg3_4, &stg3_5);
butterfly_self(&v18, &v29, &stg3_6, &stg3_4);
butterfly_self(&v21, &v26, &stg3_8, &stg3_9);
butterfly_self(&v22, &v25, &stg3_10, &stg3_8);
}
u16 = _mm_add_epi16(v16, v19);
u17 = _mm_add_epi16(v17, v18);
u18 = _mm_sub_epi16(v17, v18);
u19 = _mm_sub_epi16(v16, v19);
u20 = _mm_sub_epi16(v23, v20);
u21 = _mm_sub_epi16(v22, v21);
u22 = _mm_add_epi16(v22, v21);
u23 = _mm_add_epi16(v23, v20);
u24 = _mm_add_epi16(v24, v27);
u25 = _mm_add_epi16(v25, v26);
u26 = _mm_sub_epi16(v25, v26);
u27 = _mm_sub_epi16(v24, v27);
u28 = _mm_sub_epi16(v31, v28);
u29 = _mm_sub_epi16(v30, v29);
u30 = _mm_add_epi16(v29, v30);
u31 = _mm_add_epi16(v28, v31);
{
const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
butterfly_self(&u18, &u29, &stg4_4, &stg4_5);
butterfly_self(&u19, &u28, &stg4_4, &stg4_5);
butterfly_self(&u20, &u27, &stg4_6, &stg4_4);
butterfly_self(&u21, &u26, &stg4_6, &stg4_4);
}
out[0] = _mm_add_epi16(u16, u23);
out[1] = _mm_add_epi16(u17, u22);
out[2] = _mm_add_epi16(u18, u21);
out[3] = _mm_add_epi16(u19, u20);
out[4] = _mm_sub_epi16(u19, u20);
out[5] = _mm_sub_epi16(u18, u21);
out[6] = _mm_sub_epi16(u17, u22);
out[7] = _mm_sub_epi16(u16, u23);
out[8] = _mm_sub_epi16(u31, u24);
out[9] = _mm_sub_epi16(u30, u25);
out[10] = _mm_sub_epi16(u29, u26);
out[11] = _mm_sub_epi16(u28, u27);
out[12] = _mm_add_epi16(u27, u28);
out[13] = _mm_add_epi16(u26, u29);
out[14] = _mm_add_epi16(u25, u30);
out[15] = _mm_add_epi16(u24, u31);
{
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
butterfly_self(&out[4], &out[11], &stg6_0, &stg4_0);
butterfly_self(&out[5], &out[10], &stg6_0, &stg4_0);
butterfly_self(&out[6], &out[9], &stg6_0, &stg4_0);
butterfly_self(&out[7], &out[8], &stg6_0, &stg4_0);
}
}
static void idct32_full_8x32_quarter_1_2(const __m128i *in /*in[32]*/,
__m128i *out /*out[32]*/) {
__m128i temp[16];
idct32_full_8x32_quarter_1(in, temp);
idct32_full_8x32_quarter_2(in, &temp[8]);
add_sub_butterfly(temp, out, 16);
}
static void idct32_full_8x32(const __m128i *in /*in[32]*/,
__m128i *out /*out[32]*/) {
__m128i temp[32];
idct32_full_8x32_quarter_1_2(in, temp);
idct32_full_8x32_quarter_3_4(in, &temp[16]);
add_sub_butterfly(temp, out, 32);
}
static void load_buffer_8x32(const tran_low_t *input, __m128i *in) {
int i;
for (i = 0; i < 8; ++i) {
in[i] = load_input_data(input);
in[i + 8] = load_input_data(input + 8);
in[i + 16] = load_input_data(input + 16);
in[i + 24] = load_input_data(input + 24);
input += 32;
}
}
void vpx_idct32x32_1024_add_ssse3(const tran_low_t *input, uint8_t *dest,
int stride) {
__m128i col[128], in[32];
int i, j;
// rows
for (i = 0; i < 4; ++i) {
load_buffer_8x32(input, in);
input += 32 << 3;
// Transpose 32x8 block to 8x32 block
transpose_16bit_8x8(in, in);
transpose_16bit_8x8(in + 8, in + 8);
transpose_16bit_8x8(in + 16, in + 16);
transpose_16bit_8x8(in + 24, in + 24);
idct32_full_8x32(in, col + (i << 5));
}
// columns
for (i = 0; i < 4; ++i) {
j = i << 3;
// Transpose 32x8 block to 8x32 block
transpose_16bit_8x8(col + j, in);
transpose_16bit_8x8(col + j + 32, in + 8);
transpose_16bit_8x8(col + j + 64, in + 16);
transpose_16bit_8x8(col + j + 96, in + 24);
idct32_full_8x32(in, in);
store_buffer_8x32(in, dest, stride);
dest += 8;
}
}