generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "Replace idct32x32_34_add_ssse3 assembly with intrinsics"

Browse Source
This commit is contained in:
Yi Luo
2017-02-14 20:13:26 +00:00
committed by Gerrit Code Review
parent 25301a84a8 bd86de1ac8
commit c1a90dc160
5 changed files with 468 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vpx_dsp/vpx_dsp_rtcd_defs.pl
View File

@@ -806,7 +806,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
$vpx_idct32x32_135_add_msa=vpx_idct32x32_1024_add_msa;
add_proto qw/void vpx_idct32x32_34_add/, "const tran_low_t *input, uint8_t *dest, int stride";
specialize qw/vpx_idct32x32_34_add sse2 neon dspr2 msa/, "$ssse3_x86_64";
specialize qw/vpx_idct32x32_34_add sse2 ssse3 neon dspr2 msa/;
add_proto qw/void vpx_idct32x32_1_add/, "const tran_low_t *input, uint8_t *dest, int stride";
specialize qw/vpx_idct32x32_1_add sse2 neon dspr2 msa/;

37
vpx_dsp/x86/inv_txfm_sse2.c
View File

@@ -263,43 +263,6 @@ void iadst4_sse2(__m128i *in) {
in[1] = _mm_packs_epi32(u[2], u[3]);
}
// Define Macro for multiplying elements by constants and adding them together.
#define MULTIPLICATION_AND_ADD(lo_0, hi_0, lo_1, hi_1, cst0, cst1, cst2, cst3, \
res0, res1, res2, res3) \
{ \
tmp0 = _mm_madd_epi16(lo_0, cst0); \
tmp1 = _mm_madd_epi16(hi_0, cst0); \
tmp2 = _mm_madd_epi16(lo_0, cst1); \
tmp3 = _mm_madd_epi16(hi_0, cst1); \
tmp4 = _mm_madd_epi16(lo_1, cst2); \
tmp5 = _mm_madd_epi16(hi_1, cst2); \
tmp6 = _mm_madd_epi16(lo_1, cst3); \
tmp7 = _mm_madd_epi16(hi_1, cst3); \
\
tmp0 = _mm_add_epi32(tmp0, rounding); \
tmp1 = _mm_add_epi32(tmp1, rounding); \
tmp2 = _mm_add_epi32(tmp2, rounding); \
tmp3 = _mm_add_epi32(tmp3, rounding); \
tmp4 = _mm_add_epi32(tmp4, rounding); \
tmp5 = _mm_add_epi32(tmp5, rounding); \
tmp6 = _mm_add_epi32(tmp6, rounding); \
tmp7 = _mm_add_epi32(tmp7, rounding); \
\
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); \
tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); \
tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); \
tmp7 = _mm_srai_epi32(tmp7, DCT_CONST_BITS); \
\
res0 = _mm_packs_epi32(tmp0, tmp1); \
res1 = _mm_packs_epi32(tmp2, tmp3); \
res2 = _mm_packs_epi32(tmp4, tmp5); \
res3 = _mm_packs_epi32(tmp6, tmp7); \
}
#define MULTIPLICATION_AND_ADD_2(lo_0, hi_0, cst0, cst1, res0, res1) \
{ \
tmp0 = _mm_madd_epi16(lo_0, cst0); \

37
vpx_dsp/x86/inv_txfm_sse2.h
View File

@@ -242,6 +242,43 @@ static INLINE void write_buffer_8x16(uint8_t *dest, __m128i *in, int stride) {
out1 = _mm_unpackhi_epi32(tr0_0, tr0_1); \
}
// Define Macro for multiplying elements by constants and adding them together.
#define MULTIPLICATION_AND_ADD(lo_0, hi_0, lo_1, hi_1, cst0, cst1, cst2, cst3, \
res0, res1, res2, res3) \
{ \
tmp0 = _mm_madd_epi16(lo_0, cst0); \
tmp1 = _mm_madd_epi16(hi_0, cst0); \
tmp2 = _mm_madd_epi16(lo_0, cst1); \
tmp3 = _mm_madd_epi16(hi_0, cst1); \
tmp4 = _mm_madd_epi16(lo_1, cst2); \
tmp5 = _mm_madd_epi16(hi_1, cst2); \
tmp6 = _mm_madd_epi16(lo_1, cst3); \
tmp7 = _mm_madd_epi16(hi_1, cst3); \
\
tmp0 = _mm_add_epi32(tmp0, rounding); \
tmp1 = _mm_add_epi32(tmp1, rounding); \
tmp2 = _mm_add_epi32(tmp2, rounding); \
tmp3 = _mm_add_epi32(tmp3, rounding); \
tmp4 = _mm_add_epi32(tmp4, rounding); \
tmp5 = _mm_add_epi32(tmp5, rounding); \
tmp6 = _mm_add_epi32(tmp6, rounding); \
tmp7 = _mm_add_epi32(tmp7, rounding); \
\
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); \
tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); \
tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); \
tmp7 = _mm_srai_epi32(tmp7, DCT_CONST_BITS); \
\
res0 = _mm_packs_epi32(tmp0, tmp1); \
res1 = _mm_packs_epi32(tmp2, tmp3); \
res2 = _mm_packs_epi32(tmp4, tmp5); \
res3 = _mm_packs_epi32(tmp6, tmp7); \
}
void idct4_sse2(__m128i *in);
void idct8_sse2(__m128i *in);
void idct16_sse2(__m128i *in0, __m128i *in1);

430
vpx_dsp/x86/inv_txfm_ssse3.c
View File

@@ -322,3 +322,433 @@ void vpx_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest,
RECON_AND_STORE(dest + 6 * stride, in6);
RECON_AND_STORE(dest + 7 * stride, in7);
}
static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
// idct constants for each stage
const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
__m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15,
stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23,
stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
/* Stage1 */
stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0);
stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1);
stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6);
stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7);
stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8);
stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9);
stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14);
stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15);
/* Stage2 */
stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0);
stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1);
stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6);
stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7);
/* Stage3 */
{
const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]);
const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]);
const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]);
const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]);
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]);
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]);
const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]);
const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]);
stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0);
stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1);
MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4,
stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18],
stp1[29])
MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8,
stg3_9, stg3_10, stg3_8, stp1[21], stp1[26],
stp1[22], stp1[25])
}
/* Stage4 */
{
const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15);
const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15);
const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12);
const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12);
stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0);
stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0
stp1[2] = stp1[0];
stp1[3] = stp1[1];
MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5,
stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13)
stp2_16 = _mm_add_epi16(stp1[16], stp1[19]);
stp2_17 = _mm_add_epi16(stp1[17], stp1[18]);
stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]);
stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]);
stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]);
stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]);
stp2_22 = _mm_add_epi16(stp1[22], stp1[21]);
stp2_23 = _mm_add_epi16(stp1[23], stp1[20]);
stp2_24 = _mm_add_epi16(stp1[24], stp1[27]);
stp2_25 = _mm_add_epi16(stp1[25], stp1[26]);
stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]);
stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]);
stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]);
stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]);
stp2_30 = _mm_add_epi16(stp1[29], stp1[30]);
stp2_31 = _mm_add_epi16(stp1[28], stp1[31]);
}
/* Stage5 */
{
// Note:
// #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass
// SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop
// to 23198 and -23197, respectively.
#define AVOID_OVERFLOW (1)
#if AVOID_OVERFLOW
const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]);
const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]);
#endif
const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28);
const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28);
const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
#if AVOID_OVERFLOW
tmp0 = _mm_madd_epi16(lo_6_5, stg4_1);
tmp1 = _mm_madd_epi16(hi_6_5, stg4_1);
tmp2 = _mm_madd_epi16(lo_6_5, stg4_0);
tmp3 = _mm_madd_epi16(hi_6_5, stg4_0);
tmp0 = _mm_add_epi32(tmp0, rounding);
tmp1 = _mm_add_epi32(tmp1, rounding);
tmp2 = _mm_add_epi32(tmp2, rounding);
tmp3 = _mm_add_epi32(tmp3, rounding);
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
stp1[5] = _mm_packs_epi32(tmp0, tmp1);
stp1[6] = _mm_packs_epi32(tmp2, tmp3);
#else
tmp0 = _mm_sub_epi16(stp1[7], stp1[4]);
tmp1 = _mm_adds_epi16(stp1[7], stp1[4]);
stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0);
stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0);
#endif
stp1[8] = _mm_add_epi16(stp2_8, stp2_11);
stp1[9] = _mm_add_epi16(stp2_9, stp2_10);
stp1[10] = _mm_sub_epi16(stp2_9, stp2_10);
stp1[11] = _mm_sub_epi16(stp2_8, stp2_11);
stp1[12] = _mm_sub_epi16(stp2_15, stp2_12);
stp1[13] = _mm_sub_epi16(stp2_14, stp2_13);
stp1[14] = _mm_add_epi16(stp2_14, stp2_13);
stp1[15] = _mm_add_epi16(stp2_15, stp2_12);
MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4,
stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19],
stp1[28])
MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6,
stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21],
stp1[26])
stp1[16] = stp2_16;
stp1[17] = stp2_17;
stp1[22] = stp2_22;
stp1[23] = stp2_23;
stp1[24] = stp2_24;
stp1[25] = stp2_25;
stp1[30] = stp2_30;
stp1[31] = stp2_31;
}
/* Stage6 */
{
#if AVOID_OVERFLOW
const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]);
const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]);
const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]);
const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]);
#endif
stp2_0 = _mm_add_epi16(stp1[0], stp1[7]);
stp2_1 = _mm_add_epi16(stp1[1], stp1[6]);
stp2_2 = _mm_add_epi16(stp1[2], stp1[5]);
stp2_3 = _mm_add_epi16(stp1[3], stp1[4]);
stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]);
stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]);
stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]);
stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]);
stp2_8 = stp1[8];
stp2_9 = stp1[9];
stp2_14 = stp1[14];
stp2_15 = stp1[15];
#if AVOID_OVERFLOW
MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0,
stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11,
stp2_12)
#else
tmp0 = _mm_add_epi16(stp1[10], stp1[13]);
tmp1 = _mm_sub_epi16(stp1[13], stp1[10]);
tmp2 = _mm_add_epi16(stp1[11], stp1[12]);
tmp3 = _mm_sub_epi16(stp1[12], stp1[11]);
stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0);
stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0);
stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0);
stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0);
#endif
stp2_16 = _mm_add_epi16(stp1[16], stp1[23]);
stp2_17 = _mm_add_epi16(stp1[17], stp1[22]);
stp2_18 = _mm_add_epi16(stp1[18], stp1[21]);
stp2_19 = _mm_add_epi16(stp1[19], stp1[20]);
stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]);
stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]);
stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]);
stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]);
stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]);
stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]);
stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]);
stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]);
stp2_28 = _mm_add_epi16(stp1[27], stp1[28]);
stp2_29 = _mm_add_epi16(stp1[26], stp1[29]);
stp2_30 = _mm_add_epi16(stp1[25], stp1[30]);
stp2_31 = _mm_add_epi16(stp1[24], stp1[31]);
}
/* Stage7 */
{
#if AVOID_OVERFLOW
const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24);
const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24);
#endif
stp1[0] = _mm_add_epi16(stp2_0, stp2_15);
stp1[1] = _mm_add_epi16(stp2_1, stp2_14);
stp1[2] = _mm_add_epi16(stp2_2, stp2_13);
stp1[3] = _mm_add_epi16(stp2_3, stp2_12);
stp1[4] = _mm_add_epi16(stp2_4, stp2_11);
stp1[5] = _mm_add_epi16(stp2_5, stp2_10);
stp1[6] = _mm_add_epi16(stp2_6, stp2_9);
stp1[7] = _mm_add_epi16(stp2_7, stp2_8);
stp1[8] = _mm_sub_epi16(stp2_7, stp2_8);
stp1[9] = _mm_sub_epi16(stp2_6, stp2_9);
stp1[10] = _mm_sub_epi16(stp2_5, stp2_10);
stp1[11] = _mm_sub_epi16(stp2_4, stp2_11);
stp1[12] = _mm_sub_epi16(stp2_3, stp2_12);
stp1[13] = _mm_sub_epi16(stp2_2, stp2_13);
stp1[14] = _mm_sub_epi16(stp2_1, stp2_14);
stp1[15] = _mm_sub_epi16(stp2_0, stp2_15);
stp1[16] = stp2_16;
stp1[17] = stp2_17;
stp1[18] = stp2_18;
stp1[19] = stp2_19;
#if AVOID_OVERFLOW
MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0,
stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21],
stp1[26])
MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0,
stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23],
stp1[24])
#else
tmp0 = _mm_add_epi16(stp2_20, stp2_27);
tmp1 = _mm_sub_epi16(stp2_27, stp2_20);
tmp2 = _mm_add_epi16(stp2_21, stp2_26);
tmp3 = _mm_sub_epi16(stp2_26, stp2_21);
stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0);
stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0);
stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0);
stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0);
tmp0 = _mm_add_epi16(stp2_22, stp2_25);
tmp1 = _mm_sub_epi16(stp2_25, stp2_22);
tmp2 = _mm_add_epi16(stp2_23, stp2_24);
tmp3 = _mm_sub_epi16(stp2_24, stp2_23);
stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0);
stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0);
stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0);
stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0);
#endif
stp1[28] = stp2_28;
stp1[29] = stp2_29;
stp1[30] = stp2_30;
stp1[31] = stp2_31;
}
#undef AVOID_OVERFLOW
}
// Only upper-left 8x8 has non-zero coeff
void vpx_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest,
int stride) {
const __m128i zero = _mm_setzero_si128();
const __m128i final_rounding = _mm_set1_epi16(1 << 5);
__m128i in[32], col[32];
__m128i stp1[32];
int i;
// Load input data. Only need to load the top left 8x8 block.
in[0] = load_input_data(input);
in[1] = load_input_data(input + 32);
in[2] = load_input_data(input + 64);
in[3] = load_input_data(input + 96);
in[4] = load_input_data(input + 128);
in[5] = load_input_data(input + 160);
in[6] = load_input_data(input + 192);
in[7] = load_input_data(input + 224);
array_transpose_8x8(in, in);
idct32_34(in, stp1);
// 1_D: Store 32 intermediate results for each 8x32 block.
col[0] = _mm_add_epi16(stp1[0], stp1[31]);
col[1] = _mm_add_epi16(stp1[1], stp1[30]);
col[2] = _mm_add_epi16(stp1[2], stp1[29]);
col[3] = _mm_add_epi16(stp1[3], stp1[28]);
col[4] = _mm_add_epi16(stp1[4], stp1[27]);
col[5] = _mm_add_epi16(stp1[5], stp1[26]);
col[6] = _mm_add_epi16(stp1[6], stp1[25]);
col[7] = _mm_add_epi16(stp1[7], stp1[24]);
col[8] = _mm_add_epi16(stp1[8], stp1[23]);
col[9] = _mm_add_epi16(stp1[9], stp1[22]);
col[10] = _mm_add_epi16(stp1[10], stp1[21]);
col[11] = _mm_add_epi16(stp1[11], stp1[20]);
col[12] = _mm_add_epi16(stp1[12], stp1[19]);
col[13] = _mm_add_epi16(stp1[13], stp1[18]);
col[14] = _mm_add_epi16(stp1[14], stp1[17]);
col[15] = _mm_add_epi16(stp1[15], stp1[16]);
col[16] = _mm_sub_epi16(stp1[15], stp1[16]);
col[17] = _mm_sub_epi16(stp1[14], stp1[17]);
col[18] = _mm_sub_epi16(stp1[13], stp1[18]);
col[19] = _mm_sub_epi16(stp1[12], stp1[19]);
col[20] = _mm_sub_epi16(stp1[11], stp1[20]);
col[21] = _mm_sub_epi16(stp1[10], stp1[21]);
col[22] = _mm_sub_epi16(stp1[9], stp1[22]);
col[23] = _mm_sub_epi16(stp1[8], stp1[23]);
col[24] = _mm_sub_epi16(stp1[7], stp1[24]);
col[25] = _mm_sub_epi16(stp1[6], stp1[25]);
col[26] = _mm_sub_epi16(stp1[5], stp1[26]);
col[27] = _mm_sub_epi16(stp1[4], stp1[27]);
col[28] = _mm_sub_epi16(stp1[3], stp1[28]);
col[29] = _mm_sub_epi16(stp1[2], stp1[29]);
col[30] = _mm_sub_epi16(stp1[1], stp1[30]);
col[31] = _mm_sub_epi16(stp1[0], stp1[31]);
for (i = 0; i < 4; i++) {
int j;
// Transpose 32x8 block to 8x32 block
array_transpose_8x8(col + i * 8, in);
idct32_34(in, stp1);
// 2_D: Calculate the results and store them to destination.
in[0] = _mm_add_epi16(stp1[0], stp1[31]);
in[1] = _mm_add_epi16(stp1[1], stp1[30]);
in[2] = _mm_add_epi16(stp1[2], stp1[29]);
in[3] = _mm_add_epi16(stp1[3], stp1[28]);
in[4] = _mm_add_epi16(stp1[4], stp1[27]);
in[5] = _mm_add_epi16(stp1[5], stp1[26]);
in[6] = _mm_add_epi16(stp1[6], stp1[25]);
in[7] = _mm_add_epi16(stp1[7], stp1[24]);
in[8] = _mm_add_epi16(stp1[8], stp1[23]);
in[9] = _mm_add_epi16(stp1[9], stp1[22]);
in[10] = _mm_add_epi16(stp1[10], stp1[21]);
in[11] = _mm_add_epi16(stp1[11], stp1[20]);
in[12] = _mm_add_epi16(stp1[12], stp1[19]);
in[13] = _mm_add_epi16(stp1[13], stp1[18]);
in[14] = _mm_add_epi16(stp1[14], stp1[17]);
in[15] = _mm_add_epi16(stp1[15], stp1[16]);
in[16] = _mm_sub_epi16(stp1[15], stp1[16]);
in[17] = _mm_sub_epi16(stp1[14], stp1[17]);
in[18] = _mm_sub_epi16(stp1[13], stp1[18]);
in[19] = _mm_sub_epi16(stp1[12], stp1[19]);
in[20] = _mm_sub_epi16(stp1[11], stp1[20]);
in[21] = _mm_sub_epi16(stp1[10], stp1[21]);
in[22] = _mm_sub_epi16(stp1[9], stp1[22]);
in[23] = _mm_sub_epi16(stp1[8], stp1[23]);
in[24] = _mm_sub_epi16(stp1[7], stp1[24]);
in[25] = _mm_sub_epi16(stp1[6], stp1[25]);
in[26] = _mm_sub_epi16(stp1[5], stp1[26]);
in[27] = _mm_sub_epi16(stp1[4], stp1[27]);
in[28] = _mm_sub_epi16(stp1[3], stp1[28]);
in[29] = _mm_sub_epi16(stp1[2], stp1[29]);
in[30] = _mm_sub_epi16(stp1[1], stp1[30]);
in[31] = _mm_sub_epi16(stp1[0], stp1[31]);
for (j = 0; j < 32; ++j) {
// Final rounding and shift
in[j] = _mm_adds_epi16(in[j], final_rounding);
in[j] = _mm_srai_epi16(in[j], 6);
RECON_AND_STORE(dest + j * stride, in[j]);
}
dest += 8;
}
}

55
vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
View File

@@ -617,61 +617,6 @@ SECTION .text
%define transposed_in 16*32*4
%define pass_one_start 16*32*0
%define stp r8
INIT_XMM ssse3
cglobal idct32x32_34_add, 3, 11, 16, i32x32_size, input, output, stride
mova m8, [pd_8192]
lea stp, [rsp + pass_one_start]
idct32x32_34:
mov r3, inputq
lea r4, [rsp + transposed_in]
idct32x32_34_transpose:
LOAD_TRAN_LOW 0, r3, 0
LOAD_TRAN_LOW 1, r3, 4
LOAD_TRAN_LOW 2, r3, 8
LOAD_TRAN_LOW 3, r3, 12
LOAD_TRAN_LOW 4, r3, 16
LOAD_TRAN_LOW 5, r3, 20
LOAD_TRAN_LOW 6, r3, 24
LOAD_TRAN_LOW 7, r3, 28
TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9, 10
IDCT32X32_34 16*0, 16*32, 16*64, 16*96
lea stp, [stp + 16 * 8]
mov r6, 4
lea stp, [rsp + pass_one_start]
lea r9, [rsp + pass_one_start]
idct32x32_34_2:
lea r4, [rsp + transposed_in]
mov r3, r9
idct32x32_34_transpose_2:
mova m0, [r3 + 0]
mova m1, [r3 + 16 * 1]
mova m2, [r3 + 16 * 2]
mova m3, [r3 + 16 * 3]
mova m4, [r3 + 16 * 4]
mova m5, [r3 + 16 * 5]
mova m6, [r3 + 16 * 6]
mova m7, [r3 + 16 * 7]
TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9, 10
IDCT32X32_34 16*0, 16*8, 16*16, 16*24
lea stp, [stp + 16 * 32]
add r9, 16 * 32
dec r6
jnz idct32x32_34_2
RECON_AND_STORE pass_two_start
RET
%macro IDCT32X32_135 4
; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mova m1, [rsp + transposed_in + 16 * 1]