Improve idct32x32_34_add SSSE3 intrinsics performance
- Split the transform into first half and second half. - Reschedule the instructions to avoid stack spillover. - Function level speed improves ~16%. Change-Id: I166889840d23aa8a273eca00f6fbdae8b4566f35
This commit is contained in:
Yi Luo
@@ -378,18 +378,71 @@ void vpx_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest,
|
||||
RECON_AND_STORE(dest + 7 * stride, in7);
|
||||
}
|
||||
|
||||
static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {
|
||||
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
|
||||
// idct constants for each stage
|
||||
const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
|
||||
const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
|
||||
const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
|
||||
const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
|
||||
const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
|
||||
const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
|
||||
const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
|
||||
const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
|
||||
static INLINE void simple_group_butterfly(const __m128i *in, __m128i *out) {
|
||||
int i = 0;
|
||||
while (i < 16) {
|
||||
out[i] = _mm_add_epi16(in[i], in[31 - i]);
|
||||
out[31 - i] = _mm_sub_epi16(in[i], in[31 - i]);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void butterfly(const __m128i x0, const __m128i x1,
|
||||
const __m128i c0, const __m128i c1, __m128i *y0,
|
||||
__m128i *y1) {
|
||||
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
|
||||
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
|
||||
|
||||
u0 = _mm_unpacklo_epi16(x0, x1);
|
||||
u1 = _mm_unpackhi_epi16(x0, x1);
|
||||
|
||||
tmp0 = _mm_madd_epi16(u0, c0);
|
||||
tmp1 = _mm_madd_epi16(u1, c0);
|
||||
tmp2 = _mm_madd_epi16(u0, c1);
|
||||
tmp3 = _mm_madd_epi16(u1, c1);
|
||||
|
||||
tmp0 = _mm_add_epi32(tmp0, rounding);
|
||||
tmp1 = _mm_add_epi32(tmp1, rounding);
|
||||
tmp2 = _mm_add_epi32(tmp2, rounding);
|
||||
tmp3 = _mm_add_epi32(tmp3, rounding);
|
||||
|
||||
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
|
||||
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
|
||||
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
|
||||
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
|
||||
|
||||
*y0 = _mm_packs_epi32(tmp0, tmp1);
|
||||
*y1 = _mm_packs_epi32(tmp2, tmp3);
|
||||
}
|
||||
|
||||
static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i c0,
|
||||
const __m128i c1) {
|
||||
__m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
|
||||
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
|
||||
|
||||
u0 = _mm_unpacklo_epi16(*x0, *x1);
|
||||
u1 = _mm_unpackhi_epi16(*x0, *x1);
|
||||
|
||||
tmp0 = _mm_madd_epi16(u0, c0);
|
||||
tmp1 = _mm_madd_epi16(u1, c0);
|
||||
tmp2 = _mm_madd_epi16(u0, c1);
|
||||
tmp3 = _mm_madd_epi16(u1, c1);
|
||||
|
||||
tmp0 = _mm_add_epi32(tmp0, rounding);
|
||||
tmp1 = _mm_add_epi32(tmp1, rounding);
|
||||
tmp2 = _mm_add_epi32(tmp2, rounding);
|
||||
tmp3 = _mm_add_epi32(tmp3, rounding);
|
||||
|
||||
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
|
||||
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
|
||||
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
|
||||
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
|
||||
|
||||
*x0 = _mm_packs_epi32(tmp0, tmp1);
|
||||
*x1 = _mm_packs_epi32(tmp2, tmp3);
|
||||
}
|
||||
|
||||
static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {
|
||||
const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
|
||||
const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
|
||||
const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
|
||||
@@ -397,12 +450,6 @@ static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {
|
||||
|
||||
const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
|
||||
const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
|
||||
const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
|
||||
const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
|
||||
const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
|
||||
const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
|
||||
const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
|
||||
const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
|
||||
|
||||
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
|
||||
const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
|
||||
@@ -412,294 +459,170 @@ static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {
|
||||
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
|
||||
|
||||
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
|
||||
__m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
|
||||
stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15,
|
||||
stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23,
|
||||
stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31;
|
||||
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
|
||||
__m128i u0, u1, u2, u3, u4, u5, u6, u7;
|
||||
__m128i x0, x1, x4, x5, x6, x7;
|
||||
__m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15;
|
||||
|
||||
/* Stage1 */
|
||||
// phase 1
|
||||
|
||||
stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0);
|
||||
stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1);
|
||||
// 0, 15
|
||||
u2 = _mm_mulhrs_epi16(in[2], stk2_1); // stp2_15
|
||||
u3 = _mm_mulhrs_epi16(in[6], stk2_7); // stp2_12
|
||||
v15 = _mm_add_epi16(u2, u3);
|
||||
// in[0], in[4]
|
||||
x0 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[0]
|
||||
x7 = _mm_mulhrs_epi16(in[4], stk3_1); // stp1[7]
|
||||
v0 = _mm_add_epi16(x0, x7); // stp2_0
|
||||
stp1[0] = _mm_add_epi16(v0, v15);
|
||||
stp1[15] = _mm_sub_epi16(v0, v15);
|
||||
|
||||
stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6);
|
||||
stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7);
|
||||
// in[2], in[6]
|
||||
u0 = _mm_mulhrs_epi16(in[2], stk2_0); // stp2_8
|
||||
u1 = _mm_mulhrs_epi16(in[6], stk2_6); // stp2_11
|
||||
butterfly(u0, u2, stg4_4, stg4_5, &u4, &u5); // stp2_9, stp2_14
|
||||
butterfly(u1, u3, stg4_6, stg4_4, &u6, &u7); // stp2_10, stp2_13
|
||||
|
||||
stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8);
|
||||
stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9);
|
||||
v8 = _mm_add_epi16(u0, u1);
|
||||
v9 = _mm_add_epi16(u4, u6);
|
||||
v10 = _mm_sub_epi16(u4, u6);
|
||||
v11 = _mm_sub_epi16(u0, u1);
|
||||
v12 = _mm_sub_epi16(u2, u3);
|
||||
v13 = _mm_sub_epi16(u5, u7);
|
||||
v14 = _mm_add_epi16(u5, u7);
|
||||
|
||||
stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14);
|
||||
stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15);
|
||||
butterfly_self(&v10, &v13, stg6_0, stg4_0);
|
||||
butterfly_self(&v11, &v12, stg6_0, stg4_0);
|
||||
|
||||
/* Stage2 */
|
||||
// 1, 14
|
||||
x1 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[1], stk4_1 = stk4_0
|
||||
// stp1[2] = stp1[0], stp1[3] = stp1[1]
|
||||
x4 = _mm_mulhrs_epi16(in[4], stk3_0); // stp1[4]
|
||||
butterfly(x7, x4, stg4_1, stg4_0, &x5, &x6);
|
||||
v1 = _mm_add_epi16(x1, x6); // stp2_1
|
||||
v2 = _mm_add_epi16(x0, x5); // stp2_2
|
||||
stp1[1] = _mm_add_epi16(v1, v14);
|
||||
stp1[14] = _mm_sub_epi16(v1, v14);
|
||||
|
||||
stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0);
|
||||
stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1);
|
||||
stp1[2] = _mm_add_epi16(v2, v13);
|
||||
stp1[13] = _mm_sub_epi16(v2, v13);
|
||||
|
||||
stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6);
|
||||
stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7);
|
||||
v3 = _mm_add_epi16(x1, x4); // stp2_3
|
||||
v4 = _mm_sub_epi16(x1, x4); // stp2_4
|
||||
|
||||
/* Stage3 */
|
||||
{
|
||||
const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]);
|
||||
const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]);
|
||||
const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]);
|
||||
const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]);
|
||||
v5 = _mm_sub_epi16(x0, x5); // stp2_5
|
||||
|
||||
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]);
|
||||
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]);
|
||||
const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]);
|
||||
const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]);
|
||||
v6 = _mm_sub_epi16(x1, x6); // stp2_6
|
||||
v7 = _mm_sub_epi16(x0, x7); // stp2_7
|
||||
stp1[3] = _mm_add_epi16(v3, v12);
|
||||
stp1[12] = _mm_sub_epi16(v3, v12);
|
||||
|
||||
stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0);
|
||||
stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1);
|
||||
stp1[6] = _mm_add_epi16(v6, v9);
|
||||
stp1[9] = _mm_sub_epi16(v6, v9);
|
||||
|
||||
MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4,
|
||||
stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18],
|
||||
stp1[29])
|
||||
MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8,
|
||||
stg3_9, stg3_10, stg3_8, stp1[21], stp1[26],
|
||||
stp1[22], stp1[25])
|
||||
}
|
||||
stp1[7] = _mm_add_epi16(v7, v8);
|
||||
stp1[8] = _mm_sub_epi16(v7, v8);
|
||||
|
||||
/* Stage4 */
|
||||
{
|
||||
const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15);
|
||||
const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15);
|
||||
const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12);
|
||||
const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12);
|
||||
stp1[4] = _mm_add_epi16(v4, v11);
|
||||
stp1[11] = _mm_sub_epi16(v4, v11);
|
||||
|
||||
stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0);
|
||||
stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0
|
||||
stp1[2] = stp1[0];
|
||||
stp1[3] = stp1[1];
|
||||
stp1[5] = _mm_add_epi16(v5, v10);
|
||||
stp1[10] = _mm_sub_epi16(v5, v10);
|
||||
}
|
||||
|
||||
MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5,
|
||||
stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13)
|
||||
static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {
|
||||
const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
|
||||
const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
|
||||
const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
|
||||
const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
|
||||
const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
|
||||
const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
|
||||
const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
|
||||
const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
|
||||
const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
|
||||
const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
|
||||
const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
|
||||
const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
|
||||
const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
|
||||
const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
|
||||
|
||||
stp2_16 = _mm_add_epi16(stp1[16], stp1[19]);
|
||||
stp2_17 = _mm_add_epi16(stp1[17], stp1[18]);
|
||||
stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]);
|
||||
stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]);
|
||||
stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]);
|
||||
stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]);
|
||||
stp2_22 = _mm_add_epi16(stp1[22], stp1[21]);
|
||||
stp2_23 = _mm_add_epi16(stp1[23], stp1[20]);
|
||||
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
|
||||
const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
|
||||
const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
|
||||
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
|
||||
|
||||
stp2_24 = _mm_add_epi16(stp1[24], stp1[27]);
|
||||
stp2_25 = _mm_add_epi16(stp1[25], stp1[26]);
|
||||
stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]);
|
||||
stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]);
|
||||
stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]);
|
||||
stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]);
|
||||
stp2_30 = _mm_add_epi16(stp1[29], stp1[30]);
|
||||
stp2_31 = _mm_add_epi16(stp1[28], stp1[31]);
|
||||
}
|
||||
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
|
||||
__m128i v16, v17, v18, v19, v20, v21, v22, v23;
|
||||
__m128i v24, v25, v26, v27, v28, v29, v30, v31;
|
||||
__m128i u16, u17, u18, u19, u20, u21, u22, u23;
|
||||
__m128i u24, u25, u26, u27, u28, u29, u30, u31;
|
||||
|
||||
/* Stage5 */
|
||||
{
|
||||
// Note:
|
||||
// #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass
|
||||
// SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop
|
||||
// to 23198 and -23197, respectively.
|
||||
#define AVOID_OVERFLOW (1)
|
||||
v16 = _mm_mulhrs_epi16(in[1], stk1_0);
|
||||
v31 = _mm_mulhrs_epi16(in[1], stk1_1);
|
||||
|
||||
#if AVOID_OVERFLOW
|
||||
const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]);
|
||||
const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]);
|
||||
#endif
|
||||
const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
|
||||
const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
|
||||
v19 = _mm_mulhrs_epi16(in[7], stk1_6);
|
||||
v28 = _mm_mulhrs_epi16(in[7], stk1_7);
|
||||
|
||||
const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28);
|
||||
const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28);
|
||||
const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
|
||||
const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
|
||||
v20 = _mm_mulhrs_epi16(in[5], stk1_8);
|
||||
v27 = _mm_mulhrs_epi16(in[5], stk1_9);
|
||||
|
||||
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
|
||||
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
|
||||
v23 = _mm_mulhrs_epi16(in[3], stk1_14);
|
||||
v24 = _mm_mulhrs_epi16(in[3], stk1_15);
|
||||
|
||||
#if AVOID_OVERFLOW
|
||||
tmp0 = _mm_madd_epi16(lo_6_5, stg4_1);
|
||||
tmp1 = _mm_madd_epi16(hi_6_5, stg4_1);
|
||||
tmp2 = _mm_madd_epi16(lo_6_5, stg4_0);
|
||||
tmp3 = _mm_madd_epi16(hi_6_5, stg4_0);
|
||||
butterfly(v16, v31, stg3_4, stg3_5, &v17, &v30);
|
||||
butterfly(v19, v28, stg3_6, stg3_4, &v18, &v29);
|
||||
butterfly(v20, v27, stg3_8, stg3_9, &v21, &v26);
|
||||
butterfly(v23, v24, stg3_10, stg3_8, &v22, &v25);
|
||||
|
||||
tmp0 = _mm_add_epi32(tmp0, rounding);
|
||||
tmp1 = _mm_add_epi32(tmp1, rounding);
|
||||
tmp2 = _mm_add_epi32(tmp2, rounding);
|
||||
tmp3 = _mm_add_epi32(tmp3, rounding);
|
||||
u16 = _mm_add_epi16(v16, v19);
|
||||
u17 = _mm_add_epi16(v17, v18);
|
||||
u18 = _mm_sub_epi16(v17, v18);
|
||||
u19 = _mm_sub_epi16(v16, v19);
|
||||
u20 = _mm_sub_epi16(v23, v20);
|
||||
u21 = _mm_sub_epi16(v22, v21);
|
||||
u22 = _mm_add_epi16(v22, v21);
|
||||
u23 = _mm_add_epi16(v23, v20);
|
||||
u24 = _mm_add_epi16(v24, v27);
|
||||
u27 = _mm_sub_epi16(v24, v27);
|
||||
u25 = _mm_add_epi16(v25, v26);
|
||||
u26 = _mm_sub_epi16(v25, v26);
|
||||
u28 = _mm_sub_epi16(v31, v28);
|
||||
u31 = _mm_add_epi16(v28, v31);
|
||||
u29 = _mm_sub_epi16(v30, v29);
|
||||
u30 = _mm_add_epi16(v29, v30);
|
||||
|
||||
tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
|
||||
tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
|
||||
tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
|
||||
tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
|
||||
butterfly_self(&u18, &u29, stg4_4, stg4_5);
|
||||
butterfly_self(&u19, &u28, stg4_4, stg4_5);
|
||||
butterfly_self(&u20, &u27, stg4_6, stg4_4);
|
||||
butterfly_self(&u21, &u26, stg4_6, stg4_4);
|
||||
|
||||
stp1[5] = _mm_packs_epi32(tmp0, tmp1);
|
||||
stp1[6] = _mm_packs_epi32(tmp2, tmp3);
|
||||
#else
|
||||
tmp0 = _mm_sub_epi16(stp1[7], stp1[4]);
|
||||
tmp1 = _mm_adds_epi16(stp1[7], stp1[4]);
|
||||
stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0);
|
||||
stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0);
|
||||
#endif
|
||||
stp1[16] = _mm_add_epi16(u16, u23);
|
||||
v23 = _mm_sub_epi16(u16, u23);
|
||||
|
||||
stp1[8] = _mm_add_epi16(stp2_8, stp2_11);
|
||||
stp1[9] = _mm_add_epi16(stp2_9, stp2_10);
|
||||
stp1[10] = _mm_sub_epi16(stp2_9, stp2_10);
|
||||
stp1[11] = _mm_sub_epi16(stp2_8, stp2_11);
|
||||
stp1[12] = _mm_sub_epi16(stp2_15, stp2_12);
|
||||
stp1[13] = _mm_sub_epi16(stp2_14, stp2_13);
|
||||
stp1[14] = _mm_add_epi16(stp2_14, stp2_13);
|
||||
stp1[15] = _mm_add_epi16(stp2_15, stp2_12);
|
||||
stp1[17] = _mm_add_epi16(u17, u22);
|
||||
v22 = _mm_sub_epi16(u17, u22);
|
||||
|
||||
MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4,
|
||||
stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19],
|
||||
stp1[28])
|
||||
MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6,
|
||||
stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21],
|
||||
stp1[26])
|
||||
stp1[18] = _mm_add_epi16(u18, u21);
|
||||
v21 = _mm_sub_epi16(u18, u21);
|
||||
|
||||
stp1[16] = stp2_16;
|
||||
stp1[17] = stp2_17;
|
||||
stp1[22] = stp2_22;
|
||||
stp1[23] = stp2_23;
|
||||
stp1[24] = stp2_24;
|
||||
stp1[25] = stp2_25;
|
||||
stp1[30] = stp2_30;
|
||||
stp1[31] = stp2_31;
|
||||
}
|
||||
stp1[19] = _mm_add_epi16(u19, u20);
|
||||
v20 = _mm_sub_epi16(u19, u20);
|
||||
|
||||
/* Stage6 */
|
||||
{
|
||||
#if AVOID_OVERFLOW
|
||||
const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]);
|
||||
const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]);
|
||||
const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]);
|
||||
const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]);
|
||||
#endif
|
||||
v24 = _mm_sub_epi16(u31, u24);
|
||||
stp1[31] = _mm_add_epi16(u24, u31);
|
||||
|
||||
stp2_0 = _mm_add_epi16(stp1[0], stp1[7]);
|
||||
stp2_1 = _mm_add_epi16(stp1[1], stp1[6]);
|
||||
stp2_2 = _mm_add_epi16(stp1[2], stp1[5]);
|
||||
stp2_3 = _mm_add_epi16(stp1[3], stp1[4]);
|
||||
stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]);
|
||||
stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]);
|
||||
stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]);
|
||||
stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]);
|
||||
v25 = _mm_sub_epi16(u30, u25);
|
||||
stp1[30] = _mm_add_epi16(u25, u30);
|
||||
|
||||
stp2_8 = stp1[8];
|
||||
stp2_9 = stp1[9];
|
||||
stp2_14 = stp1[14];
|
||||
stp2_15 = stp1[15];
|
||||
v26 = _mm_sub_epi16(u29, u26);
|
||||
stp1[29] = _mm_add_epi16(u26, u29);
|
||||
|
||||
#if AVOID_OVERFLOW
|
||||
MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0,
|
||||
stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11,
|
||||
stp2_12)
|
||||
#else
|
||||
tmp0 = _mm_add_epi16(stp1[10], stp1[13]);
|
||||
tmp1 = _mm_sub_epi16(stp1[13], stp1[10]);
|
||||
tmp2 = _mm_add_epi16(stp1[11], stp1[12]);
|
||||
tmp3 = _mm_sub_epi16(stp1[12], stp1[11]);
|
||||
v27 = _mm_sub_epi16(u28, u27);
|
||||
stp1[28] = _mm_add_epi16(u27, u28);
|
||||
|
||||
stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0);
|
||||
stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0);
|
||||
stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0);
|
||||
stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0);
|
||||
|
||||
#endif
|
||||
|
||||
stp2_16 = _mm_add_epi16(stp1[16], stp1[23]);
|
||||
stp2_17 = _mm_add_epi16(stp1[17], stp1[22]);
|
||||
stp2_18 = _mm_add_epi16(stp1[18], stp1[21]);
|
||||
stp2_19 = _mm_add_epi16(stp1[19], stp1[20]);
|
||||
stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]);
|
||||
stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]);
|
||||
stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]);
|
||||
stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]);
|
||||
|
||||
stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]);
|
||||
stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]);
|
||||
stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]);
|
||||
stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]);
|
||||
stp2_28 = _mm_add_epi16(stp1[27], stp1[28]);
|
||||
stp2_29 = _mm_add_epi16(stp1[26], stp1[29]);
|
||||
stp2_30 = _mm_add_epi16(stp1[25], stp1[30]);
|
||||
stp2_31 = _mm_add_epi16(stp1[24], stp1[31]);
|
||||
}
|
||||
|
||||
/* Stage7 */
|
||||
{
|
||||
#if AVOID_OVERFLOW
|
||||
const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
|
||||
const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
|
||||
const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
|
||||
const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
|
||||
|
||||
const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
|
||||
const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
|
||||
const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24);
|
||||
const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24);
|
||||
#endif
|
||||
stp1[0] = _mm_add_epi16(stp2_0, stp2_15);
|
||||
stp1[1] = _mm_add_epi16(stp2_1, stp2_14);
|
||||
stp1[2] = _mm_add_epi16(stp2_2, stp2_13);
|
||||
stp1[3] = _mm_add_epi16(stp2_3, stp2_12);
|
||||
stp1[4] = _mm_add_epi16(stp2_4, stp2_11);
|
||||
stp1[5] = _mm_add_epi16(stp2_5, stp2_10);
|
||||
stp1[6] = _mm_add_epi16(stp2_6, stp2_9);
|
||||
stp1[7] = _mm_add_epi16(stp2_7, stp2_8);
|
||||
stp1[8] = _mm_sub_epi16(stp2_7, stp2_8);
|
||||
stp1[9] = _mm_sub_epi16(stp2_6, stp2_9);
|
||||
stp1[10] = _mm_sub_epi16(stp2_5, stp2_10);
|
||||
stp1[11] = _mm_sub_epi16(stp2_4, stp2_11);
|
||||
stp1[12] = _mm_sub_epi16(stp2_3, stp2_12);
|
||||
stp1[13] = _mm_sub_epi16(stp2_2, stp2_13);
|
||||
stp1[14] = _mm_sub_epi16(stp2_1, stp2_14);
|
||||
stp1[15] = _mm_sub_epi16(stp2_0, stp2_15);
|
||||
|
||||
stp1[16] = stp2_16;
|
||||
stp1[17] = stp2_17;
|
||||
stp1[18] = stp2_18;
|
||||
stp1[19] = stp2_19;
|
||||
|
||||
#if AVOID_OVERFLOW
|
||||
MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0,
|
||||
stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21],
|
||||
stp1[26])
|
||||
MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0,
|
||||
stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23],
|
||||
stp1[24])
|
||||
#else
|
||||
tmp0 = _mm_add_epi16(stp2_20, stp2_27);
|
||||
tmp1 = _mm_sub_epi16(stp2_27, stp2_20);
|
||||
tmp2 = _mm_add_epi16(stp2_21, stp2_26);
|
||||
tmp3 = _mm_sub_epi16(stp2_26, stp2_21);
|
||||
|
||||
stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0);
|
||||
stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0);
|
||||
stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0);
|
||||
stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0);
|
||||
|
||||
tmp0 = _mm_add_epi16(stp2_22, stp2_25);
|
||||
tmp1 = _mm_sub_epi16(stp2_25, stp2_22);
|
||||
tmp2 = _mm_add_epi16(stp2_23, stp2_24);
|
||||
tmp3 = _mm_sub_epi16(stp2_24, stp2_23);
|
||||
|
||||
stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0);
|
||||
stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0);
|
||||
stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0);
|
||||
stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0);
|
||||
#endif
|
||||
|
||||
stp1[28] = stp2_28;
|
||||
stp1[29] = stp2_29;
|
||||
stp1[30] = stp2_30;
|
||||
stp1[31] = stp2_31;
|
||||
}
|
||||
#undef AVOID_OVERFLOW
|
||||
butterfly(v20, v27, stg6_0, stg4_0, &stp1[20], &stp1[27]);
|
||||
butterfly(v21, v26, stg6_0, stg4_0, &stp1[21], &stp1[26]);
|
||||
butterfly(v22, v25, stg6_0, stg4_0, &stp1[22], &stp1[25]);
|
||||
butterfly(v23, v24, stg6_0, stg4_0, &stp1[23], &stp1[24]);
|
||||
}
|
||||
|
||||
// Only upper-left 8x8 has non-zero coeff
|
||||
@@ -722,81 +645,20 @@ void vpx_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest,
|
||||
in[7] = load_input_data(input + 224);
|
||||
|
||||
array_transpose_8x8(in, in);
|
||||
idct32_34(in, stp1);
|
||||
idct32_34_first_half(in, stp1);
|
||||
idct32_34_second_half(in, stp1);
|
||||
|
||||
// 1_D: Store 32 intermediate results for each 8x32 block.
|
||||
col[0] = _mm_add_epi16(stp1[0], stp1[31]);
|
||||
col[1] = _mm_add_epi16(stp1[1], stp1[30]);
|
||||
col[2] = _mm_add_epi16(stp1[2], stp1[29]);
|
||||
col[3] = _mm_add_epi16(stp1[3], stp1[28]);
|
||||
col[4] = _mm_add_epi16(stp1[4], stp1[27]);
|
||||
col[5] = _mm_add_epi16(stp1[5], stp1[26]);
|
||||
col[6] = _mm_add_epi16(stp1[6], stp1[25]);
|
||||
col[7] = _mm_add_epi16(stp1[7], stp1[24]);
|
||||
col[8] = _mm_add_epi16(stp1[8], stp1[23]);
|
||||
col[9] = _mm_add_epi16(stp1[9], stp1[22]);
|
||||
col[10] = _mm_add_epi16(stp1[10], stp1[21]);
|
||||
col[11] = _mm_add_epi16(stp1[11], stp1[20]);
|
||||
col[12] = _mm_add_epi16(stp1[12], stp1[19]);
|
||||
col[13] = _mm_add_epi16(stp1[13], stp1[18]);
|
||||
col[14] = _mm_add_epi16(stp1[14], stp1[17]);
|
||||
col[15] = _mm_add_epi16(stp1[15], stp1[16]);
|
||||
col[16] = _mm_sub_epi16(stp1[15], stp1[16]);
|
||||
col[17] = _mm_sub_epi16(stp1[14], stp1[17]);
|
||||
col[18] = _mm_sub_epi16(stp1[13], stp1[18]);
|
||||
col[19] = _mm_sub_epi16(stp1[12], stp1[19]);
|
||||
col[20] = _mm_sub_epi16(stp1[11], stp1[20]);
|
||||
col[21] = _mm_sub_epi16(stp1[10], stp1[21]);
|
||||
col[22] = _mm_sub_epi16(stp1[9], stp1[22]);
|
||||
col[23] = _mm_sub_epi16(stp1[8], stp1[23]);
|
||||
col[24] = _mm_sub_epi16(stp1[7], stp1[24]);
|
||||
col[25] = _mm_sub_epi16(stp1[6], stp1[25]);
|
||||
col[26] = _mm_sub_epi16(stp1[5], stp1[26]);
|
||||
col[27] = _mm_sub_epi16(stp1[4], stp1[27]);
|
||||
col[28] = _mm_sub_epi16(stp1[3], stp1[28]);
|
||||
col[29] = _mm_sub_epi16(stp1[2], stp1[29]);
|
||||
col[30] = _mm_sub_epi16(stp1[1], stp1[30]);
|
||||
col[31] = _mm_sub_epi16(stp1[0], stp1[31]);
|
||||
simple_group_butterfly(stp1, col);
|
||||
for (i = 0; i < 4; i++) {
|
||||
int j;
|
||||
// Transpose 32x8 block to 8x32 block
|
||||
array_transpose_8x8(col + i * 8, in);
|
||||
idct32_34(in, stp1);
|
||||
idct32_34_first_half(in, stp1);
|
||||
idct32_34_second_half(in, stp1);
|
||||
|
||||
// 2_D: Calculate the results and store them to destination.
|
||||
in[0] = _mm_add_epi16(stp1[0], stp1[31]);
|
||||
in[1] = _mm_add_epi16(stp1[1], stp1[30]);
|
||||
in[2] = _mm_add_epi16(stp1[2], stp1[29]);
|
||||
in[3] = _mm_add_epi16(stp1[3], stp1[28]);
|
||||
in[4] = _mm_add_epi16(stp1[4], stp1[27]);
|
||||
in[5] = _mm_add_epi16(stp1[5], stp1[26]);
|
||||
in[6] = _mm_add_epi16(stp1[6], stp1[25]);
|
||||
in[7] = _mm_add_epi16(stp1[7], stp1[24]);
|
||||
in[8] = _mm_add_epi16(stp1[8], stp1[23]);
|
||||
in[9] = _mm_add_epi16(stp1[9], stp1[22]);
|
||||
in[10] = _mm_add_epi16(stp1[10], stp1[21]);
|
||||
in[11] = _mm_add_epi16(stp1[11], stp1[20]);
|
||||
in[12] = _mm_add_epi16(stp1[12], stp1[19]);
|
||||
in[13] = _mm_add_epi16(stp1[13], stp1[18]);
|
||||
in[14] = _mm_add_epi16(stp1[14], stp1[17]);
|
||||
in[15] = _mm_add_epi16(stp1[15], stp1[16]);
|
||||
in[16] = _mm_sub_epi16(stp1[15], stp1[16]);
|
||||
in[17] = _mm_sub_epi16(stp1[14], stp1[17]);
|
||||
in[18] = _mm_sub_epi16(stp1[13], stp1[18]);
|
||||
in[19] = _mm_sub_epi16(stp1[12], stp1[19]);
|
||||
in[20] = _mm_sub_epi16(stp1[11], stp1[20]);
|
||||
in[21] = _mm_sub_epi16(stp1[10], stp1[21]);
|
||||
in[22] = _mm_sub_epi16(stp1[9], stp1[22]);
|
||||
in[23] = _mm_sub_epi16(stp1[8], stp1[23]);
|
||||
in[24] = _mm_sub_epi16(stp1[7], stp1[24]);
|
||||
in[25] = _mm_sub_epi16(stp1[6], stp1[25]);
|
||||
in[26] = _mm_sub_epi16(stp1[5], stp1[26]);
|
||||
in[27] = _mm_sub_epi16(stp1[4], stp1[27]);
|
||||
in[28] = _mm_sub_epi16(stp1[3], stp1[28]);
|
||||
in[29] = _mm_sub_epi16(stp1[2], stp1[29]);
|
||||
in[30] = _mm_sub_epi16(stp1[1], stp1[30]);
|
||||
in[31] = _mm_sub_epi16(stp1[0], stp1[31]);
|
||||
|
||||
simple_group_butterfly(stp1, in);
|
||||
for (j = 0; j < 32; ++j) {
|
||||
// Final rounding and shift
|
||||
in[j] = _mm_adds_epi16(in[j], final_rounding);
|
||||
|
||||
Reference in New Issue
Block a user