From 3bbd62ed27ee43a685a1b7b93e985e8c14e528a1 Mon Sep 17 00:00:00 2001 From: Scott LaVarnway Date: Fri, 29 Sep 2017 06:34:16 -0700 Subject: [PATCH] vpxdsp: [x86] add highbd_d135_predictor functions C vs SSE2 speed gains: _4x4 : ~1.81x C vs SSSE3 speed gains: _8x8 : ~1.96x _16x16 : ~1.88x _32x32 : ~2.02x BUG=webm:1411 Change-Id: Iefaf8b39afbbfe34c1ad1d21e3a003b20f1f61e0 --- test/test_intra_pred_speed.cc | 11 +- test/vp9_intrapred_test.cc | 24 ++++ vpx_dsp/vpx_dsp_rtcd_defs.pl | 8 +- vpx_dsp/x86/highbd_intrapred_intrin_sse2.c | 28 +++++ vpx_dsp/x86/highbd_intrapred_intrin_ssse3.c | 120 ++++++++++++++++++++ 5 files changed, 183 insertions(+), 8 deletions(-) diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index 7f1707156..1cdeda410 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc @@ -484,7 +484,7 @@ HIGHBD_INTRA_PRED_TEST( SSE2, TestHighbdIntraPred4, vpx_highbd_dc_predictor_4x4_sse2, vpx_highbd_dc_left_predictor_4x4_sse2, vpx_highbd_dc_top_predictor_4x4_sse2, vpx_highbd_dc_128_predictor_4x4_sse2, vpx_highbd_v_predictor_4x4_sse2, - vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, + vpx_highbd_h_predictor_4x4_sse2, NULL, vpx_highbd_d135_predictor_4x4_sse2, vpx_highbd_d117_predictor_4x4_sse2, vpx_highbd_d153_predictor_4x4_sse2, vpx_highbd_d207_predictor_4x4_sse2, vpx_highbd_d63_predictor_4x4_sse2, vpx_highbd_tm_predictor_4x4_c) @@ -522,19 +522,22 @@ HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred4, NULL, NULL, NULL, NULL, NULL, NULL, vpx_highbd_d45_predictor_4x4_ssse3, NULL, NULL, NULL, NULL, NULL, NULL) HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred8, NULL, NULL, NULL, NULL, - NULL, NULL, vpx_highbd_d45_predictor_8x8_ssse3, NULL, + NULL, NULL, vpx_highbd_d45_predictor_8x8_ssse3, + vpx_highbd_d135_predictor_8x8_ssse3, vpx_highbd_d117_predictor_8x8_ssse3, vpx_highbd_d153_predictor_8x8_ssse3, vpx_highbd_d207_predictor_8x8_ssse3, vpx_highbd_d63_predictor_8x8_ssse3, NULL) HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred16, NULL, NULL, NULL, NULL, - NULL, NULL, vpx_highbd_d45_predictor_16x16_ssse3, NULL, + NULL, NULL, vpx_highbd_d45_predictor_16x16_ssse3, + vpx_highbd_d135_predictor_16x16_ssse3, vpx_highbd_d117_predictor_16x16_ssse3, vpx_highbd_d153_predictor_16x16_ssse3, vpx_highbd_d207_predictor_16x16_ssse3, vpx_highbd_d63_predictor_16x16_ssse3, NULL) HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred32, NULL, NULL, NULL, NULL, - NULL, NULL, vpx_highbd_d45_predictor_32x32_ssse3, NULL, + NULL, NULL, vpx_highbd_d45_predictor_32x32_ssse3, + vpx_highbd_d135_predictor_32x32_ssse3, vpx_highbd_d117_predictor_32x32_ssse3, vpx_highbd_d153_predictor_32x32_ssse3, vpx_highbd_d207_predictor_32x32_ssse3, diff --git a/test/vp9_intrapred_test.cc b/test/vp9_intrapred_test.cc index e074240fb..39c5e79eb 100644 --- a/test/vp9_intrapred_test.cc +++ b/test/vp9_intrapred_test.cc @@ -491,6 +491,12 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d117_predictor_16x16_c, 16, 8), HighbdIntraPredParam(&vpx_highbd_d117_predictor_32x32_c, &vpx_highbd_d117_predictor_32x32_ssse3, 32, 8), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_8x8_ssse3, + &vpx_highbd_d135_predictor_8x8_c, 8, 8), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_16x16_ssse3, + &vpx_highbd_d135_predictor_16x16_c, 16, 8), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_32x32_ssse3, + &vpx_highbd_d135_predictor_32x32_c, 32, 8), HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3, &vpx_highbd_d153_predictor_8x8_c, 8, 8), HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3, @@ -527,6 +533,12 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d117_predictor_16x16_c, 16, 10), HighbdIntraPredParam(&vpx_highbd_d117_predictor_32x32_c, &vpx_highbd_d117_predictor_32x32_ssse3, 32, 10), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_8x8_ssse3, + &vpx_highbd_d135_predictor_8x8_c, 8, 10), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_16x16_ssse3, + &vpx_highbd_d135_predictor_16x16_c, 16, 10), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_32x32_ssse3, + &vpx_highbd_d135_predictor_32x32_c, 32, 10), HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3, &vpx_highbd_d153_predictor_8x8_c, 8, 10), HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3, @@ -563,6 +575,12 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d117_predictor_16x16_c, 16, 12), HighbdIntraPredParam(&vpx_highbd_d117_predictor_32x32_c, &vpx_highbd_d117_predictor_32x32_ssse3, 32, 12), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_8x8_ssse3, + &vpx_highbd_d135_predictor_8x8_c, 8, 12), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_16x16_ssse3, + &vpx_highbd_d135_predictor_16x16_c, 16, 12), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_32x32_ssse3, + &vpx_highbd_d135_predictor_32x32_c, 32, 12), HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3, &vpx_highbd_d153_predictor_8x8_c, 8, 12), HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3, @@ -593,6 +611,8 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d63_predictor_4x4_c, 4, 8), HighbdIntraPredParam(&vpx_highbd_d117_predictor_4x4_sse2, &vpx_highbd_d117_predictor_4x4_c, 4, 8), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_4x4_sse2, + &vpx_highbd_d135_predictor_4x4_c, 4, 8), HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2, &vpx_highbd_d153_predictor_4x4_c, 4, 8), HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2, @@ -661,6 +681,8 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d63_predictor_4x4_c, 4, 10), HighbdIntraPredParam(&vpx_highbd_d117_predictor_4x4_sse2, &vpx_highbd_d117_predictor_4x4_c, 4, 10), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_4x4_sse2, + &vpx_highbd_d135_predictor_4x4_c, 4, 10), HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2, &vpx_highbd_d153_predictor_4x4_c, 4, 10), HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2, @@ -729,6 +751,8 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_d63_predictor_4x4_c, 4, 12), HighbdIntraPredParam(&vpx_highbd_d117_predictor_4x4_sse2, &vpx_highbd_d117_predictor_4x4_c, 4, 12), + HighbdIntraPredParam(&vpx_highbd_d135_predictor_4x4_sse2, + &vpx_highbd_d135_predictor_4x4_c, 4, 12), HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2, &vpx_highbd_d153_predictor_4x4_c, 4, 12), HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2, diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index 412afe24c..3c1abf9f6 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -204,7 +204,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_d117_predictor_4x4 sse2/; add_proto qw/void vpx_highbd_d135_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_d135_predictor_4x4 neon/; + specialize qw/vpx_highbd_d135_predictor_4x4 neon sse2/; add_proto qw/void vpx_highbd_d153_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_d153_predictor_4x4 sse2/; @@ -243,7 +243,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_d117_predictor_8x8 ssse3/; add_proto qw/void vpx_highbd_d135_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_d135_predictor_8x8 neon/; + specialize qw/vpx_highbd_d135_predictor_8x8 neon ssse3/; add_proto qw/void vpx_highbd_d153_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_d153_predictor_8x8 ssse3/; @@ -282,7 +282,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_d117_predictor_16x16 ssse3/; add_proto qw/void vpx_highbd_d135_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_d135_predictor_16x16 neon/; + specialize qw/vpx_highbd_d135_predictor_16x16 neon ssse3/; add_proto qw/void vpx_highbd_d153_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_d153_predictor_16x16 ssse3/; @@ -321,7 +321,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_d117_predictor_32x32 ssse3/; add_proto qw/void vpx_highbd_d135_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_d135_predictor_32x32 neon/; + specialize qw/vpx_highbd_d135_predictor_32x32 neon ssse3/; add_proto qw/void vpx_highbd_d153_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_d153_predictor_32x32 ssse3/; diff --git a/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c b/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c index 21d7d6359..2051381aa 100644 --- a/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c +++ b/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c @@ -425,6 +425,34 @@ void vpx_highbd_d117_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, dst[stride] = _mm_extract_epi16(avg3, 0); } +void vpx_highbd_d135_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int I = left[0]; + const int J = left[1]; + const int K = left[2]; + const int L = left[3]; + const __m128i XXXXABCD = _mm_loadu_si128((const __m128i *)(above - 4)); + const __m128i KXXXABCD = _mm_insert_epi16(XXXXABCD, K, 0); + const __m128i KJXXABCD = _mm_insert_epi16(KXXXABCD, J, 1); + const __m128i KJIXABCD = _mm_insert_epi16(KJXXABCD, I, 2); + const __m128i JIXABCD0 = _mm_srli_si128(KJIXABCD, 2); + const __m128i LKJIXABC = _mm_insert_epi16(_mm_slli_si128(KJIXABCD, 2), L, 0); + const __m128i avg3 = avg3_epu16(&JIXABCD0, &KJIXABCD, &LKJIXABC); + const __m128i row0 = _mm_srli_si128(avg3, 6); + const __m128i row1 = _mm_srli_si128(avg3, 4); + const __m128i row2 = _mm_srli_si128(avg3, 2); + const __m128i row3 = avg3; + (void)bd; + _mm_storel_epi64((__m128i *)dst, row0); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row1); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row2); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row3); +} + void vpx_highbd_d153_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, const uint16_t *above, const uint16_t *left, int bd) { diff --git a/vpx_dsp/x86/highbd_intrapred_intrin_ssse3.c b/vpx_dsp/x86/highbd_intrapred_intrin_ssse3.c index 8fd6a6f54..b9dcef205 100644 --- a/vpx_dsp/x86/highbd_intrapred_intrin_ssse3.c +++ b/vpx_dsp/x86/highbd_intrapred_intrin_ssse3.c @@ -330,6 +330,126 @@ void vpx_highbd_d117_predictor_32x32_ssse3(uint16_t *dst, ptrdiff_t stride, } } +void vpx_highbd_d135_predictor_8x8_ssse3(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i rotrw = _mm_load_si128((const __m128i *)rotate_right_epu16); + const __m128i XABCDEFG = _mm_loadu_si128((const __m128i *)(above - 1)); + const __m128i ABCDEFGH = _mm_load_si128((const __m128i *)above); + const __m128i BCDEFGH0 = _mm_srli_si128(ABCDEFGH, 2); + const __m128i IJKLMNOP = _mm_load_si128((const __m128i *)left); + const __m128i XIJKLMNO = + _mm_alignr_epi8(IJKLMNOP, _mm_slli_si128(XABCDEFG, 14), 14); + const __m128i AXIJKLMN = + _mm_alignr_epi8(XIJKLMNO, _mm_slli_si128(ABCDEFGH, 14), 14); + const __m128i avg3 = avg3_epu16(&XABCDEFG, &ABCDEFGH, &BCDEFGH0); + __m128i avg3_left = avg3_epu16(&IJKLMNOP, &XIJKLMNO, &AXIJKLMN); + __m128i rowa = avg3; + int i; + (void)bd; + for (i = 0; i < 8; ++i) { + rowa = _mm_alignr_epi8(rowa, rotr_epu16(&avg3_left, &rotrw), 14); + _mm_store_si128((__m128i *)dst, rowa); + dst += stride; + } +} + +void vpx_highbd_d135_predictor_16x16_ssse3(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i rotrw = _mm_load_si128((const __m128i *)rotate_right_epu16); + const __m128i A0 = _mm_loadu_si128((const __m128i *)(above - 1)); + const __m128i B0 = _mm_load_si128((const __m128i *)above); + const __m128i A1 = _mm_loadu_si128((const __m128i *)(above + 7)); + const __m128i B1 = _mm_load_si128((const __m128i *)(above + 8)); + const __m128i L0 = _mm_load_si128((const __m128i *)left); + const __m128i L1 = _mm_load_si128((const __m128i *)(left + 8)); + const __m128i C0 = _mm_alignr_epi8(B1, B0, 2); + const __m128i C1 = _mm_srli_si128(B1, 2); + const __m128i avg3_0 = avg3_epu16(&A0, &B0, &C0); + const __m128i avg3_1 = avg3_epu16(&A1, &B1, &C1); + const __m128i XL0 = _mm_alignr_epi8(L0, _mm_slli_si128(A0, 14), 14); + const __m128i XL1 = _mm_alignr_epi8(L1, L0, 14); + const __m128i L0_ = _mm_alignr_epi8(XL0, _mm_slli_si128(B0, 14), 14); + const __m128i L1_ = _mm_alignr_epi8(XL1, XL0, 14); + __m128i rowa_0 = avg3_0; + __m128i rowa_1 = avg3_1; + __m128i avg3_left[2]; + int i, j; + (void)bd; + avg3_left[0] = avg3_epu16(&L0, &XL0, &L0_); + avg3_left[1] = avg3_epu16(&L1, &XL1, &L1_); + for (i = 0; i < 2; ++i) { + __m128i avg_left = avg3_left[i]; + for (j = 0; j < 8; ++j) { + rowa_1 = _mm_alignr_epi8(rowa_1, rowa_0, 14); + rowa_0 = _mm_alignr_epi8(rowa_0, rotr_epu16(&avg_left, &rotrw), 14); + _mm_store_si128((__m128i *)dst, rowa_0); + _mm_store_si128((__m128i *)(dst + 8), rowa_1); + dst += stride; + } + } +} + +void vpx_highbd_d135_predictor_32x32_ssse3(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i rotrw = _mm_load_si128((const __m128i *)rotate_right_epu16); + const __m128i A0 = _mm_loadu_si128((const __m128i *)(above - 1)); + const __m128i A1 = _mm_loadu_si128((const __m128i *)(above + 7)); + const __m128i A2 = _mm_loadu_si128((const __m128i *)(above + 15)); + const __m128i A3 = _mm_loadu_si128((const __m128i *)(above + 23)); + const __m128i B0 = _mm_load_si128((const __m128i *)above); + const __m128i B1 = _mm_load_si128((const __m128i *)(above + 8)); + const __m128i B2 = _mm_load_si128((const __m128i *)(above + 16)); + const __m128i B3 = _mm_load_si128((const __m128i *)(above + 24)); + const __m128i L0 = _mm_load_si128((const __m128i *)left); + const __m128i L1 = _mm_load_si128((const __m128i *)(left + 8)); + const __m128i L2 = _mm_load_si128((const __m128i *)(left + 16)); + const __m128i L3 = _mm_load_si128((const __m128i *)(left + 24)); + const __m128i C0 = _mm_alignr_epi8(B1, B0, 2); + const __m128i C1 = _mm_alignr_epi8(B2, B1, 2); + const __m128i C2 = _mm_alignr_epi8(B3, B2, 2); + const __m128i C3 = _mm_srli_si128(B3, 2); + const __m128i avg3_0 = avg3_epu16(&A0, &B0, &C0); + const __m128i avg3_1 = avg3_epu16(&A1, &B1, &C1); + const __m128i avg3_2 = avg3_epu16(&A2, &B2, &C2); + const __m128i avg3_3 = avg3_epu16(&A3, &B3, &C3); + const __m128i XL0 = _mm_alignr_epi8(L0, _mm_slli_si128(A0, 14), 14); + const __m128i XL1 = _mm_alignr_epi8(L1, L0, 14); + const __m128i XL2 = _mm_alignr_epi8(L2, L1, 14); + const __m128i XL3 = _mm_alignr_epi8(L3, L2, 14); + const __m128i L0_ = _mm_alignr_epi8(XL0, _mm_slli_si128(B0, 14), 14); + const __m128i L1_ = _mm_alignr_epi8(XL1, XL0, 14); + const __m128i L2_ = _mm_alignr_epi8(XL2, XL1, 14); + const __m128i L3_ = _mm_alignr_epi8(XL3, XL2, 14); + __m128i rowa_0 = avg3_0; + __m128i rowa_1 = avg3_1; + __m128i rowa_2 = avg3_2; + __m128i rowa_3 = avg3_3; + __m128i avg3_left[4]; + int i, j; + (void)bd; + avg3_left[0] = avg3_epu16(&L0, &XL0, &L0_); + avg3_left[1] = avg3_epu16(&L1, &XL1, &L1_); + avg3_left[2] = avg3_epu16(&L2, &XL2, &L2_); + avg3_left[3] = avg3_epu16(&L3, &XL3, &L3_); + for (i = 0; i < 4; ++i) { + __m128i avg_left = avg3_left[i]; + for (j = 0; j < 8; ++j) { + rowa_3 = _mm_alignr_epi8(rowa_3, rowa_2, 14); + rowa_2 = _mm_alignr_epi8(rowa_2, rowa_1, 14); + rowa_1 = _mm_alignr_epi8(rowa_1, rowa_0, 14); + rowa_0 = _mm_alignr_epi8(rowa_0, rotr_epu16(&avg_left, &rotrw), 14); + _mm_store_si128((__m128i *)dst, rowa_0); + _mm_store_si128((__m128i *)(dst + 8), rowa_1); + _mm_store_si128((__m128i *)(dst + 16), rowa_2); + _mm_store_si128((__m128i *)(dst + 24), rowa_3); + dst += stride; + } + } +} + void vpx_highbd_d153_predictor_8x8_ssse3(uint16_t *dst, ptrdiff_t stride, const uint16_t *above, const uint16_t *left, int bd) {