From f783e3a75d2debf41bb50793edd50b7459ea84f0 Mon Sep 17 00:00:00 2001 From: Scott LaVarnway Date: Tue, 29 Aug 2017 11:25:32 -0700 Subject: [PATCH] vpxdsp: [x86] add highbd_dc_top_predictor functions C vs SSE2 speed gains: _4x4 : ~7.39x _8x8 : ~11.36x _16x16 : ~8.68x _32x32 : ~4.33x BUG=webm:1411 Change-Id: I7f1487cd1531d4e7f0fbb4596fed3bfb72a59d58 --- test/test_intra_pred_speed.cc | 12 +- test/vp9_intrapred_test.cc | 24 ++++ vpx_dsp/vpx_dsp_rtcd_defs.pl | 8 +- vpx_dsp/x86/highbd_intrapred_intrin_sse2.c | 130 +++++++++++++++++++++ 4 files changed, 166 insertions(+), 8 deletions(-) diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index b35727f2c..1239897b7 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc @@ -481,25 +481,29 @@ HIGHBD_INTRA_PRED_TEST( #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, - vpx_highbd_dc_predictor_4x4_sse2, NULL, NULL, NULL, + vpx_highbd_dc_predictor_4x4_sse2, NULL, + vpx_highbd_dc_top_predictor_4x4_sse2, NULL, vpx_highbd_v_predictor_4x4_sse2, vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, NULL, NULL, vpx_highbd_tm_predictor_4x4_c) HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, - vpx_highbd_dc_predictor_8x8_sse2, NULL, NULL, NULL, + vpx_highbd_dc_predictor_8x8_sse2, NULL, + vpx_highbd_dc_top_predictor_8x8_sse2, NULL, vpx_highbd_v_predictor_8x8_sse2, vpx_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, NULL, NULL, vpx_highbd_tm_predictor_8x8_sse2) HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, - vpx_highbd_dc_predictor_16x16_sse2, NULL, NULL, NULL, + vpx_highbd_dc_predictor_16x16_sse2, NULL, + vpx_highbd_dc_top_predictor_16x16_sse2, NULL, vpx_highbd_v_predictor_16x16_sse2, vpx_highbd_h_predictor_16x16_sse2, NULL, NULL, NULL, NULL, NULL, NULL, vpx_highbd_tm_predictor_16x16_sse2) HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, - vpx_highbd_dc_predictor_32x32_sse2, NULL, NULL, NULL, + vpx_highbd_dc_predictor_32x32_sse2, NULL, + vpx_highbd_dc_top_predictor_32x32_sse2, NULL, vpx_highbd_v_predictor_32x32_sse2, vpx_highbd_h_predictor_32x32_sse2, NULL, NULL, NULL, NULL, NULL, NULL, vpx_highbd_tm_predictor_32x32_sse2) diff --git a/test/vp9_intrapred_test.cc b/test/vp9_intrapred_test.cc index 79ba56199..22ef3174d 100644 --- a/test/vp9_intrapred_test.cc +++ b/test/vp9_intrapred_test.cc @@ -479,6 +479,14 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_dc_predictor_16x16_c, 16, 8), HighbdIntraPredParam(&vpx_highbd_dc_predictor_32x32_sse2, &vpx_highbd_dc_predictor_32x32_c, 32, 8), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_4x4_sse2, + &vpx_highbd_dc_top_predictor_4x4_c, 4, 8), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_8x8_sse2, + &vpx_highbd_dc_top_predictor_8x8_c, 8, 8), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_16x16_sse2, + &vpx_highbd_dc_top_predictor_16x16_c, 16, 8), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_32x32_sse2, + &vpx_highbd_dc_top_predictor_32x32_c, 32, 8), HighbdIntraPredParam(&vpx_highbd_tm_predictor_4x4_sse2, &vpx_highbd_tm_predictor_4x4_c, 4, 8), HighbdIntraPredParam(&vpx_highbd_tm_predictor_8x8_sse2, @@ -515,6 +523,14 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_dc_predictor_16x16_c, 16, 10), HighbdIntraPredParam(&vpx_highbd_dc_predictor_32x32_sse2, &vpx_highbd_dc_predictor_32x32_c, 32, 10), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_4x4_sse2, + &vpx_highbd_dc_top_predictor_4x4_c, 4, 10), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_8x8_sse2, + &vpx_highbd_dc_top_predictor_8x8_c, 8, 10), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_16x16_sse2, + &vpx_highbd_dc_top_predictor_16x16_c, 16, 10), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_32x32_sse2, + &vpx_highbd_dc_top_predictor_32x32_c, 32, 10), HighbdIntraPredParam(&vpx_highbd_tm_predictor_4x4_sse2, &vpx_highbd_tm_predictor_4x4_c, 4, 10), HighbdIntraPredParam(&vpx_highbd_tm_predictor_8x8_sse2, @@ -551,6 +567,14 @@ INSTANTIATE_TEST_CASE_P( &vpx_highbd_dc_predictor_16x16_c, 16, 12), HighbdIntraPredParam(&vpx_highbd_dc_predictor_32x32_sse2, &vpx_highbd_dc_predictor_32x32_c, 32, 12), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_4x4_sse2, + &vpx_highbd_dc_top_predictor_4x4_c, 4, 12), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_8x8_sse2, + &vpx_highbd_dc_top_predictor_8x8_c, 8, 12), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_16x16_sse2, + &vpx_highbd_dc_top_predictor_16x16_c, 16, 12), + HighbdIntraPredParam(&vpx_highbd_dc_top_predictor_32x32_sse2, + &vpx_highbd_dc_top_predictor_32x32_c, 32, 12), HighbdIntraPredParam(&vpx_highbd_tm_predictor_4x4_sse2, &vpx_highbd_tm_predictor_4x4_c, 4, 12), HighbdIntraPredParam(&vpx_highbd_tm_predictor_8x8_sse2, diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index ede556f22..7f280efbe 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -214,7 +214,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_dc_predictor_4x4 neon sse2/; add_proto qw/void vpx_highbd_dc_top_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_dc_top_predictor_4x4 neon/; + specialize qw/vpx_highbd_dc_top_predictor_4x4 neon sse2/; add_proto qw/void vpx_highbd_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_dc_left_predictor_4x4 neon/; @@ -249,7 +249,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_dc_predictor_8x8 neon sse2/; add_proto qw/void vpx_highbd_dc_top_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_dc_top_predictor_8x8 neon/; + specialize qw/vpx_highbd_dc_top_predictor_8x8 neon sse2/; add_proto qw/void vpx_highbd_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_dc_left_predictor_8x8 neon/; @@ -284,7 +284,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_dc_predictor_16x16 neon sse2/; add_proto qw/void vpx_highbd_dc_top_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_dc_top_predictor_16x16 neon/; + specialize qw/vpx_highbd_dc_top_predictor_16x16 neon sse2/; add_proto qw/void vpx_highbd_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_dc_left_predictor_16x16 neon/; @@ -319,7 +319,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vpx_highbd_dc_predictor_32x32 neon sse2/; add_proto qw/void vpx_highbd_dc_top_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; - specialize qw/vpx_highbd_dc_top_predictor_32x32 neon/; + specialize qw/vpx_highbd_dc_top_predictor_32x32 neon sse2/; add_proto qw/void vpx_highbd_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vpx_highbd_dc_left_predictor_32x32 neon/; diff --git a/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c b/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c index 059f900ff..68464e61c 100644 --- a/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c +++ b/vpx_dsp/x86/highbd_intrapred_intrin_sse2.c @@ -157,3 +157,133 @@ void vpx_highbd_h_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, h_store_32_unpackhi(&dst, stride, &row7); } } + +//------------------------------------------------------------------------------ +// DC 4x4 + +static INLINE __m128i dc_sum_4(const uint16_t *ref) { + const __m128i _dcba = _mm_loadl_epi64((const __m128i *)ref); + const __m128i _xxdc = _mm_shufflelo_epi16(_dcba, 0xe); + const __m128i a = _mm_add_epi16(_dcba, _xxdc); + return _mm_add_epi16(a, _mm_shufflelo_epi16(a, 0x1)); +} + +static INLINE void dc_store_4x4(uint16_t *dst, ptrdiff_t stride, + const __m128i *dc) { + const __m128i dc_dup = _mm_shufflelo_epi16(*dc, 0x0); + int i; + for (i = 0; i < 4; ++i, dst += stride) { + _mm_storel_epi64((__m128i *)dst, dc_dup); + } +} + +void vpx_highbd_dc_top_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i two = _mm_cvtsi32_si128(2); + const __m128i sum = dc_sum_4(above); + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, two), 2); + (void)left; + (void)bd; + dc_store_4x4(dst, stride, &dc); +} + +//------------------------------------------------------------------------------ +// DC 8x8 + +static INLINE __m128i dc_sum_8(const uint16_t *ref) { + const __m128i ref_u16 = _mm_load_si128((const __m128i *)ref); + const __m128i _dcba = _mm_add_epi16(ref_u16, _mm_srli_si128(ref_u16, 8)); + const __m128i _xxdc = _mm_shufflelo_epi16(_dcba, 0xe); + const __m128i a = _mm_add_epi16(_dcba, _xxdc); + + return _mm_add_epi16(a, _mm_shufflelo_epi16(a, 0x1)); +} + +static INLINE void dc_store_8x8(uint16_t *dst, ptrdiff_t stride, + const __m128i *dc) { + const __m128i dc_dup_lo = _mm_shufflelo_epi16(*dc, 0); + const __m128i dc_dup = _mm_unpacklo_epi64(dc_dup_lo, dc_dup_lo); + int i; + for (i = 0; i < 8; ++i, dst += stride) { + _mm_store_si128((__m128i *)dst, dc_dup); + } +} + +void vpx_highbd_dc_top_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i four = _mm_cvtsi32_si128(4); + const __m128i sum = dc_sum_8(above); + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, four), 3); + (void)left; + (void)bd; + dc_store_8x8(dst, stride, &dc); +} + +//------------------------------------------------------------------------------ +// DC 16x16 + +static INLINE __m128i dc_sum_16(const uint16_t *ref) { + const __m128i sum_lo = dc_sum_8(ref); + const __m128i sum_hi = dc_sum_8(ref + 8); + return _mm_add_epi16(sum_lo, sum_hi); +} + +static INLINE void dc_store_16x16(uint16_t *dst, ptrdiff_t stride, + const __m128i *dc) { + const __m128i dc_dup_lo = _mm_shufflelo_epi16(*dc, 0); + const __m128i dc_dup = _mm_unpacklo_epi64(dc_dup_lo, dc_dup_lo); + int i; + for (i = 0; i < 16; ++i, dst += stride) { + _mm_store_si128((__m128i *)dst, dc_dup); + _mm_store_si128((__m128i *)(dst + 8), dc_dup); + } +} + +void vpx_highbd_dc_top_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i eight = _mm_cvtsi32_si128(8); + const __m128i sum = dc_sum_16(above); + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, eight), 4); + (void)left; + (void)bd; + dc_store_16x16(dst, stride, &dc); +} + +//------------------------------------------------------------------------------ +// DC 32x32 + +static INLINE __m128i dc_sum_32(const uint16_t *ref) { + const __m128i zero = _mm_setzero_si128(); + const __m128i sum_a = dc_sum_16(ref); + const __m128i sum_b = dc_sum_16(ref + 16); + // 12 bit bd will outrange, so expand to 32 bit before adding final total + return _mm_add_epi32(_mm_unpacklo_epi16(sum_a, zero), + _mm_unpacklo_epi16(sum_b, zero)); +} + +static INLINE void dc_store_32x32(uint16_t *dst, ptrdiff_t stride, + const __m128i *dc) { + const __m128i dc_dup_lo = _mm_shufflelo_epi16(*dc, 0); + const __m128i dc_dup = _mm_unpacklo_epi64(dc_dup_lo, dc_dup_lo); + int i; + for (i = 0; i < 32; ++i, dst += stride) { + _mm_store_si128((__m128i *)dst, dc_dup); + _mm_store_si128((__m128i *)(dst + 8), dc_dup); + _mm_store_si128((__m128i *)(dst + 16), dc_dup); + _mm_store_si128((__m128i *)(dst + 24), dc_dup); + } +} + +void vpx_highbd_dc_top_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i sixteen = _mm_cvtsi32_si128(16); + const __m128i sum = dc_sum_32(above); + const __m128i dc = _mm_srli_epi32(_mm_add_epi32(sum, sixteen), 5); + (void)left; + (void)bd; + dc_store_32x32(dst, stride, &dc); +}