diff --git a/test/dct_test.cc b/test/dct_test.cc index 71b36c7bb..5b09cebae 100644 --- a/test/dct_test.cc +++ b/test/dct_test.cc @@ -792,6 +792,43 @@ const DctParam neon_ht_tests[] = { make_tuple(&vp9_highbd_fht4x4_c, &highbd_iht_wrapper, 4, 3, VPX_BITS_12, 2), + + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 0, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 1, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 2, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 3, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 0, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 1, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 2, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 3, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 0, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 1, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 2, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht8x8_c, + &highbd_iht_wrapper, 8, 3, + VPX_BITS_12, 2), #endif // CONFIG_VP9_HIGHBITDEPTH make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 0, VPX_BITS_8, 1), diff --git a/vp9/common/arm/neon/vp9_highbd_iht8x8_add_neon.c b/vp9/common/arm/neon/vp9_highbd_iht8x8_add_neon.c new file mode 100644 index 000000000..498f03530 --- /dev/null +++ b/vp9/common/arm/neon/vp9_highbd_iht8x8_add_neon.c @@ -0,0 +1,505 @@ +/* + * Copyright (c) 2018 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./vpx_dsp_rtcd.h" +#include "vp9/common/vp9_enums.h" +#include "vp9/common/arm/neon/vp9_iht_neon.h" +#include "vpx_dsp/arm/highbd_idct_neon.h" +#include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/transpose_neon.h" +#include "vpx_dsp/inv_txfm.h" + +static INLINE void iadst_half_butterfly_bd10_neon(int32x4_t *const x, + const int32x2_t c) { + const int32x4_t sum = vaddq_s32(x[0], x[1]); + const int32x4_t sub = vsubq_s32(x[0], x[1]); + + x[0] = vmulq_lane_s32(sum, c, 0); + x[1] = vmulq_lane_s32(sub, c, 0); + x[0] = vrshrq_n_s32(x[0], DCT_CONST_BITS); + x[1] = vrshrq_n_s32(x[1], DCT_CONST_BITS); +} + +static INLINE void iadst_half_butterfly_bd12_neon(int32x4_t *const x, + const int32x2_t c) { + const int32x4_t sum = vaddq_s32(x[0], x[1]); + const int32x4_t sub = vsubq_s32(x[0], x[1]); + const int64x2_t t0_lo = vmull_lane_s32(vget_low_s32(sum), c, 0); + const int64x2_t t1_lo = vmull_lane_s32(vget_low_s32(sub), c, 0); + const int64x2_t t0_hi = vmull_lane_s32(vget_high_s32(sum), c, 0); + const int64x2_t t1_hi = vmull_lane_s32(vget_high_s32(sub), c, 0); + const int32x2_t out0_lo = vrshrn_n_s64(t0_lo, DCT_CONST_BITS); + const int32x2_t out1_lo = vrshrn_n_s64(t1_lo, DCT_CONST_BITS); + const int32x2_t out0_hi = vrshrn_n_s64(t0_hi, DCT_CONST_BITS); + const int32x2_t out1_hi = vrshrn_n_s64(t1_hi, DCT_CONST_BITS); + + x[0] = vcombine_s32(out0_lo, out0_hi); + x[1] = vcombine_s32(out1_lo, out1_hi); +} + +static INLINE void iadst_butterfly_lane_0_1_bd10_neon(const int32x4_t in0, + const int32x4_t in1, + const int32x2_t c, + int32x4_t *const s0, + int32x4_t *const s1) { + const int32x4_t t0 = vmulq_lane_s32(in0, c, 0); + const int32x4_t t1 = vmulq_lane_s32(in0, c, 1); + + *s0 = vmlaq_lane_s32(t0, in1, c, 1); + *s1 = vmlsq_lane_s32(t1, in1, c, 0); +} + +static INLINE void iadst_butterfly_lane_1_0_bd10_neon(const int32x4_t in0, + const int32x4_t in1, + const int32x2_t c, + int32x4_t *const s0, + int32x4_t *const s1) { + const int32x4_t t0 = vmulq_lane_s32(in0, c, 1); + const int32x4_t t1 = vmulq_lane_s32(in0, c, 0); + + *s0 = vmlaq_lane_s32(t0, in1, c, 0); + *s1 = vmlsq_lane_s32(t1, in1, c, 1); +} + +static INLINE void iadst_butterfly_lane_0_1_bd12_neon(const int32x4_t in0, + const int32x4_t in1, + const int32x2_t c, + int64x2_t *const s0, + int64x2_t *const s1) { + const int64x2_t t0_lo = vmull_lane_s32(vget_low_s32(in0), c, 0); + const int64x2_t t1_lo = vmull_lane_s32(vget_low_s32(in0), c, 1); + const int64x2_t t0_hi = vmull_lane_s32(vget_high_s32(in0), c, 0); + const int64x2_t t1_hi = vmull_lane_s32(vget_high_s32(in0), c, 1); + + s0[0] = vmlal_lane_s32(t0_lo, vget_low_s32(in1), c, 1); + s1[0] = vmlsl_lane_s32(t1_lo, vget_low_s32(in1), c, 0); + s0[1] = vmlal_lane_s32(t0_hi, vget_high_s32(in1), c, 1); + s1[1] = vmlsl_lane_s32(t1_hi, vget_high_s32(in1), c, 0); +} + +static INLINE void iadst_butterfly_lane_1_0_bd12_neon(const int32x4_t in0, + const int32x4_t in1, + const int32x2_t c, + int64x2_t *const s0, + int64x2_t *const s1) { + const int64x2_t t0_lo = vmull_lane_s32(vget_low_s32(in0), c, 1); + const int64x2_t t1_lo = vmull_lane_s32(vget_low_s32(in0), c, 0); + const int64x2_t t0_hi = vmull_lane_s32(vget_high_s32(in0), c, 1); + const int64x2_t t1_hi = vmull_lane_s32(vget_high_s32(in0), c, 0); + + s0[0] = vmlal_lane_s32(t0_lo, vget_low_s32(in1), c, 0); + s1[0] = vmlsl_lane_s32(t1_lo, vget_low_s32(in1), c, 1); + s0[1] = vmlal_lane_s32(t0_hi, vget_high_s32(in1), c, 0); + s1[1] = vmlsl_lane_s32(t1_hi, vget_high_s32(in1), c, 1); +} + +static INLINE int32x4_t +add_dct_const_round_shift_low_8_bd10(const int32x4_t in0, const int32x4_t in1) { + const int32x4_t sum = vaddq_s32(in0, in1); + return vrshrq_n_s32(sum, DCT_CONST_BITS); +} + +static INLINE int32x4_t +sub_dct_const_round_shift_low_8_bd10(const int32x4_t in0, const int32x4_t in1) { + const int32x4_t sub = vsubq_s32(in0, in1); + return vrshrq_n_s32(sub, DCT_CONST_BITS); +} + +static INLINE int32x4_t add_dct_const_round_shift_low_8_bd12( + const int64x2_t *const in0, const int64x2_t *const in1) { + const int64x2_t sum_lo = vaddq_s64(in0[0], in1[0]); + const int64x2_t sum_hi = vaddq_s64(in0[1], in1[1]); + const int32x2_t out_lo = vrshrn_n_s64(sum_lo, DCT_CONST_BITS); + const int32x2_t out_hi = vrshrn_n_s64(sum_hi, DCT_CONST_BITS); + return vcombine_s32(out_lo, out_hi); +} + +static INLINE int32x4_t sub_dct_const_round_shift_low_8_bd12( + const int64x2_t *const in0, const int64x2_t *const in1) { + const int64x2_t sub_lo = vsubq_s64(in0[0], in1[0]); + const int64x2_t sub_hi = vsubq_s64(in0[1], in1[1]); + const int32x2_t out_lo = vrshrn_n_s64(sub_lo, DCT_CONST_BITS); + const int32x2_t out_hi = vrshrn_n_s64(sub_hi, DCT_CONST_BITS); + return vcombine_s32(out_lo, out_hi); +} + +static INLINE void iadst8_bd10(int32x4_t *const io0, int32x4_t *const io1, + int32x4_t *const io2, int32x4_t *const io3, + int32x4_t *const io4, int32x4_t *const io5, + int32x4_t *const io6, int32x4_t *const io7) { + const int32x4_t c0 = + create_s32x4_neon(cospi_2_64, cospi_30_64, cospi_10_64, cospi_22_64); + const int32x4_t c1 = + create_s32x4_neon(cospi_18_64, cospi_14_64, cospi_26_64, cospi_6_64); + const int32x4_t c2 = + create_s32x4_neon(cospi_16_64, 0, cospi_8_64, cospi_24_64); + int32x4_t x[8], t[4]; + int32x4_t s[8]; + + x[0] = *io7; + x[1] = *io0; + x[2] = *io5; + x[3] = *io2; + x[4] = *io3; + x[5] = *io4; + x[6] = *io1; + x[7] = *io6; + + // stage 1 + iadst_butterfly_lane_0_1_bd10_neon(x[0], x[1], vget_low_s32(c0), &s[0], + &s[1]); + iadst_butterfly_lane_0_1_bd10_neon(x[2], x[3], vget_high_s32(c0), &s[2], + &s[3]); + iadst_butterfly_lane_0_1_bd10_neon(x[4], x[5], vget_low_s32(c1), &s[4], + &s[5]); + iadst_butterfly_lane_0_1_bd10_neon(x[6], x[7], vget_high_s32(c1), &s[6], + &s[7]); + + x[0] = add_dct_const_round_shift_low_8_bd10(s[0], s[4]); + x[1] = add_dct_const_round_shift_low_8_bd10(s[1], s[5]); + x[2] = add_dct_const_round_shift_low_8_bd10(s[2], s[6]); + x[3] = add_dct_const_round_shift_low_8_bd10(s[3], s[7]); + x[4] = sub_dct_const_round_shift_low_8_bd10(s[0], s[4]); + x[5] = sub_dct_const_round_shift_low_8_bd10(s[1], s[5]); + x[6] = sub_dct_const_round_shift_low_8_bd10(s[2], s[6]); + x[7] = sub_dct_const_round_shift_low_8_bd10(s[3], s[7]); + + // stage 2 + t[0] = x[0]; + t[1] = x[1]; + t[2] = x[2]; + t[3] = x[3]; + iadst_butterfly_lane_0_1_bd10_neon(x[4], x[5], vget_high_s32(c2), &s[4], + &s[5]); + iadst_butterfly_lane_1_0_bd10_neon(x[7], x[6], vget_high_s32(c2), &s[7], + &s[6]); + + x[0] = vaddq_s32(t[0], t[2]); + x[1] = vaddq_s32(t[1], t[3]); + x[2] = vsubq_s32(t[0], t[2]); + x[3] = vsubq_s32(t[1], t[3]); + x[4] = add_dct_const_round_shift_low_8_bd10(s[4], s[6]); + x[5] = add_dct_const_round_shift_low_8_bd10(s[5], s[7]); + x[6] = sub_dct_const_round_shift_low_8_bd10(s[4], s[6]); + x[7] = sub_dct_const_round_shift_low_8_bd10(s[5], s[7]); + + // stage 3 + iadst_half_butterfly_bd10_neon(x + 2, vget_low_s32(c2)); + iadst_half_butterfly_bd10_neon(x + 6, vget_low_s32(c2)); + + *io0 = x[0]; + *io1 = vnegq_s32(x[4]); + *io2 = x[6]; + *io3 = vnegq_s32(x[2]); + *io4 = x[3]; + *io5 = vnegq_s32(x[7]); + *io6 = x[5]; + *io7 = vnegq_s32(x[1]); +} + +static INLINE void iadst8_bd12(int32x4_t *const io0, int32x4_t *const io1, + int32x4_t *const io2, int32x4_t *const io3, + int32x4_t *const io4, int32x4_t *const io5, + int32x4_t *const io6, int32x4_t *const io7) { + const int32x4_t c0 = + create_s32x4_neon(cospi_2_64, cospi_30_64, cospi_10_64, cospi_22_64); + const int32x4_t c1 = + create_s32x4_neon(cospi_18_64, cospi_14_64, cospi_26_64, cospi_6_64); + const int32x4_t c2 = + create_s32x4_neon(cospi_16_64, 0, cospi_8_64, cospi_24_64); + int32x4_t x[8], t[4]; + int64x2_t s[8][2]; + + x[0] = *io7; + x[1] = *io0; + x[2] = *io5; + x[3] = *io2; + x[4] = *io3; + x[5] = *io4; + x[6] = *io1; + x[7] = *io6; + + // stage 1 + iadst_butterfly_lane_0_1_bd12_neon(x[0], x[1], vget_low_s32(c0), s[0], s[1]); + iadst_butterfly_lane_0_1_bd12_neon(x[2], x[3], vget_high_s32(c0), s[2], s[3]); + iadst_butterfly_lane_0_1_bd12_neon(x[4], x[5], vget_low_s32(c1), s[4], s[5]); + iadst_butterfly_lane_0_1_bd12_neon(x[6], x[7], vget_high_s32(c1), s[6], s[7]); + + x[0] = add_dct_const_round_shift_low_8_bd12(s[0], s[4]); + x[1] = add_dct_const_round_shift_low_8_bd12(s[1], s[5]); + x[2] = add_dct_const_round_shift_low_8_bd12(s[2], s[6]); + x[3] = add_dct_const_round_shift_low_8_bd12(s[3], s[7]); + x[4] = sub_dct_const_round_shift_low_8_bd12(s[0], s[4]); + x[5] = sub_dct_const_round_shift_low_8_bd12(s[1], s[5]); + x[6] = sub_dct_const_round_shift_low_8_bd12(s[2], s[6]); + x[7] = sub_dct_const_round_shift_low_8_bd12(s[3], s[7]); + + // stage 2 + t[0] = x[0]; + t[1] = x[1]; + t[2] = x[2]; + t[3] = x[3]; + iadst_butterfly_lane_0_1_bd12_neon(x[4], x[5], vget_high_s32(c2), s[4], s[5]); + iadst_butterfly_lane_1_0_bd12_neon(x[7], x[6], vget_high_s32(c2), s[7], s[6]); + + x[0] = vaddq_s32(t[0], t[2]); + x[1] = vaddq_s32(t[1], t[3]); + x[2] = vsubq_s32(t[0], t[2]); + x[3] = vsubq_s32(t[1], t[3]); + x[4] = add_dct_const_round_shift_low_8_bd12(s[4], s[6]); + x[5] = add_dct_const_round_shift_low_8_bd12(s[5], s[7]); + x[6] = sub_dct_const_round_shift_low_8_bd12(s[4], s[6]); + x[7] = sub_dct_const_round_shift_low_8_bd12(s[5], s[7]); + + // stage 3 + iadst_half_butterfly_bd12_neon(x + 2, vget_low_s32(c2)); + iadst_half_butterfly_bd12_neon(x + 6, vget_low_s32(c2)); + + *io0 = x[0]; + *io1 = vnegq_s32(x[4]); + *io2 = x[6]; + *io3 = vnegq_s32(x[2]); + *io4 = x[3]; + *io5 = vnegq_s32(x[7]); + *io6 = x[5]; + *io7 = vnegq_s32(x[1]); +} + +void vp9_highbd_iht8x8_64_add_neon(const tran_low_t *input, uint16_t *dest, + int stride, int tx_type, int bd) { + int32x4_t a[16]; + int16x8_t c[8]; + + a[0] = vld1q_s32(input); + a[1] = vld1q_s32(input + 4); + a[2] = vld1q_s32(input + 8); + a[3] = vld1q_s32(input + 12); + a[4] = vld1q_s32(input + 16); + a[5] = vld1q_s32(input + 20); + a[6] = vld1q_s32(input + 24); + a[7] = vld1q_s32(input + 28); + a[8] = vld1q_s32(input + 32); + a[9] = vld1q_s32(input + 36); + a[10] = vld1q_s32(input + 40); + a[11] = vld1q_s32(input + 44); + a[12] = vld1q_s32(input + 48); + a[13] = vld1q_s32(input + 52); + a[14] = vld1q_s32(input + 56); + a[15] = vld1q_s32(input + 60); + + if (bd == 8) { + c[0] = vcombine_s16(vmovn_s32(a[0]), vmovn_s32(a[1])); + c[1] = vcombine_s16(vmovn_s32(a[2]), vmovn_s32(a[3])); + c[2] = vcombine_s16(vmovn_s32(a[4]), vmovn_s32(a[5])); + c[3] = vcombine_s16(vmovn_s32(a[6]), vmovn_s32(a[7])); + c[4] = vcombine_s16(vmovn_s32(a[8]), vmovn_s32(a[9])); + c[5] = vcombine_s16(vmovn_s32(a[10]), vmovn_s32(a[11])); + c[6] = vcombine_s16(vmovn_s32(a[12]), vmovn_s32(a[13])); + c[7] = vcombine_s16(vmovn_s32(a[14]), vmovn_s32(a[15])); + + switch (tx_type) { + case DCT_DCT: { + const int16x8_t cospis = vld1q_s16(kCospi); + const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24 + const int16x4_t cospis1 = vget_high_s16(cospis); // cospi 4, 12, 20, 28 + + idct8x8_64_1d_bd8(cospis0, cospis1, c); + idct8x8_64_1d_bd8(cospis0, cospis1, c); + break; + } + + case ADST_DCT: { + const int16x8_t cospis = vld1q_s16(kCospi); + const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24 + const int16x4_t cospis1 = vget_high_s16(cospis); // cospi 4, 12, 20, 28 + + idct8x8_64_1d_bd8(cospis0, cospis1, c); + transpose_s16_8x8(&c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], + &c[7]); + iadst8(c); + break; + } + + case DCT_ADST: { + const int16x8_t cospis = vld1q_s16(kCospi); + const int16x4_t cospis0 = vget_low_s16(cospis); // cospi 0, 8, 16, 24 + const int16x4_t cospis1 = vget_high_s16(cospis); // cospi 4, 12, 20, 28 + + transpose_s16_8x8(&c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], + &c[7]); + iadst8(c); + idct8x8_64_1d_bd8(cospis0, cospis1, c); + break; + } + + default: { + transpose_s16_8x8(&c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], + &c[7]); + iadst8(c); + transpose_s16_8x8(&c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], + &c[7]); + iadst8(c); + break; + } + } + + c[0] = vrshrq_n_s16(c[0], 5); + c[1] = vrshrq_n_s16(c[1], 5); + c[2] = vrshrq_n_s16(c[2], 5); + c[3] = vrshrq_n_s16(c[3], 5); + c[4] = vrshrq_n_s16(c[4], 5); + c[5] = vrshrq_n_s16(c[5], 5); + c[6] = vrshrq_n_s16(c[6], 5); + c[7] = vrshrq_n_s16(c[7], 5); + } else { + switch (tx_type) { + case DCT_DCT: { + const int32x4_t cospis0 = vld1q_s32(kCospi32); // cospi 0, 8, 16, 24 + const int32x4_t cospis1 = + vld1q_s32(kCospi32 + 4); // cospi 4, 12, 20, 28 + + if (bd == 10) { + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6], &a[7]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11], + &a[12], &a[13], &a[14], &a[15]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9], + &a[2], &a[10], &a[3], &a[11]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13], + &a[6], &a[14], &a[7], &a[15]); + } else { + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6], &a[7]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11], + &a[12], &a[13], &a[14], &a[15]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9], + &a[2], &a[10], &a[3], &a[11]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13], + &a[6], &a[14], &a[7], &a[15]); + } + break; + } + + case ADST_DCT: { + const int32x4_t cospis0 = vld1q_s32(kCospi32); // cospi 0, 8, 16, 24 + const int32x4_t cospis1 = + vld1q_s32(kCospi32 + 4); // cospi 4, 12, 20, 28 + + if (bd == 10) { + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6], &a[7]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11], + &a[12], &a[13], &a[14], &a[15]); + transpose_s32_8x4(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], + &a[11]); + iadst8_bd10(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], &a[11]); + transpose_s32_8x4(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + iadst8_bd10(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + } else { + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6], &a[7]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[8], &a[9], &a[10], &a[11], + &a[12], &a[13], &a[14], &a[15]); + transpose_s32_8x4(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], + &a[11]); + iadst8_bd12(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], &a[11]); + transpose_s32_8x4(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + iadst8_bd12(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + } + break; + } + + case DCT_ADST: { + const int32x4_t cospis0 = vld1q_s32(kCospi32); // cospi 0, 8, 16, 24 + const int32x4_t cospis1 = + vld1q_s32(kCospi32 + 4); // cospi 4, 12, 20, 28 + + if (bd == 10) { + transpose_s32_8x4(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], + &a[7]); + iadst8_bd10(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); + transpose_s32_8x4(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], + &a[14], &a[15]); + iadst8_bd10(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], &a[14], + &a[15]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9], + &a[2], &a[10], &a[3], &a[11]); + idct8x8_64_half1d_bd10(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13], + &a[6], &a[14], &a[7], &a[15]); + } else { + transpose_s32_8x4(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], + &a[7]); + iadst8_bd12(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); + transpose_s32_8x4(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], + &a[14], &a[15]); + iadst8_bd12(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], &a[14], + &a[15]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[0], &a[8], &a[1], &a[9], + &a[2], &a[10], &a[3], &a[11]); + idct8x8_64_half1d_bd12(cospis0, cospis1, &a[4], &a[12], &a[5], &a[13], + &a[6], &a[14], &a[7], &a[15]); + } + break; + } + + default: { + assert(tx_type == ADST_ADST); + if (bd == 10) { + transpose_s32_8x4(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], + &a[7]); + iadst8_bd10(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); + transpose_s32_8x4(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], + &a[14], &a[15]); + iadst8_bd10(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], &a[14], + &a[15]); + transpose_s32_8x4(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], + &a[11]); + iadst8_bd10(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], &a[11]); + transpose_s32_8x4(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + iadst8_bd10(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + } else { + transpose_s32_8x4(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], + &a[7]); + iadst8_bd12(&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); + transpose_s32_8x4(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], + &a[14], &a[15]); + iadst8_bd12(&a[8], &a[9], &a[10], &a[11], &a[12], &a[13], &a[14], + &a[15]); + transpose_s32_8x4(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], + &a[11]); + iadst8_bd12(&a[0], &a[8], &a[1], &a[9], &a[2], &a[10], &a[3], &a[11]); + transpose_s32_8x4(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + iadst8_bd12(&a[4], &a[12], &a[5], &a[13], &a[6], &a[14], &a[7], + &a[15]); + } + break; + } + } + + c[0] = vcombine_s16(vrshrn_n_s32(a[0], 5), vrshrn_n_s32(a[4], 5)); + c[1] = vcombine_s16(vrshrn_n_s32(a[8], 5), vrshrn_n_s32(a[12], 5)); + c[2] = vcombine_s16(vrshrn_n_s32(a[1], 5), vrshrn_n_s32(a[5], 5)); + c[3] = vcombine_s16(vrshrn_n_s32(a[9], 5), vrshrn_n_s32(a[13], 5)); + c[4] = vcombine_s16(vrshrn_n_s32(a[2], 5), vrshrn_n_s32(a[6], 5)); + c[5] = vcombine_s16(vrshrn_n_s32(a[10], 5), vrshrn_n_s32(a[14], 5)); + c[6] = vcombine_s16(vrshrn_n_s32(a[3], 5), vrshrn_n_s32(a[7], 5)); + c[7] = vcombine_s16(vrshrn_n_s32(a[11], 5), vrshrn_n_s32(a[15], 5)); + } + highbd_add8x8(c, dest, stride, bd); +} diff --git a/vp9/common/arm/neon/vp9_iht8x8_add_neon.c b/vp9/common/arm/neon/vp9_iht8x8_add_neon.c index 19163bc87..46ee632e0 100644 --- a/vp9/common/arm/neon/vp9_iht8x8_add_neon.c +++ b/vp9/common/arm/neon/vp9_iht8x8_add_neon.c @@ -14,155 +14,11 @@ #include "./vp9_rtcd.h" #include "./vpx_config.h" #include "vp9/common/vp9_common.h" +#include "vp9/common/arm/neon/vp9_iht_neon.h" #include "vpx_dsp/arm/idct_neon.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/arm/transpose_neon.h" -static INLINE void iadst_half_butterfly_neon(int16x8_t *const x, - const int16x4_t c) { - const int16x8_t sum = vaddq_s16(x[0], x[1]); - const int16x8_t sub = vsubq_s16(x[0], x[1]); - int32x4_t t0[2], t1[2]; - - t0[0] = vmull_lane_s16(vget_low_s16(sum), c, 0); - t0[1] = vmull_lane_s16(vget_high_s16(sum), c, 0); - t1[0] = vmull_lane_s16(vget_low_s16(sub), c, 0); - t1[1] = vmull_lane_s16(vget_high_s16(sub), c, 0); - x[0] = dct_const_round_shift_low_8(t0); - x[1] = dct_const_round_shift_low_8(t1); -} - -static INLINE void iadst_butterfly_lane_0_1_neon(const int16x8_t in0, - const int16x8_t in1, - const int16x4_t c, - int32x4_t *const s0, - int32x4_t *const s1) { - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 0); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 0); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 1); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 1); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 1); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 1); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 0); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 0); -} - -static INLINE void iadst_butterfly_lane_2_3_neon(const int16x8_t in0, - const int16x8_t in1, - const int16x4_t c, - int32x4_t *const s0, - int32x4_t *const s1) { - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 3); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 3); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 2); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 2); -} - -static INLINE void iadst_butterfly_lane_3_2_neon(const int16x8_t in0, - const int16x8_t in1, - const int16x4_t c, - int32x4_t *const s0, - int32x4_t *const s1) { - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 2); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 2); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 3); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 3); -} - -static INLINE int16x8_t add_dct_const_round_shift_low_8( - const int32x4_t *const in0, const int32x4_t *const in1) { - int32x4_t sum[2]; - - sum[0] = vaddq_s32(in0[0], in1[0]); - sum[1] = vaddq_s32(in0[1], in1[1]); - return dct_const_round_shift_low_8(sum); -} - -static INLINE int16x8_t sub_dct_const_round_shift_low_8( - const int32x4_t *const in0, const int32x4_t *const in1) { - int32x4_t sum[2]; - - sum[0] = vsubq_s32(in0[0], in1[0]); - sum[1] = vsubq_s32(in0[1], in1[1]); - return dct_const_round_shift_low_8(sum); -} - -static INLINE void iadst8(int16x8_t *const io) { - const int16x4_t c0 = - create_s16x4_neon(cospi_2_64, cospi_30_64, cospi_10_64, cospi_22_64); - const int16x4_t c1 = - create_s16x4_neon(cospi_18_64, cospi_14_64, cospi_26_64, cospi_6_64); - const int16x4_t c2 = - create_s16x4_neon(cospi_16_64, 0, cospi_8_64, cospi_24_64); - int16x8_t x[8], t[4]; - int32x4_t s0[2], s1[2], s2[2], s3[2], s4[2], s5[2], s6[2], s7[2]; - - x[0] = io[7]; - x[1] = io[0]; - x[2] = io[5]; - x[3] = io[2]; - x[4] = io[3]; - x[5] = io[4]; - x[6] = io[1]; - x[7] = io[6]; - - // stage 1 - iadst_butterfly_lane_0_1_neon(x[0], x[1], c0, s0, s1); - iadst_butterfly_lane_2_3_neon(x[2], x[3], c0, s2, s3); - iadst_butterfly_lane_0_1_neon(x[4], x[5], c1, s4, s5); - iadst_butterfly_lane_2_3_neon(x[6], x[7], c1, s6, s7); - - x[0] = add_dct_const_round_shift_low_8(s0, s4); - x[1] = add_dct_const_round_shift_low_8(s1, s5); - x[2] = add_dct_const_round_shift_low_8(s2, s6); - x[3] = add_dct_const_round_shift_low_8(s3, s7); - x[4] = sub_dct_const_round_shift_low_8(s0, s4); - x[5] = sub_dct_const_round_shift_low_8(s1, s5); - x[6] = sub_dct_const_round_shift_low_8(s2, s6); - x[7] = sub_dct_const_round_shift_low_8(s3, s7); - - // stage 2 - t[0] = x[0]; - t[1] = x[1]; - t[2] = x[2]; - t[3] = x[3]; - iadst_butterfly_lane_2_3_neon(x[4], x[5], c2, s4, s5); - iadst_butterfly_lane_3_2_neon(x[7], x[6], c2, s7, s6); - - x[0] = vaddq_s16(t[0], t[2]); - x[1] = vaddq_s16(t[1], t[3]); - x[2] = vsubq_s16(t[0], t[2]); - x[3] = vsubq_s16(t[1], t[3]); - x[4] = add_dct_const_round_shift_low_8(s4, s6); - x[5] = add_dct_const_round_shift_low_8(s5, s7); - x[6] = sub_dct_const_round_shift_low_8(s4, s6); - x[7] = sub_dct_const_round_shift_low_8(s5, s7); - - // stage 3 - iadst_half_butterfly_neon(x + 2, c2); - iadst_half_butterfly_neon(x + 6, c2); - - io[0] = x[0]; - io[1] = vnegq_s16(x[4]); - io[2] = x[6]; - io[3] = vnegq_s16(x[2]); - io[4] = x[3]; - io[5] = vnegq_s16(x[7]); - io[6] = x[5]; - io[7] = vnegq_s16(x[1]); -} - void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type) { const int16x8_t cospis = vld1q_s16(kCospi); diff --git a/vp9/common/arm/neon/vp9_iht_neon.h b/vp9/common/arm/neon/vp9_iht_neon.h index 08daa1a4a..e98084b45 100644 --- a/vp9/common/arm/neon/vp9_iht_neon.h +++ b/vp9/common/arm/neon/vp9_iht_neon.h @@ -57,4 +57,149 @@ static INLINE void iadst4(int16x8_t *const io) { dct_const_round_shift_low_8_dual(output, &io[0], &io[1]); } +static INLINE void iadst_half_butterfly_neon(int16x8_t *const x, + const int16x4_t c) { + const int16x8_t sum = vaddq_s16(x[0], x[1]); + const int16x8_t sub = vsubq_s16(x[0], x[1]); + int32x4_t t0[2], t1[2]; + + t0[0] = vmull_lane_s16(vget_low_s16(sum), c, 0); + t0[1] = vmull_lane_s16(vget_high_s16(sum), c, 0); + t1[0] = vmull_lane_s16(vget_low_s16(sub), c, 0); + t1[1] = vmull_lane_s16(vget_high_s16(sub), c, 0); + x[0] = dct_const_round_shift_low_8(t0); + x[1] = dct_const_round_shift_low_8(t1); +} + +static INLINE void iadst_butterfly_lane_0_1_neon(const int16x8_t in0, + const int16x8_t in1, + const int16x4_t c, + int32x4_t *const s0, + int32x4_t *const s1) { + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 0); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 0); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 1); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 1); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 1); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 1); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 0); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 0); +} + +static INLINE void iadst_butterfly_lane_2_3_neon(const int16x8_t in0, + const int16x8_t in1, + const int16x4_t c, + int32x4_t *const s0, + int32x4_t *const s1) { + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 3); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 3); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 2); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 2); +} + +static INLINE void iadst_butterfly_lane_3_2_neon(const int16x8_t in0, + const int16x8_t in1, + const int16x4_t c, + int32x4_t *const s0, + int32x4_t *const s1) { + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 2); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 2); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 3); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 3); +} + +static INLINE int16x8_t add_dct_const_round_shift_low_8( + const int32x4_t *const in0, const int32x4_t *const in1) { + int32x4_t sum[2]; + + sum[0] = vaddq_s32(in0[0], in1[0]); + sum[1] = vaddq_s32(in0[1], in1[1]); + return dct_const_round_shift_low_8(sum); +} + +static INLINE int16x8_t sub_dct_const_round_shift_low_8( + const int32x4_t *const in0, const int32x4_t *const in1) { + int32x4_t sum[2]; + + sum[0] = vsubq_s32(in0[0], in1[0]); + sum[1] = vsubq_s32(in0[1], in1[1]); + return dct_const_round_shift_low_8(sum); +} + +static INLINE void iadst8(int16x8_t *const io) { + const int16x4_t c0 = + create_s16x4_neon(cospi_2_64, cospi_30_64, cospi_10_64, cospi_22_64); + const int16x4_t c1 = + create_s16x4_neon(cospi_18_64, cospi_14_64, cospi_26_64, cospi_6_64); + const int16x4_t c2 = + create_s16x4_neon(cospi_16_64, 0, cospi_8_64, cospi_24_64); + int16x8_t x[8], t[4]; + int32x4_t s0[2], s1[2], s2[2], s3[2], s4[2], s5[2], s6[2], s7[2]; + + x[0] = io[7]; + x[1] = io[0]; + x[2] = io[5]; + x[3] = io[2]; + x[4] = io[3]; + x[5] = io[4]; + x[6] = io[1]; + x[7] = io[6]; + + // stage 1 + iadst_butterfly_lane_0_1_neon(x[0], x[1], c0, s0, s1); + iadst_butterfly_lane_2_3_neon(x[2], x[3], c0, s2, s3); + iadst_butterfly_lane_0_1_neon(x[4], x[5], c1, s4, s5); + iadst_butterfly_lane_2_3_neon(x[6], x[7], c1, s6, s7); + + x[0] = add_dct_const_round_shift_low_8(s0, s4); + x[1] = add_dct_const_round_shift_low_8(s1, s5); + x[2] = add_dct_const_round_shift_low_8(s2, s6); + x[3] = add_dct_const_round_shift_low_8(s3, s7); + x[4] = sub_dct_const_round_shift_low_8(s0, s4); + x[5] = sub_dct_const_round_shift_low_8(s1, s5); + x[6] = sub_dct_const_round_shift_low_8(s2, s6); + x[7] = sub_dct_const_round_shift_low_8(s3, s7); + + // stage 2 + t[0] = x[0]; + t[1] = x[1]; + t[2] = x[2]; + t[3] = x[3]; + iadst_butterfly_lane_2_3_neon(x[4], x[5], c2, s4, s5); + iadst_butterfly_lane_3_2_neon(x[7], x[6], c2, s7, s6); + + x[0] = vaddq_s16(t[0], t[2]); + x[1] = vaddq_s16(t[1], t[3]); + x[2] = vsubq_s16(t[0], t[2]); + x[3] = vsubq_s16(t[1], t[3]); + x[4] = add_dct_const_round_shift_low_8(s4, s6); + x[5] = add_dct_const_round_shift_low_8(s5, s7); + x[6] = sub_dct_const_round_shift_low_8(s4, s6); + x[7] = sub_dct_const_round_shift_low_8(s5, s7); + + // stage 3 + iadst_half_butterfly_neon(x + 2, c2); + iadst_half_butterfly_neon(x + 6, c2); + + io[0] = x[0]; + io[1] = vnegq_s16(x[4]); + io[2] = x[6]; + io[3] = vnegq_s16(x[2]); + io[4] = x[3]; + io[5] = vnegq_s16(x[7]); + io[6] = x[5]; + io[7] = vnegq_s16(x[1]); +} + #endif // VP9_COMMON_ARM_NEON_VP9_IHT_NEON_H_ diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl index d048857dd..5dbe5a659 100644 --- a/vp9/common/vp9_rtcd_defs.pl +++ b/vp9/common/vp9_rtcd_defs.pl @@ -104,7 +104,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { if (vpx_config("CONFIG_EMULATE_HARDWARE") ne "yes") { specialize qw/vp9_highbd_iht4x4_16_add neon sse4_1/; - specialize qw/vp9_highbd_iht8x8_64_add sse4_1/; + specialize qw/vp9_highbd_iht8x8_64_add neon sse4_1/; specialize qw/vp9_highbd_iht16x16_256_add sse4_1/; } } diff --git a/vp9/vp9_common.mk b/vp9/vp9_common.mk index 721e170bf..1b21c4342 100644 --- a/vp9/vp9_common.mk +++ b/vp9/vp9_common.mk @@ -83,6 +83,7 @@ VP9_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/vp9_itrans8_dspr2.c VP9_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/vp9_itrans16_dspr2.c else VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_highbd_iht4x4_add_neon.c +VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_highbd_iht8x8_add_neon.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht4x4_add_sse4.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht8x8_add_sse4.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht16x16_add_sse4.c diff --git a/vpx_dsp/arm/highbd_idct8x8_add_neon.c b/vpx_dsp/arm/highbd_idct8x8_add_neon.c index e51e574cc..bed3227ca 100644 --- a/vpx_dsp/arm/highbd_idct8x8_add_neon.c +++ b/vpx_dsp/arm/highbd_idct8x8_add_neon.c @@ -11,6 +11,7 @@ #include #include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/arm/highbd_idct_neon.h" #include "vpx_dsp/arm/idct_neon.h" #include "vpx_dsp/arm/transpose_neon.h" #include "vpx_dsp/inv_txfm.h" @@ -222,73 +223,6 @@ static INLINE void idct8x8_12_half1d_bd12( *io7 = vsubq_s32(step1[0], step2[7]); } -static INLINE void highbd_add8x8(int16x8_t *const a, uint16_t *dest, - const int stride, const int bd) { - const int16x8_t max = vdupq_n_s16((1 << bd) - 1); - const uint16_t *dst = dest; - uint16x8_t d0, d1, d2, d3, d4, d5, d6, d7; - uint16x8_t d0_u16, d1_u16, d2_u16, d3_u16, d4_u16, d5_u16, d6_u16, d7_u16; - int16x8_t d0_s16, d1_s16, d2_s16, d3_s16, d4_s16, d5_s16, d6_s16, d7_s16; - - d0 = vld1q_u16(dst); - dst += stride; - d1 = vld1q_u16(dst); - dst += stride; - d2 = vld1q_u16(dst); - dst += stride; - d3 = vld1q_u16(dst); - dst += stride; - d4 = vld1q_u16(dst); - dst += stride; - d5 = vld1q_u16(dst); - dst += stride; - d6 = vld1q_u16(dst); - dst += stride; - d7 = vld1q_u16(dst); - - d0_s16 = vqaddq_s16(a[0], vreinterpretq_s16_u16(d0)); - d1_s16 = vqaddq_s16(a[1], vreinterpretq_s16_u16(d1)); - d2_s16 = vqaddq_s16(a[2], vreinterpretq_s16_u16(d2)); - d3_s16 = vqaddq_s16(a[3], vreinterpretq_s16_u16(d3)); - d4_s16 = vqaddq_s16(a[4], vreinterpretq_s16_u16(d4)); - d5_s16 = vqaddq_s16(a[5], vreinterpretq_s16_u16(d5)); - d6_s16 = vqaddq_s16(a[6], vreinterpretq_s16_u16(d6)); - d7_s16 = vqaddq_s16(a[7], vreinterpretq_s16_u16(d7)); - - d0_s16 = vminq_s16(d0_s16, max); - d1_s16 = vminq_s16(d1_s16, max); - d2_s16 = vminq_s16(d2_s16, max); - d3_s16 = vminq_s16(d3_s16, max); - d4_s16 = vminq_s16(d4_s16, max); - d5_s16 = vminq_s16(d5_s16, max); - d6_s16 = vminq_s16(d6_s16, max); - d7_s16 = vminq_s16(d7_s16, max); - d0_u16 = vqshluq_n_s16(d0_s16, 0); - d1_u16 = vqshluq_n_s16(d1_s16, 0); - d2_u16 = vqshluq_n_s16(d2_s16, 0); - d3_u16 = vqshluq_n_s16(d3_s16, 0); - d4_u16 = vqshluq_n_s16(d4_s16, 0); - d5_u16 = vqshluq_n_s16(d5_s16, 0); - d6_u16 = vqshluq_n_s16(d6_s16, 0); - d7_u16 = vqshluq_n_s16(d7_s16, 0); - - vst1q_u16(dest, d0_u16); - dest += stride; - vst1q_u16(dest, d1_u16); - dest += stride; - vst1q_u16(dest, d2_u16); - dest += stride; - vst1q_u16(dest, d3_u16); - dest += stride; - vst1q_u16(dest, d4_u16); - dest += stride; - vst1q_u16(dest, d5_u16); - dest += stride; - vst1q_u16(dest, d6_u16); - dest += stride; - vst1q_u16(dest, d7_u16); -} - void vpx_highbd_idct8x8_12_add_neon(const tran_low_t *input, uint16_t *dest, int stride, int bd) { int32x4_t a[16]; @@ -353,202 +287,6 @@ void vpx_highbd_idct8x8_12_add_neon(const tran_low_t *input, uint16_t *dest, highbd_add8x8(c, dest, stride, bd); } -static INLINE void idct8x8_64_half1d_bd10( - const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0, - int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3, - int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6, - int32x4_t *const io7) { - int32x4_t step1[8], step2[8]; - - transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7); - - // stage 1 - step1[4] = vmulq_lane_s32(*io1, vget_high_s32(cospis1), 1); - step1[5] = vmulq_lane_s32(*io3, vget_high_s32(cospis1), 0); - step1[6] = vmulq_lane_s32(*io3, vget_low_s32(cospis1), 1); - step1[7] = vmulq_lane_s32(*io1, vget_low_s32(cospis1), 0); - - step1[4] = vmlsq_lane_s32(step1[4], *io7, vget_low_s32(cospis1), 0); - step1[5] = vmlaq_lane_s32(step1[5], *io5, vget_low_s32(cospis1), 1); - step1[6] = vmlsq_lane_s32(step1[6], *io5, vget_high_s32(cospis1), 0); - step1[7] = vmlaq_lane_s32(step1[7], *io7, vget_high_s32(cospis1), 1); - - step1[4] = vrshrq_n_s32(step1[4], DCT_CONST_BITS); - step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS); - step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS); - step1[7] = vrshrq_n_s32(step1[7], DCT_CONST_BITS); - - // stage 2 - step2[1] = vmulq_lane_s32(*io0, vget_high_s32(cospis0), 0); - step2[2] = vmulq_lane_s32(*io2, vget_high_s32(cospis0), 1); - step2[3] = vmulq_lane_s32(*io2, vget_low_s32(cospis0), 1); - - step2[0] = vmlaq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0); - step2[1] = vmlsq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0); - step2[2] = vmlsq_lane_s32(step2[2], *io6, vget_low_s32(cospis0), 1); - step2[3] = vmlaq_lane_s32(step2[3], *io6, vget_high_s32(cospis0), 1); - - step2[0] = vrshrq_n_s32(step2[0], DCT_CONST_BITS); - step2[1] = vrshrq_n_s32(step2[1], DCT_CONST_BITS); - step2[2] = vrshrq_n_s32(step2[2], DCT_CONST_BITS); - step2[3] = vrshrq_n_s32(step2[3], DCT_CONST_BITS); - - step2[4] = vaddq_s32(step1[4], step1[5]); - step2[5] = vsubq_s32(step1[4], step1[5]); - step2[6] = vsubq_s32(step1[7], step1[6]); - step2[7] = vaddq_s32(step1[7], step1[6]); - - // stage 3 - step1[0] = vaddq_s32(step2[0], step2[3]); - step1[1] = vaddq_s32(step2[1], step2[2]); - step1[2] = vsubq_s32(step2[1], step2[2]); - step1[3] = vsubq_s32(step2[0], step2[3]); - - step1[6] = vmulq_lane_s32(step2[6], vget_high_s32(cospis0), 0); - step1[5] = vmlsq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0); - step1[6] = vmlaq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0); - step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS); - step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS); - - // stage 4 - *io0 = vaddq_s32(step1[0], step2[7]); - *io1 = vaddq_s32(step1[1], step1[6]); - *io2 = vaddq_s32(step1[2], step1[5]); - *io3 = vaddq_s32(step1[3], step2[4]); - *io4 = vsubq_s32(step1[3], step2[4]); - *io5 = vsubq_s32(step1[2], step1[5]); - *io6 = vsubq_s32(step1[1], step1[6]); - *io7 = vsubq_s32(step1[0], step2[7]); -} - -static INLINE void idct8x8_64_half1d_bd12( - const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0, - int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3, - int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6, - int32x4_t *const io7) { - int32x2_t input1l, input1h, input3l, input3h, input5l, input5h, input7l, - input7h; - int32x2_t step1l[4], step1h[4]; - int32x4_t step1[8], step2[8]; - int64x2_t t64[8]; - int32x2_t t32[8]; - - transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7); - - // stage 1 - input1l = vget_low_s32(*io1); - input1h = vget_high_s32(*io1); - input3l = vget_low_s32(*io3); - input3h = vget_high_s32(*io3); - input5l = vget_low_s32(*io5); - input5h = vget_high_s32(*io5); - input7l = vget_low_s32(*io7); - input7h = vget_high_s32(*io7); - step1l[0] = vget_low_s32(*io0); - step1h[0] = vget_high_s32(*io0); - step1l[1] = vget_low_s32(*io2); - step1h[1] = vget_high_s32(*io2); - step1l[2] = vget_low_s32(*io4); - step1h[2] = vget_high_s32(*io4); - step1l[3] = vget_low_s32(*io6); - step1h[3] = vget_high_s32(*io6); - - t64[0] = vmull_lane_s32(input1l, vget_high_s32(cospis1), 1); - t64[1] = vmull_lane_s32(input1h, vget_high_s32(cospis1), 1); - t64[2] = vmull_lane_s32(input3l, vget_high_s32(cospis1), 0); - t64[3] = vmull_lane_s32(input3h, vget_high_s32(cospis1), 0); - t64[4] = vmull_lane_s32(input3l, vget_low_s32(cospis1), 1); - t64[5] = vmull_lane_s32(input3h, vget_low_s32(cospis1), 1); - t64[6] = vmull_lane_s32(input1l, vget_low_s32(cospis1), 0); - t64[7] = vmull_lane_s32(input1h, vget_low_s32(cospis1), 0); - t64[0] = vmlsl_lane_s32(t64[0], input7l, vget_low_s32(cospis1), 0); - t64[1] = vmlsl_lane_s32(t64[1], input7h, vget_low_s32(cospis1), 0); - t64[2] = vmlal_lane_s32(t64[2], input5l, vget_low_s32(cospis1), 1); - t64[3] = vmlal_lane_s32(t64[3], input5h, vget_low_s32(cospis1), 1); - t64[4] = vmlsl_lane_s32(t64[4], input5l, vget_high_s32(cospis1), 0); - t64[5] = vmlsl_lane_s32(t64[5], input5h, vget_high_s32(cospis1), 0); - t64[6] = vmlal_lane_s32(t64[6], input7l, vget_high_s32(cospis1), 1); - t64[7] = vmlal_lane_s32(t64[7], input7h, vget_high_s32(cospis1), 1); - t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); - t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); - t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); - t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); - t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS); - t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS); - t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS); - t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS); - step1[4] = vcombine_s32(t32[0], t32[1]); - step1[5] = vcombine_s32(t32[2], t32[3]); - step1[6] = vcombine_s32(t32[4], t32[5]); - step1[7] = vcombine_s32(t32[6], t32[7]); - - // stage 2 - t64[2] = vmull_lane_s32(step1l[0], vget_high_s32(cospis0), 0); - t64[3] = vmull_lane_s32(step1h[0], vget_high_s32(cospis0), 0); - t64[4] = vmull_lane_s32(step1l[1], vget_high_s32(cospis0), 1); - t64[5] = vmull_lane_s32(step1h[1], vget_high_s32(cospis0), 1); - t64[6] = vmull_lane_s32(step1l[1], vget_low_s32(cospis0), 1); - t64[7] = vmull_lane_s32(step1h[1], vget_low_s32(cospis0), 1); - t64[0] = vmlal_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0); - t64[1] = vmlal_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0); - t64[2] = vmlsl_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0); - t64[3] = vmlsl_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0); - t64[4] = vmlsl_lane_s32(t64[4], step1l[3], vget_low_s32(cospis0), 1); - t64[5] = vmlsl_lane_s32(t64[5], step1h[3], vget_low_s32(cospis0), 1); - t64[6] = vmlal_lane_s32(t64[6], step1l[3], vget_high_s32(cospis0), 1); - t64[7] = vmlal_lane_s32(t64[7], step1h[3], vget_high_s32(cospis0), 1); - t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); - t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); - t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); - t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); - t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS); - t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS); - t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS); - t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS); - step2[0] = vcombine_s32(t32[0], t32[1]); - step2[1] = vcombine_s32(t32[2], t32[3]); - step2[2] = vcombine_s32(t32[4], t32[5]); - step2[3] = vcombine_s32(t32[6], t32[7]); - - step2[4] = vaddq_s32(step1[4], step1[5]); - step2[5] = vsubq_s32(step1[4], step1[5]); - step2[6] = vsubq_s32(step1[7], step1[6]); - step2[7] = vaddq_s32(step1[7], step1[6]); - - // stage 3 - step1[0] = vaddq_s32(step2[0], step2[3]); - step1[1] = vaddq_s32(step2[1], step2[2]); - step1[2] = vsubq_s32(step2[1], step2[2]); - step1[3] = vsubq_s32(step2[0], step2[3]); - - t64[2] = vmull_lane_s32(vget_low_s32(step2[6]), vget_high_s32(cospis0), 0); - t64[3] = vmull_lane_s32(vget_high_s32(step2[6]), vget_high_s32(cospis0), 0); - t64[0] = - vmlsl_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0); - t64[1] = vmlsl_lane_s32(t64[3], vget_high_s32(step2[5]), - vget_high_s32(cospis0), 0); - t64[2] = - vmlal_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0); - t64[3] = vmlal_lane_s32(t64[3], vget_high_s32(step2[5]), - vget_high_s32(cospis0), 0); - t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); - t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); - t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); - t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); - step1[5] = vcombine_s32(t32[0], t32[1]); - step1[6] = vcombine_s32(t32[2], t32[3]); - - // stage 4 - *io0 = vaddq_s32(step1[0], step2[7]); - *io1 = vaddq_s32(step1[1], step1[6]); - *io2 = vaddq_s32(step1[2], step1[5]); - *io3 = vaddq_s32(step1[3], step2[4]); - *io4 = vsubq_s32(step1[3], step2[4]); - *io5 = vsubq_s32(step1[2], step1[5]); - *io6 = vsubq_s32(step1[1], step1[6]); - *io7 = vsubq_s32(step1[0], step2[7]); -} - void vpx_highbd_idct8x8_64_add_neon(const tran_low_t *input, uint16_t *dest, int stride, int bd) { int32x4_t a[16]; diff --git a/vpx_dsp/arm/highbd_idct_neon.h b/vpx_dsp/arm/highbd_idct_neon.h index 92fcb7f3a..612bcf5a5 100644 --- a/vpx_dsp/arm/highbd_idct_neon.h +++ b/vpx_dsp/arm/highbd_idct_neon.h @@ -96,4 +96,267 @@ static INLINE void idct4x4_16_kernel_bd12(const int32x4_t cospis, a[3] = vsubq_s32(b0, b3); } +static INLINE void highbd_add8x8(int16x8_t *const a, uint16_t *dest, + const int stride, const int bd) { + const int16x8_t max = vdupq_n_s16((1 << bd) - 1); + const uint16_t *dst = dest; + uint16x8_t d0, d1, d2, d3, d4, d5, d6, d7; + uint16x8_t d0_u16, d1_u16, d2_u16, d3_u16, d4_u16, d5_u16, d6_u16, d7_u16; + int16x8_t d0_s16, d1_s16, d2_s16, d3_s16, d4_s16, d5_s16, d6_s16, d7_s16; + + d0 = vld1q_u16(dst); + dst += stride; + d1 = vld1q_u16(dst); + dst += stride; + d2 = vld1q_u16(dst); + dst += stride; + d3 = vld1q_u16(dst); + dst += stride; + d4 = vld1q_u16(dst); + dst += stride; + d5 = vld1q_u16(dst); + dst += stride; + d6 = vld1q_u16(dst); + dst += stride; + d7 = vld1q_u16(dst); + + d0_s16 = vqaddq_s16(a[0], vreinterpretq_s16_u16(d0)); + d1_s16 = vqaddq_s16(a[1], vreinterpretq_s16_u16(d1)); + d2_s16 = vqaddq_s16(a[2], vreinterpretq_s16_u16(d2)); + d3_s16 = vqaddq_s16(a[3], vreinterpretq_s16_u16(d3)); + d4_s16 = vqaddq_s16(a[4], vreinterpretq_s16_u16(d4)); + d5_s16 = vqaddq_s16(a[5], vreinterpretq_s16_u16(d5)); + d6_s16 = vqaddq_s16(a[6], vreinterpretq_s16_u16(d6)); + d7_s16 = vqaddq_s16(a[7], vreinterpretq_s16_u16(d7)); + + d0_s16 = vminq_s16(d0_s16, max); + d1_s16 = vminq_s16(d1_s16, max); + d2_s16 = vminq_s16(d2_s16, max); + d3_s16 = vminq_s16(d3_s16, max); + d4_s16 = vminq_s16(d4_s16, max); + d5_s16 = vminq_s16(d5_s16, max); + d6_s16 = vminq_s16(d6_s16, max); + d7_s16 = vminq_s16(d7_s16, max); + d0_u16 = vqshluq_n_s16(d0_s16, 0); + d1_u16 = vqshluq_n_s16(d1_s16, 0); + d2_u16 = vqshluq_n_s16(d2_s16, 0); + d3_u16 = vqshluq_n_s16(d3_s16, 0); + d4_u16 = vqshluq_n_s16(d4_s16, 0); + d5_u16 = vqshluq_n_s16(d5_s16, 0); + d6_u16 = vqshluq_n_s16(d6_s16, 0); + d7_u16 = vqshluq_n_s16(d7_s16, 0); + + vst1q_u16(dest, d0_u16); + dest += stride; + vst1q_u16(dest, d1_u16); + dest += stride; + vst1q_u16(dest, d2_u16); + dest += stride; + vst1q_u16(dest, d3_u16); + dest += stride; + vst1q_u16(dest, d4_u16); + dest += stride; + vst1q_u16(dest, d5_u16); + dest += stride; + vst1q_u16(dest, d6_u16); + dest += stride; + vst1q_u16(dest, d7_u16); +} + +static INLINE void idct8x8_64_half1d_bd10( + const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0, + int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3, + int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6, + int32x4_t *const io7) { + int32x4_t step1[8], step2[8]; + + transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7); + + // stage 1 + step1[4] = vmulq_lane_s32(*io1, vget_high_s32(cospis1), 1); + step1[5] = vmulq_lane_s32(*io3, vget_high_s32(cospis1), 0); + step1[6] = vmulq_lane_s32(*io3, vget_low_s32(cospis1), 1); + step1[7] = vmulq_lane_s32(*io1, vget_low_s32(cospis1), 0); + + step1[4] = vmlsq_lane_s32(step1[4], *io7, vget_low_s32(cospis1), 0); + step1[5] = vmlaq_lane_s32(step1[5], *io5, vget_low_s32(cospis1), 1); + step1[6] = vmlsq_lane_s32(step1[6], *io5, vget_high_s32(cospis1), 0); + step1[7] = vmlaq_lane_s32(step1[7], *io7, vget_high_s32(cospis1), 1); + + step1[4] = vrshrq_n_s32(step1[4], DCT_CONST_BITS); + step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS); + step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS); + step1[7] = vrshrq_n_s32(step1[7], DCT_CONST_BITS); + + // stage 2 + step2[1] = vmulq_lane_s32(*io0, vget_high_s32(cospis0), 0); + step2[2] = vmulq_lane_s32(*io2, vget_high_s32(cospis0), 1); + step2[3] = vmulq_lane_s32(*io2, vget_low_s32(cospis0), 1); + + step2[0] = vmlaq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0); + step2[1] = vmlsq_lane_s32(step2[1], *io4, vget_high_s32(cospis0), 0); + step2[2] = vmlsq_lane_s32(step2[2], *io6, vget_low_s32(cospis0), 1); + step2[3] = vmlaq_lane_s32(step2[3], *io6, vget_high_s32(cospis0), 1); + + step2[0] = vrshrq_n_s32(step2[0], DCT_CONST_BITS); + step2[1] = vrshrq_n_s32(step2[1], DCT_CONST_BITS); + step2[2] = vrshrq_n_s32(step2[2], DCT_CONST_BITS); + step2[3] = vrshrq_n_s32(step2[3], DCT_CONST_BITS); + + step2[4] = vaddq_s32(step1[4], step1[5]); + step2[5] = vsubq_s32(step1[4], step1[5]); + step2[6] = vsubq_s32(step1[7], step1[6]); + step2[7] = vaddq_s32(step1[7], step1[6]); + + // stage 3 + step1[0] = vaddq_s32(step2[0], step2[3]); + step1[1] = vaddq_s32(step2[1], step2[2]); + step1[2] = vsubq_s32(step2[1], step2[2]); + step1[3] = vsubq_s32(step2[0], step2[3]); + + step1[6] = vmulq_lane_s32(step2[6], vget_high_s32(cospis0), 0); + step1[5] = vmlsq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0); + step1[6] = vmlaq_lane_s32(step1[6], step2[5], vget_high_s32(cospis0), 0); + step1[5] = vrshrq_n_s32(step1[5], DCT_CONST_BITS); + step1[6] = vrshrq_n_s32(step1[6], DCT_CONST_BITS); + + // stage 4 + *io0 = vaddq_s32(step1[0], step2[7]); + *io1 = vaddq_s32(step1[1], step1[6]); + *io2 = vaddq_s32(step1[2], step1[5]); + *io3 = vaddq_s32(step1[3], step2[4]); + *io4 = vsubq_s32(step1[3], step2[4]); + *io5 = vsubq_s32(step1[2], step1[5]); + *io6 = vsubq_s32(step1[1], step1[6]); + *io7 = vsubq_s32(step1[0], step2[7]); +} + +static INLINE void idct8x8_64_half1d_bd12( + const int32x4_t cospis0, const int32x4_t cospis1, int32x4_t *const io0, + int32x4_t *const io1, int32x4_t *const io2, int32x4_t *const io3, + int32x4_t *const io4, int32x4_t *const io5, int32x4_t *const io6, + int32x4_t *const io7) { + int32x2_t input1l, input1h, input3l, input3h, input5l, input5h, input7l, + input7h; + int32x2_t step1l[4], step1h[4]; + int32x4_t step1[8], step2[8]; + int64x2_t t64[8]; + int32x2_t t32[8]; + + transpose_s32_8x4(io0, io1, io2, io3, io4, io5, io6, io7); + + // stage 1 + input1l = vget_low_s32(*io1); + input1h = vget_high_s32(*io1); + input3l = vget_low_s32(*io3); + input3h = vget_high_s32(*io3); + input5l = vget_low_s32(*io5); + input5h = vget_high_s32(*io5); + input7l = vget_low_s32(*io7); + input7h = vget_high_s32(*io7); + step1l[0] = vget_low_s32(*io0); + step1h[0] = vget_high_s32(*io0); + step1l[1] = vget_low_s32(*io2); + step1h[1] = vget_high_s32(*io2); + step1l[2] = vget_low_s32(*io4); + step1h[2] = vget_high_s32(*io4); + step1l[3] = vget_low_s32(*io6); + step1h[3] = vget_high_s32(*io6); + + t64[0] = vmull_lane_s32(input1l, vget_high_s32(cospis1), 1); + t64[1] = vmull_lane_s32(input1h, vget_high_s32(cospis1), 1); + t64[2] = vmull_lane_s32(input3l, vget_high_s32(cospis1), 0); + t64[3] = vmull_lane_s32(input3h, vget_high_s32(cospis1), 0); + t64[4] = vmull_lane_s32(input3l, vget_low_s32(cospis1), 1); + t64[5] = vmull_lane_s32(input3h, vget_low_s32(cospis1), 1); + t64[6] = vmull_lane_s32(input1l, vget_low_s32(cospis1), 0); + t64[7] = vmull_lane_s32(input1h, vget_low_s32(cospis1), 0); + t64[0] = vmlsl_lane_s32(t64[0], input7l, vget_low_s32(cospis1), 0); + t64[1] = vmlsl_lane_s32(t64[1], input7h, vget_low_s32(cospis1), 0); + t64[2] = vmlal_lane_s32(t64[2], input5l, vget_low_s32(cospis1), 1); + t64[3] = vmlal_lane_s32(t64[3], input5h, vget_low_s32(cospis1), 1); + t64[4] = vmlsl_lane_s32(t64[4], input5l, vget_high_s32(cospis1), 0); + t64[5] = vmlsl_lane_s32(t64[5], input5h, vget_high_s32(cospis1), 0); + t64[6] = vmlal_lane_s32(t64[6], input7l, vget_high_s32(cospis1), 1); + t64[7] = vmlal_lane_s32(t64[7], input7h, vget_high_s32(cospis1), 1); + t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); + t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); + t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); + t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); + t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS); + t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS); + t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS); + t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS); + step1[4] = vcombine_s32(t32[0], t32[1]); + step1[5] = vcombine_s32(t32[2], t32[3]); + step1[6] = vcombine_s32(t32[4], t32[5]); + step1[7] = vcombine_s32(t32[6], t32[7]); + + // stage 2 + t64[2] = vmull_lane_s32(step1l[0], vget_high_s32(cospis0), 0); + t64[3] = vmull_lane_s32(step1h[0], vget_high_s32(cospis0), 0); + t64[4] = vmull_lane_s32(step1l[1], vget_high_s32(cospis0), 1); + t64[5] = vmull_lane_s32(step1h[1], vget_high_s32(cospis0), 1); + t64[6] = vmull_lane_s32(step1l[1], vget_low_s32(cospis0), 1); + t64[7] = vmull_lane_s32(step1h[1], vget_low_s32(cospis0), 1); + t64[0] = vmlal_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0); + t64[1] = vmlal_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0); + t64[2] = vmlsl_lane_s32(t64[2], step1l[2], vget_high_s32(cospis0), 0); + t64[3] = vmlsl_lane_s32(t64[3], step1h[2], vget_high_s32(cospis0), 0); + t64[4] = vmlsl_lane_s32(t64[4], step1l[3], vget_low_s32(cospis0), 1); + t64[5] = vmlsl_lane_s32(t64[5], step1h[3], vget_low_s32(cospis0), 1); + t64[6] = vmlal_lane_s32(t64[6], step1l[3], vget_high_s32(cospis0), 1); + t64[7] = vmlal_lane_s32(t64[7], step1h[3], vget_high_s32(cospis0), 1); + t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); + t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); + t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); + t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); + t32[4] = vrshrn_n_s64(t64[4], DCT_CONST_BITS); + t32[5] = vrshrn_n_s64(t64[5], DCT_CONST_BITS); + t32[6] = vrshrn_n_s64(t64[6], DCT_CONST_BITS); + t32[7] = vrshrn_n_s64(t64[7], DCT_CONST_BITS); + step2[0] = vcombine_s32(t32[0], t32[1]); + step2[1] = vcombine_s32(t32[2], t32[3]); + step2[2] = vcombine_s32(t32[4], t32[5]); + step2[3] = vcombine_s32(t32[6], t32[7]); + + step2[4] = vaddq_s32(step1[4], step1[5]); + step2[5] = vsubq_s32(step1[4], step1[5]); + step2[6] = vsubq_s32(step1[7], step1[6]); + step2[7] = vaddq_s32(step1[7], step1[6]); + + // stage 3 + step1[0] = vaddq_s32(step2[0], step2[3]); + step1[1] = vaddq_s32(step2[1], step2[2]); + step1[2] = vsubq_s32(step2[1], step2[2]); + step1[3] = vsubq_s32(step2[0], step2[3]); + + t64[2] = vmull_lane_s32(vget_low_s32(step2[6]), vget_high_s32(cospis0), 0); + t64[3] = vmull_lane_s32(vget_high_s32(step2[6]), vget_high_s32(cospis0), 0); + t64[0] = + vmlsl_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0); + t64[1] = vmlsl_lane_s32(t64[3], vget_high_s32(step2[5]), + vget_high_s32(cospis0), 0); + t64[2] = + vmlal_lane_s32(t64[2], vget_low_s32(step2[5]), vget_high_s32(cospis0), 0); + t64[3] = vmlal_lane_s32(t64[3], vget_high_s32(step2[5]), + vget_high_s32(cospis0), 0); + t32[0] = vrshrn_n_s64(t64[0], DCT_CONST_BITS); + t32[1] = vrshrn_n_s64(t64[1], DCT_CONST_BITS); + t32[2] = vrshrn_n_s64(t64[2], DCT_CONST_BITS); + t32[3] = vrshrn_n_s64(t64[3], DCT_CONST_BITS); + step1[5] = vcombine_s32(t32[0], t32[1]); + step1[6] = vcombine_s32(t32[2], t32[3]); + + // stage 4 + *io0 = vaddq_s32(step1[0], step2[7]); + *io1 = vaddq_s32(step1[1], step1[6]); + *io2 = vaddq_s32(step1[2], step1[5]); + *io3 = vaddq_s32(step1[3], step2[4]); + *io4 = vsubq_s32(step1[3], step2[4]); + *io5 = vsubq_s32(step1[2], step1[5]); + *io6 = vsubq_s32(step1[1], step1[6]); + *io7 = vsubq_s32(step1[0], step2[7]); +} + #endif // VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_ diff --git a/vpx_dsp/arm/mem_neon.h b/vpx_dsp/arm/mem_neon.h index ea0962954..2027a1d8d 100644 --- a/vpx_dsp/arm/mem_neon.h +++ b/vpx_dsp/arm/mem_neon.h @@ -26,6 +26,15 @@ static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1, ((int64_t)(uint16_t)c3 << 48)); } +static INLINE int32x2_t create_s32x2_neon(const int32_t c0, const int32_t c1) { + return vcreate_s32((uint32_t)c0 | ((int64_t)(uint32_t)c1 << 32)); +} + +static INLINE int32x4_t create_s32x4_neon(const int32_t c0, const int32_t c1, + const int32_t c2, const int32_t c3) { + return vcombine_s32(create_s32x2_neon(c0, c1), create_s32x2_neon(c2, c3)); +} + // Helper functions used to load tran_low_t into int16, narrowing if necessary. static INLINE int16x8x2_t load_tran_low_to_s16x2q(const tran_low_t *buf) { #if CONFIG_VP9_HIGHBITDEPTH