From 9d941166973b6b8800b1d8fb1cc0fc2b8083fa6d Mon Sep 17 00:00:00 2001 From: "cduvivier@google.com" Date: Thu, 30 Jun 2011 19:19:37 +0000 Subject: [PATCH] Optimization of 'rftbsub': * scalar optimization, vectorization. * 0.5% AEC overall speedup for the straight C path. * 2.8% AEC overall speedup for the SSE2 path. Review URL: http://webrtc-codereview.appspot.com/48008 git-svn-id: http://webrtc.googlecode.com/svn/trunk@137 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../aec/main/source/aec_rdft.c | 44 ++++----- .../aec/main/source/aec_rdft.h | 5 +- .../aec/main/source/aec_rdft_sse2.c | 93 +++++++++++++++++++ 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/modules/audio_processing/aec/main/source/aec_rdft.c b/modules/audio_processing/aec/main/source/aec_rdft.c index 3d6e0eb02..15f5fcf9d 100644 --- a/modules/audio_processing/aec/main/source/aec_rdft.c +++ b/modules/audio_processing/aec/main/source/aec_rdft.c @@ -449,36 +449,32 @@ static void rftfsub_128_C(float *a, float *c) { } } -static void rftbsub_128(float *a, int nc, float *c) { - const int n = 128; - int j, k, kk, ks, m; +static void rftbsub_128_C(float *a, float *c) { + int j1, j2, k1, k2; float wkr, wki, xr, xi, yr, yi; a[1] = -a[1]; - m = n >> 1; - ks = 2 * nc / m; - kk = 0; - for (j = 2; j < m; j += 2) { - k = n - j; - kk += ks; - wkr = 0.5f - c[nc - kk]; - wki = c[kk]; - xr = a[j] - a[k]; - xi = a[j + 1] + a[k + 1]; + for (j1 = 1, j2 = 2; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; yr = wkr * xr + wki * xi; yi = wkr * xi - wki * xr; - a[j] -= yr; - a[j + 1] = yi - a[j + 1]; - a[k] += yr; - a[k + 1] = yi - a[k + 1]; + a[j2 + 0] = a[j2 + 0] - yr; + a[j2 + 1] = yi - a[j2 + 1]; + a[k2 + 0] = yr + a[k2 + 0]; + a[k2 + 1] = yi - a[k2 + 1]; } - a[m + 1] = -a[m + 1]; + a[65] = -a[65]; } void aec_rdft_128(int isgn, float *a, int *ip, float *w) { const int n = 128; - int nw, nc; + int nw; float xi; nw = ip[0]; @@ -486,9 +482,7 @@ void aec_rdft_128(int isgn, float *a, int *ip, float *w) nw = n >> 2; makewt(ip, w); } - nc = ip[1]; - if (n > (nc << 2)) { - nc = n >> 2; + if (n > (ip[1] << 2)) { makect_32(ip, w + nw); } if (isgn >= 0) { @@ -501,17 +495,19 @@ void aec_rdft_128(int isgn, float *a, int *ip, float *w) } else { a[1] = 0.5f * (a[0] - a[1]); a[0] -= a[1]; - rftbsub_128(a, nc, w + nw); + rftbsub_128(a, w + nw); bitrv2_32or128(n, ip + 2, a); cftbsub_128(a, w); } } // code path selection -rftfsub_128_t rftfsub_128; +rft_sub_128_t rftfsub_128; +rft_sub_128_t rftbsub_128; void aec_rdft_init(void) { rftfsub_128 = rftfsub_128_C; + rftbsub_128 = rftbsub_128_C; if (WebRtc_GetCPUInfo(kSSE2)) { #if defined(__SSE2__) aec_rdft_init_sse2(); diff --git a/modules/audio_processing/aec/main/source/aec_rdft.h b/modules/audio_processing/aec/main/source/aec_rdft.h index f24003c74..add162c34 100644 --- a/modules/audio_processing/aec/main/source/aec_rdft.h +++ b/modules/audio_processing/aec/main/source/aec_rdft.h @@ -9,8 +9,9 @@ */ // code path selection function pointers -typedef void (*rftfsub_128_t)(float *a, float *c); -extern rftfsub_128_t rftfsub_128; +typedef void (*rft_sub_128_t)(float *a, float *c); +extern rft_sub_128_t rftfsub_128; +extern rft_sub_128_t rftbsub_128; // entry points void aec_rdft_init(void); diff --git a/modules/audio_processing/aec/main/source/aec_rdft_sse2.c b/modules/audio_processing/aec/main/source/aec_rdft_sse2.c index 362cc187a..df76aa3f2 100644 --- a/modules/audio_processing/aec/main/source/aec_rdft_sse2.c +++ b/modules/audio_processing/aec/main/source/aec_rdft_sse2.c @@ -109,6 +109,99 @@ static void rftfsub_128_SSE2(float *a, float *c) { } } +static void rftbsub_128_SSE2(float *a, float *c) { + int j1, j2, k1, k2; + float wkr, wki, xr, xi, yr, yi; + + static const ALIGN16_BEG float ALIGN16_END k_half[4] = + {0.5f, 0.5f, 0.5f, 0.5f}; + const __m128 mm_half = _mm_load_ps(k_half); + + a[1] = -a[1]; + // Vectorized code (four at once). + // Note: commented number are indexes for the first iteration of the loop. + for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) { + // Load 'wk'. + const __m128 c_j1 = _mm_loadu_ps(&c[ j1]); // 1, 2, 3, 4, + const __m128 c_k1 = _mm_loadu_ps(&c[29 - j1]); // 28, 29, 30, 31, + const __m128 wkrt = _mm_sub_ps(mm_half, c_k1); // 28, 29, 30, 31, + const __m128 wkr_ = + _mm_shuffle_ps(wkrt, wkrt, _MM_SHUFFLE(0, 1, 2, 3)); // 31, 30, 29, 28, + const __m128 wki_ = c_j1; // 1, 2, 3, 4, + // Load and shuffle 'a'. + const __m128 a_j2_0 = _mm_loadu_ps(&a[0 + j2]); // 2, 3, 4, 5, + const __m128 a_j2_4 = _mm_loadu_ps(&a[4 + j2]); // 6, 7, 8, 9, + const __m128 a_k2_0 = _mm_loadu_ps(&a[122 - j2]); // 120, 121, 122, 123, + const __m128 a_k2_4 = _mm_loadu_ps(&a[126 - j2]); // 124, 125, 126, 127, + const __m128 a_j2_p0 = _mm_shuffle_ps(a_j2_0, a_j2_4, + _MM_SHUFFLE(2, 0, 2 ,0)); // 2, 4, 6, 8, + const __m128 a_j2_p1 = _mm_shuffle_ps(a_j2_0, a_j2_4, + _MM_SHUFFLE(3, 1, 3 ,1)); // 3, 5, 7, 9, + const __m128 a_k2_p0 = _mm_shuffle_ps(a_k2_4, a_k2_0, + _MM_SHUFFLE(0, 2, 0 ,2)); // 126, 124, 122, 120, + const __m128 a_k2_p1 = _mm_shuffle_ps(a_k2_4, a_k2_0, + _MM_SHUFFLE(1, 3, 1 ,3)); // 127, 125, 123, 121, + // Calculate 'x'. + const __m128 xr_ = _mm_sub_ps(a_j2_p0, a_k2_p0); + // 2-126, 4-124, 6-122, 8-120, + const __m128 xi_ = _mm_add_ps(a_j2_p1, a_k2_p1); + // 3-127, 5-125, 7-123, 9-121, + // Calculate product into 'y'. + // yr = wkr * xr + wki * xi; + // yi = wkr * xi - wki * xr; + const __m128 a_ = _mm_mul_ps(wkr_, xr_); + const __m128 b_ = _mm_mul_ps(wki_, xi_); + const __m128 c_ = _mm_mul_ps(wkr_, xi_); + const __m128 d_ = _mm_mul_ps(wki_, xr_); + const __m128 yr_ = _mm_add_ps(a_, b_); // 2-126, 4-124, 6-122, 8-120, + const __m128 yi_ = _mm_sub_ps(c_, d_); // 3-127, 5-125, 7-123, 9-121, + // Update 'a'. + // a[j2 + 0] = a[j2 + 0] - yr; + // a[j2 + 1] = yi - a[j2 + 1]; + // a[k2 + 0] = yr + a[k2 + 0]; + // a[k2 + 1] = yi - a[k2 + 1]; + const __m128 a_j2_p0n = _mm_sub_ps(a_j2_p0, yr_); // 2, 4, 6, 8, + const __m128 a_j2_p1n = _mm_sub_ps(yi_, a_j2_p1); // 3, 5, 7, 9, + const __m128 a_k2_p0n = _mm_add_ps(a_k2_p0, yr_); // 126, 124, 122, 120, + const __m128 a_k2_p1n = _mm_sub_ps(yi_, a_k2_p1); // 127, 125, 123, 121, + // Shuffle in right order and store. + // Shuffle in right order and store. + const __m128 a_j2_0n = _mm_unpacklo_ps(a_j2_p0n, a_j2_p1n); + // 2, 3, 4, 5, + const __m128 a_j2_4n = _mm_unpackhi_ps(a_j2_p0n, a_j2_p1n); + // 6, 7, 8, 9, + const __m128 a_k2_0nt = _mm_unpackhi_ps(a_k2_p0n, a_k2_p1n); + // 122, 123, 120, 121, + const __m128 a_k2_4nt = _mm_unpacklo_ps(a_k2_p0n, a_k2_p1n); + // 126, 127, 124, 125, + const __m128 a_k2_0n = _mm_shuffle_ps(a_k2_0nt, a_k2_0nt, + _MM_SHUFFLE(1, 0, 3 ,2)); // 120, 121, 122, 123, + const __m128 a_k2_4n = _mm_shuffle_ps(a_k2_4nt, a_k2_4nt, + _MM_SHUFFLE(1, 0, 3 ,2)); // 124, 125, 126, 127, + _mm_storeu_ps(&a[0 + j2], a_j2_0n); + _mm_storeu_ps(&a[4 + j2], a_j2_4n); + _mm_storeu_ps(&a[122 - j2], a_k2_0n); + _mm_storeu_ps(&a[126 - j2], a_k2_4n); + } + // Scalar code for the remaining items. + for (; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; + yr = wkr * xr + wki * xi; + yi = wkr * xi - wki * xr; + a[j2 + 0] = a[j2 + 0] - yr; + a[j2 + 1] = yi - a[j2 + 1]; + a[k2 + 0] = yr + a[k2 + 0]; + a[k2 + 1] = yi - a[k2 + 1]; + } + a[65] = -a[65]; +} + void aec_rdft_init_sse2(void) { rftfsub_128 = rftfsub_128_SSE2; + rftbsub_128 = rftbsub_128_SSE2; }