generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use memcpy for unaligned neon stores

Browse Source 
Advise the compiler that the store is eventually going to a uint8_t
buffer. This helps avoid getting alignment hints which would cause the
memory access to fail.

Originally added as a workaround for clang:
https://bugs.llvm.org//show_bug.cgi?id=24421

Change-Id: Ie9854b777cfb2f4baaee66764f0e51dcb094d51e
This commit is contained in:
Johann 2017-05-12 18:14:27 -07:00
parent 7498fe2e54
commit 2057d3ef75
3 changed files with 30 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
vp8/common/arm/neon/bilinearpredict_neon.c
View File

@ -11,6 +11,7 @@
#include <arm_neon.h>
#include <string.h>
#include "./vpx_config.h"
#include "vpx_dsp/arm/mem_neon.h"
static const uint8_t bifilter4_coeff[8][2] = { { 128, 0 }, { 112, 16 },
{ 96, 32 }, { 80, 48 },
@ -21,35 +22,6 @@ static INLINE uint8x8_t load_and_shift(const unsigned char *a) {
return vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(vld1_u8(a)), 32));
}
static INLINE void store4x4(unsigned char *dst, int dst_stride,
const uint8x8_t a0, const uint8x8_t a1) {
if (!((uintptr_t)dst & 0x3) && !(dst_stride & 0x3)) {
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a0), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a0), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a1), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a1), 1);
} else {
// Store to the aligned local buffer and memcpy instead of vget_lane_u8
// which is really really slow.
uint32_t output_buffer[4];
vst1_lane_u32(output_buffer, vreinterpret_u32_u8(a0), 0);
vst1_lane_u32(output_buffer + 1, vreinterpret_u32_u8(a0), 1);
vst1_lane_u32(output_buffer + 2, vreinterpret_u32_u8(a1), 0);
vst1_lane_u32(output_buffer + 3, vreinterpret_u32_u8(a1), 1);
memcpy(dst, output_buffer, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 1, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 2, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 3, 4);
}
}
void vp8_bilinear_predict4x4_neon(unsigned char *src_ptr,
int src_pixels_per_line, int xoffset,
int yoffset, unsigned char *dst_ptr,
@ -122,7 +94,7 @@ void vp8_bilinear_predict4x4_neon(unsigned char *src_ptr,
// secondpass_filter
if (yoffset == 0) { // skip_2ndpass_filter
store4x4(dst_ptr, dst_pitch, e0, e1);
store_unaligned_u8q(dst_ptr, dst_pitch, vcombine_u8(e0, e1));
} else {
uint8x8_t f0, f1;
const uint8x8_t filter0 = vdup_n_u8(bifilter4_coeff[yoffset][0]);
@ -140,7 +112,7 @@ void vp8_bilinear_predict4x4_neon(unsigned char *src_ptr,
f0 = vqrshrn_n_u16(b0, 7);
f1 = vqrshrn_n_u16(b1, 7);
store4x4(dst_ptr, dst_pitch, f0, f1);
store_unaligned_u8q(dst_ptr, dst_pitch, vcombine_u8(f0, f1));
}
}

36
vp8/common/arm/neon/sixtappredict_neon.c
View File

@ -11,6 +11,7 @@
#include <arm_neon.h>
#include <string.h>
#include "./vpx_config.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_ports/mem.h"
static const int8_t vp8_sub_pel_filters[8][8] = {
@ -42,35 +43,6 @@ static INLINE uint8x8_t load_and_shift(const unsigned char *a) {
return vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(vld1_u8(a)), 32));
}
static INLINE void store4x4(unsigned char *dst, int dst_stride,
const uint8x8_t a0, const uint8x8_t a1) {
if (!((uintptr_t)dst & 0x3) && !(dst_stride & 0x3)) {
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a0), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a0), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a1), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(a1), 1);
} else {
// Store to the aligned local buffer and memcpy instead of vget_lane_u8
// which is really really slow.
uint32_t output_buffer[4];
vst1_lane_u32(output_buffer, vreinterpret_u32_u8(a0), 0);
vst1_lane_u32(output_buffer + 1, vreinterpret_u32_u8(a0), 1);
vst1_lane_u32(output_buffer + 2, vreinterpret_u32_u8(a1), 0);
vst1_lane_u32(output_buffer + 3, vreinterpret_u32_u8(a1), 1);
memcpy(dst, output_buffer, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 1, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 2, 4);
dst += dst_stride;
memcpy(dst, output_buffer + 3, 4);
}
}
static INLINE void filter_add_accumulate(const uint8x16_t a, const uint8x16_t b,
const uint8x8_t filter, uint16x8_t *c,
uint16x8_t *d) {
@ -180,7 +152,7 @@ static INLINE void yonly4x4(const unsigned char *src, int src_stride,
e0 = vqrshrun_n_s16(d0, 7);
e1 = vqrshrun_n_s16(d1, 7);
store4x4(dst, dst_stride, e0, e1);
store_unaligned_u8q(dst, dst_stride, vcombine_u8(e0, e1));
}
void vp8_sixtap_predict4x4_neon(unsigned char *src_ptr, int src_pixels_per_line,
@ -297,7 +269,7 @@ void vp8_sixtap_predict4x4_neon(unsigned char *src_ptr, int src_pixels_per_line,
b2 = vqrshrun_n_s16(e4567, 7);
if (yoffset == 0) { // firstpass_filter4x4_only
store4x4(dst_ptr, dst_pitch, b0, b2);
store_unaligned_u8q(dst_ptr, dst_pitch, vcombine_u8(b0, b2));
return;
}
@ -411,7 +383,7 @@ void vp8_sixtap_predict4x4_neon(unsigned char *src_ptr, int src_pixels_per_line,
e0 = vqrshrun_n_s16(d0, 7);
e1 = vqrshrun_n_s16(d1, 7);
store4x4(dst_ptr, dst_pitch, e0, e1);
store_unaligned_u8q(dst_ptr, dst_pitch, vcombine_u8(e0, e1));
}
void vp8_sixtap_predict8x4_neon(unsigned char *src_ptr, int src_pixels_per_line,

23
vpx_dsp/arm/mem_neon.h
View File

@ -69,6 +69,29 @@ static INLINE void store_s16q_to_tran_low(tran_low_t *buf, const int16x8_t a) {
#endif
}
// Propagate type information to the compiler. Without this the compiler may
// assume the required alignment of uint32_t (4 bytes) and add alignment hints
// to the memory access.
//
// This is used for functions operating on uint8_t which wish to load or store 4
// values at a time but which may not be on 4 byte boundaries.
static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) {
memcpy(buf, &a, 4);
}
// Store 4 sets of 4 bytes when alignment is not guaranteed.
static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
const uint8x16_t a) {
const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 0));
buf += stride;
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 1));
buf += stride;
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 2));
buf += stride;
uint32_to_mem(buf, vgetq_lane_u32(a_u32, 3));
}
// Load 2 sets of 4 bytes when alignment is guaranteed.
static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
uint32x2_t a = vdup_n_u32(0);