generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

am 8ac8278b: Merge "Fix for slm-tuned memmove (both 32- and 64-bit)."

Browse Source 
* commit '8ac8278b0a673e30698ba4d81f495165b66acbd5':
  Fix for slm-tuned memmove (both 32- and 64-bit).
This commit is contained in:
Christopher Ferris 2014-06-05 18:29:19 +00:00 committed by Android Git Automerger
commit 675a41f56e
3 changed files with 236 additions and 460 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

339
libc/arch-x86/silvermont/string/sse2-memmove-slm.S
View File

@ -74,13 +74,13 @@ name: \
#endif #endif
#ifdef USE_AS_BCOPY #ifdef USE_AS_BCOPY
# define SRC PARMS # define SRC PARMS
# define DEST SRC+4 # define DEST SRC+4
# define LEN DEST+4 # define LEN DEST+4
#else #else
# define DEST PARMS # define DEST PARMS
# define SRC DEST+4 # define SRC DEST+4
# define LEN SRC+4 # define LEN SRC+4
#endif #endif
#define CFI_PUSH(REG) \ #define CFI_PUSH(REG) \
@ -109,15 +109,15 @@ ENTRY (MEMMOVE)
/* Check whether we should copy backward or forward. */ /* Check whether we should copy backward or forward. */
cmp %eax, %edx cmp %eax, %edx
je L(mm_return) je L(mm_return)
ja L(mm_len_0_or_more_backward) jg L(mm_len_0_or_more_backward)
/* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128] /* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128]
separately. */ separately. */
cmp $16, %ecx cmp $16, %ecx
jbe L(mm_len_0_16_bytes_forward) jbe L(mm_len_0_16_bytes_forward)
cmpl $32, %ecx cmpl $32, %ecx
jg L(mm_len_32_or_more_forward) ja L(mm_len_32_or_more_forward)
/* Copy [0..32] and return. */ /* Copy [0..32] and return. */
movdqu (%eax), %xmm0 movdqu (%eax), %xmm0
@ -127,8 +127,8 @@ ENTRY (MEMMOVE)
jmp L(mm_return) jmp L(mm_return)
L(mm_len_32_or_more_forward): L(mm_len_32_or_more_forward):
cmpl $64, %ecx cmpl $64, %ecx
jg L(mm_len_64_or_more_forward) ja L(mm_len_64_or_more_forward)
/* Copy [0..64] and return. */ /* Copy [0..64] and return. */
movdqu (%eax), %xmm0 movdqu (%eax), %xmm0
@ -142,8 +142,8 @@ L(mm_len_32_or_more_forward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_64_or_more_forward): L(mm_len_64_or_more_forward):
cmpl $128, %ecx cmpl $128, %ecx
jg L(mm_len_128_or_more_forward) ja L(mm_len_128_or_more_forward)
/* Copy [0..128] and return. */ /* Copy [0..128] and return. */
movdqu (%eax), %xmm0 movdqu (%eax), %xmm0
@ -165,72 +165,66 @@ L(mm_len_64_or_more_forward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_128_or_more_forward): L(mm_len_128_or_more_forward):
cmp $SHARED_CACHE_SIZE_HALF, %ecx
jae L(mm_large_page_forward)
PUSH (%esi) PUSH (%esi)
PUSH (%edi) PUSH (%edi)
movl %eax, %esi
movl %edx, %edi
/* Aligning the address of destination. */ /* Aligning the address of destination. */
movdqu (%esi), %xmm0 movdqu (%eax), %xmm0
movdqu 16(%esi), %xmm1 movdqu 16(%eax), %xmm1
movdqu 32(%esi), %xmm2 movdqu 32(%eax), %xmm2
movdqu 48(%esi), %xmm3 movdqu 48(%eax), %xmm3
leal 64(%edi), %edx leal 64(%edx), %edi
andl $-64, %edx andl $-64, %edi
subl %edx, %eax
movl %esi, %eax movdqu (%eax, %edi), %xmm4
subl %edi, %eax movdqu 16(%eax, %edi), %xmm5
movdqu 32(%eax, %edi), %xmm6
movdqu 48(%eax, %edi), %xmm7
movdqu (%edx, %eax), %xmm4 movdqu %xmm0, (%edx)
movdqu 16(%edx, %eax), %xmm5 movdqu %xmm1, 16(%edx)
movdqu 32(%edx, %eax), %xmm6 movdqu %xmm2, 32(%edx)
movdqu 48(%edx, %eax), %xmm7 movdqu %xmm3, 48(%edx)
movdqa %xmm4, (%edi)
movaps %xmm5, 16(%edi)
movaps %xmm6, 32(%edi)
movaps %xmm7, 48(%edi)
addl $64, %edi
movdqu %xmm0, (%edi) leal (%edx, %ecx), %ebx
movdqu %xmm1, 16(%edi)
movdqu %xmm2, 32(%edi)
movdqu %xmm3, 48(%edi)
movdqa %xmm4, (%edx)
movdqa %xmm5, 16(%edx)
movdqa %xmm6, 32(%edx)
movdqa %xmm7, 48(%edx)
addl $64, %edx
leal (%edi, %ecx), %ebx
andl $-64, %ebx andl $-64, %ebx
cmp %edi, %ebx
cmp %edx, %ebx
jbe L(mm_copy_remaining_forward) jbe L(mm_copy_remaining_forward)
cmp $SHARED_CACHE_SIZE_HALF, %ecx
jae L(mm_large_page_loop_forward)
.p2align 4 .p2align 4
L(mm_main_loop_forward): L(mm_main_loop_forward):
prefetcht0 128(%edx, %eax) prefetcht0 128(%eax, %edi)
movdqu (%edx, %eax), %xmm0 movdqu (%eax, %edi), %xmm0
movdqu 16(%edx, %eax), %xmm1 movdqu 16(%eax, %edi), %xmm1
movdqu 32(%edx, %eax), %xmm2 movdqu 32(%eax, %edi), %xmm2
movdqu 48(%edx, %eax), %xmm3 movdqu 48(%eax, %edi), %xmm3
movdqa %xmm0, (%edx) movdqa %xmm0, (%edi)
movdqa %xmm1, 16(%edx) movaps %xmm1, 16(%edi)
movdqa %xmm2, 32(%edx) movaps %xmm2, 32(%edi)
movdqa %xmm3, 48(%edx) movaps %xmm3, 48(%edi)
leal 64(%edx), %edx leal 64(%edi), %edi
cmp %edx, %ebx cmp %edi, %ebx
ja L(mm_main_loop_forward) ja L(mm_main_loop_forward)
L(mm_copy_remaining_forward): L(mm_copy_remaining_forward):
addl %edi, %ecx addl %edx, %ecx
subl %edx, %ecx subl %edi, %ecx
/* We copied all up till %edx position in the dst. /* We copied all up till %edi position in the dst.
In %ecx now is how many bytes are left to copy. In %ecx now is how many bytes are left to copy.
Now we need to advance %esi. */ Now we need to advance %esi. */
leal (%edx, %eax), %esi leal (%edi, %eax), %esi
L(mm_remaining_0_64_bytes_forward): L(mm_remaining_0_64_bytes_forward):
cmp $32, %ecx cmp $32, %ecx
@ -251,8 +245,8 @@ L(mm_remaining_0_64_bytes_forward):
ja L(mm_remaining_3_4_bytes_forward) ja L(mm_remaining_3_4_bytes_forward)
movzbl -1(%esi,%ecx), %eax movzbl -1(%esi,%ecx), %eax
movzbl (%esi), %ebx movzbl (%esi), %ebx
movb %al, -1(%edx,%ecx) movb %al, -1(%edi,%ecx)
movb %bl, (%edx) movb %bl, (%edi)
jmp L(mm_return_pop_all) jmp L(mm_return_pop_all)
L(mm_remaining_33_64_bytes_forward): L(mm_remaining_33_64_bytes_forward):
@ -260,40 +254,39 @@ L(mm_remaining_33_64_bytes_forward):
movdqu 16(%esi), %xmm1 movdqu 16(%esi), %xmm1
movdqu -32(%esi, %ecx), %xmm2 movdqu -32(%esi, %ecx), %xmm2
movdqu -16(%esi, %ecx), %xmm3 movdqu -16(%esi, %ecx), %xmm3
movdqu %xmm0, (%edx) movdqu %xmm0, (%edi)
movdqu %xmm1, 16(%edx) movdqu %xmm1, 16(%edi)
movdqu %xmm2, -32(%edx, %ecx) movdqu %xmm2, -32(%edi, %ecx)
movdqu %xmm3, -16(%edx, %ecx) movdqu %xmm3, -16(%edi, %ecx)
jmp L(mm_return_pop_all) jmp L(mm_return_pop_all)
L(mm_remaining_17_32_bytes_forward): L(mm_remaining_17_32_bytes_forward):
movdqu (%esi), %xmm0 movdqu (%esi), %xmm0
movdqu -16(%esi, %ecx), %xmm1 movdqu -16(%esi, %ecx), %xmm1
movdqu %xmm0, (%edx) movdqu %xmm0, (%edi)
movdqu %xmm1, -16(%edx, %ecx) movdqu %xmm1, -16(%edi, %ecx)
jmp L(mm_return_pop_all)
L(mm_remaining_3_4_bytes_forward):
movzwl -2(%esi,%ecx), %eax
movzwl (%esi), %ebx
movw %ax, -2(%edx,%ecx)
movw %bx, (%edx)
jmp L(mm_return_pop_all)
L(mm_remaining_5_8_bytes_forward):
movl (%esi), %eax
movl -4(%esi,%ecx), %ebx
movl %eax, (%edx)
movl %ebx, -4(%edx,%ecx)
jmp L(mm_return_pop_all) jmp L(mm_return_pop_all)
L(mm_remaining_9_16_bytes_forward): L(mm_remaining_9_16_bytes_forward):
movq (%esi), %xmm0 movq (%esi), %xmm0
movq -8(%esi, %ecx), %xmm1 movq -8(%esi, %ecx), %xmm1
movq %xmm0, (%edx) movq %xmm0, (%edi)
movq %xmm1, -8(%edx, %ecx) movq %xmm1, -8(%edi, %ecx)
jmp L(mm_return_pop_all) jmp L(mm_return_pop_all)
L(mm_remaining_5_8_bytes_forward):
movl (%esi), %eax
movl -4(%esi,%ecx), %ebx
movl %eax, (%edi)
movl %ebx, -4(%edi,%ecx)
jmp L(mm_return_pop_all)
L(mm_remaining_3_4_bytes_forward):
movzwl -2(%esi,%ecx), %eax
movzwl (%esi), %ebx
movw %ax, -2(%edi,%ecx)
movw %bx, (%edi)
jmp L(mm_return_pop_all)
L(mm_len_0_16_bytes_forward): L(mm_len_0_16_bytes_forward):
testb $24, %cl testb $24, %cl
@ -334,15 +327,20 @@ L(mm_len_9_16_bytes_forward):
movq %xmm1, -8(%edx, %ecx) movq %xmm1, -8(%edx, %ecx)
jmp L(mm_return) jmp L(mm_return)
L(mm_recalc_len):
/* Compute in %ecx how many bytes are left to copy after
the main loop stops. */
movl %ebx, %ecx
subl %edx, %ecx
/* The code for copying backwards. */ /* The code for copying backwards. */
L(mm_len_0_or_more_backward): L(mm_len_0_or_more_backward):
/* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128] /* Now do checks for lengths. We do [0..16], [16..32], [32..64], [64..128]
separately. */ separately. */
cmp $16, %ecx cmp $16, %ecx
jbe L(mm_len_0_16_bytes_backward) jbe L(mm_len_0_16_bytes_backward)
cmpl $32, %ecx cmpl $32, %ecx
jg L(mm_len_32_or_more_backward) jg L(mm_len_32_or_more_backward)
/* Copy [0..32] and return. */ /* Copy [0..32] and return. */
@ -353,7 +351,7 @@ L(mm_len_0_or_more_backward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_32_or_more_backward): L(mm_len_32_or_more_backward):
cmpl $64, %ecx cmpl $64, %ecx
jg L(mm_len_64_or_more_backward) jg L(mm_len_64_or_more_backward)
/* Copy [0..64] and return. */ /* Copy [0..64] and return. */
@ -368,7 +366,7 @@ L(mm_len_32_or_more_backward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_64_or_more_backward): L(mm_len_64_or_more_backward):
cmpl $128, %ecx cmpl $128, %ecx
jg L(mm_len_128_or_more_backward) jg L(mm_len_128_or_more_backward)
/* Copy [0..128] and return. */ /* Copy [0..128] and return. */
@ -391,10 +389,6 @@ L(mm_len_64_or_more_backward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_128_or_more_backward): L(mm_len_128_or_more_backward):
cmp $SHARED_CACHE_SIZE_HALF, %ecx
jae L(mm_large_page_backward)
PUSH (%esi) PUSH (%esi)
PUSH (%edi) PUSH (%edi)
@ -429,17 +423,11 @@ L(mm_len_128_or_more_backward):
leal 64(%edx), %ebx leal 64(%edx), %ebx
andl $-64, %ebx andl $-64, %ebx
/* Compute in %ecx how many bytes are left to copy after
the main loop stops. */
movl %ebx, %ecx
subl %edx, %ecx
cmp %edi, %ebx cmp %edi, %ebx
jb L(mm_main_loop_backward) jae L(mm_main_loop_backward_end)
POP (%edi) cmp $SHARED_CACHE_SIZE_HALF, %ecx
POP (%esi) jae L(mm_large_page_loop_backward)
jmp L(mm_len_0_or_more_backward)
.p2align 4 .p2align 4
L(mm_main_loop_backward): L(mm_main_loop_backward):
@ -457,9 +445,10 @@ L(mm_main_loop_backward):
leal -64(%edi), %edi leal -64(%edi), %edi
cmp %edi, %ebx cmp %edi, %ebx
jb L(mm_main_loop_backward) jb L(mm_main_loop_backward)
L(mm_main_loop_backward_end):
POP (%edi) POP (%edi)
POP (%esi) POP (%esi)
jmp L(mm_len_0_or_more_backward) jmp L(mm_recalc_len)
/* Copy [0..16] and return. */ /* Copy [0..16] and return. */
L(mm_len_0_16_bytes_backward): L(mm_len_0_16_bytes_backward):
@ -508,151 +497,30 @@ L(mm_return):
RETURN RETURN
L(mm_return_pop_all): L(mm_return_pop_all):
movl %edi, %eax movl %edx, %eax
POP (%edi) POP (%edi)
POP (%esi) POP (%esi)
RETURN RETURN
/* Big length copy forward part. */ /* Big length copy forward part. */
L(mm_large_page_forward):
/* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */
PUSH (%esi)
PUSH (%edi)
movl %eax, %esi
movl %edx, %edi
movdqu (%esi), %xmm0
movdqu 16(%esi), %xmm1
movdqu 32(%esi), %xmm2
movdqu 48(%esi), %xmm3
leal 64(%edi), %edx
andl $-64, %edx
movl %esi, %eax
subl %edi, %eax
movdqu (%edx, %eax), %xmm4
movdqu 16(%edx, %eax), %xmm5
movdqu 32(%edx, %eax), %xmm6
movdqu 48(%edx, %eax), %xmm7
movdqu %xmm0, (%edi)
movdqu %xmm1, 16(%edi)
movdqu %xmm2, 32(%edi)
movdqu %xmm3, 48(%edi)
movntdq %xmm4, (%edx)
movntdq %xmm5, 16(%edx)
movntdq %xmm6, 32(%edx)
movntdq %xmm7, 48(%edx)
addl $64, %edx
leal (%edi, %ecx), %ebx
andl $-128, %ebx
cmp %edx, %ebx
jbe L(mm_copy_remaining_forward)
.p2align 4 .p2align 4
L(mm_large_page_loop_forward): L(mm_large_page_loop_forward):
movdqu (%edx, %eax), %xmm0 movdqu (%eax, %edi), %xmm0
movdqu 16(%edx, %eax), %xmm1 movdqu 16(%eax, %edi), %xmm1
movdqu 32(%edx, %eax), %xmm2 movdqu 32(%eax, %edi), %xmm2
movdqu 48(%edx, %eax), %xmm3 movdqu 48(%eax, %edi), %xmm3
movdqu 64(%edx, %eax), %xmm4 movntdq %xmm0, (%edi)
movdqu 80(%edx, %eax), %xmm5 movntdq %xmm1, 16(%edi)
movdqu 96(%edx, %eax), %xmm6 movntdq %xmm2, 32(%edi)
movdqu 112(%edx, %eax), %xmm7 movntdq %xmm3, 48(%edi)
movntdq %xmm0, (%edx) leal 64(%edi), %edi
movntdq %xmm1, 16(%edx) cmp %edi, %ebx
movntdq %xmm2, 32(%edx)
movntdq %xmm3, 48(%edx)
movntdq %xmm4, 64(%edx)
movntdq %xmm5, 80(%edx)
movntdq %xmm6, 96(%edx)
movntdq %xmm7, 112(%edx)
leal 128(%edx), %edx
cmp %edx, %ebx
ja L(mm_large_page_loop_forward) ja L(mm_large_page_loop_forward)
sfence sfence
jmp L(mm_copy_remaining_forward)
addl %edi, %ecx
subl %edx, %ecx
/* We copied all up till %edx position in the dst.
In %ecx now is how many bytes are left to copy.
Now we need to advance %esi. */
leal (%edx, %eax), %esi
cmp $64, %ecx
jb L(mm_remaining_0_64_bytes_forward)
movdqu (%esi), %xmm0
movdqu 16(%esi), %xmm1
movdqu 32(%esi), %xmm2
movdqu 48(%esi), %xmm3
movdqu -64(%esi, %ecx), %xmm4
movdqu -48(%esi, %ecx), %xmm5
movdqu -32(%esi, %ecx), %xmm6
movdqu -16(%esi, %ecx), %xmm7
movdqu %xmm0, (%edx)
movdqu %xmm1, 16(%edx)
movdqu %xmm2, 32(%edx)
movdqu %xmm3, 48(%edx)
movdqu %xmm4, -64(%edx, %ecx)
movdqu %xmm5, -48(%edx, %ecx)
movdqu %xmm6, -32(%edx, %ecx)
movdqu %xmm7, -16(%edx, %ecx)
jmp L(mm_return_pop_all)
/* Big length copy backward part. */ /* Big length copy backward part. */
L(mm_large_page_backward):
/* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */
PUSH (%esi)
PUSH (%edi)
movdqu -16(%eax, %ecx), %xmm0
movdqu -32(%eax, %ecx), %xmm1
movdqu -48(%eax, %ecx), %xmm2
movdqu -64(%eax, %ecx), %xmm3
leal (%edx, %ecx), %edi
andl $-64, %edi
movl %eax, %esi
subl %edx, %esi
movdqu -16(%edi, %esi), %xmm4
movdqu -32(%edi, %esi), %xmm5
movdqu -48(%edi, %esi), %xmm6
movdqu -64(%edi, %esi), %xmm7
movdqu %xmm0, -16(%edx, %ecx)
movdqu %xmm1, -32(%edx, %ecx)
movdqu %xmm2, -48(%edx, %ecx)
movdqu %xmm3, -64(%edx, %ecx)
movntdq %xmm4, -16(%edi)
movntdq %xmm5, -32(%edi)
movntdq %xmm6, -48(%edi)
movntdq %xmm7, -64(%edi)
leal -64(%edi), %edi
leal 128(%edx), %ebx
andl $-64, %ebx
/* Compute in %ecx how many bytes are left to copy after
the main loop stops. */
movl %ebx, %ecx
subl %edx, %ecx
cmp %edi, %ebx
jae L(mm_len_0_or_more_backward)
.p2align 4 .p2align 4
L(mm_large_page_loop_backward): L(mm_large_page_loop_backward):
movdqu -64(%edi, %esi), %xmm0 movdqu -64(%edi, %esi), %xmm0
@ -666,8 +534,9 @@ L(mm_large_page_loop_backward):
leal -64(%edi), %edi leal -64(%edi), %edi
cmp %edi, %ebx cmp %edi, %ebx
jb L(mm_large_page_loop_backward) jb L(mm_large_page_loop_backward)
sfence
POP (%edi) POP (%edi)
POP (%esi) POP (%esi)
jmp L(mm_len_0_or_more_backward) jmp L(mm_recalc_len)
END (MEMMOVE) END (MEMMOVE)

328
libc/arch-x86_64/string/sse2-memmove-slm.S
View File

@ -99,7 +99,7 @@ ENTRY (MEMMOVE)
/* Check whether we should copy backward or forward. */ /* Check whether we should copy backward or forward. */
cmp %rsi, %rdi cmp %rsi, %rdi
je L(mm_return) je L(mm_return)
ja L(mm_len_0_or_more_backward) jg L(mm_len_0_or_more_backward)
/* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128] /* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128]
separately. */ separately. */
@ -107,7 +107,7 @@ ENTRY (MEMMOVE)
jbe L(mm_len_0_16_bytes_forward) jbe L(mm_len_0_16_bytes_forward)
cmp $32, %rdx cmp $32, %rdx
jg L(mm_len_32_or_more_forward) ja L(mm_len_32_or_more_forward)
/* Copy [0..32] and return. */ /* Copy [0..32] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -118,7 +118,7 @@ ENTRY (MEMMOVE)
L(mm_len_32_or_more_forward): L(mm_len_32_or_more_forward):
cmp $64, %rdx cmp $64, %rdx
jg L(mm_len_64_or_more_forward) ja L(mm_len_64_or_more_forward)
/* Copy [0..64] and return. */ /* Copy [0..64] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -133,7 +133,7 @@ L(mm_len_32_or_more_forward):
L(mm_len_64_or_more_forward): L(mm_len_64_or_more_forward):
cmp $128, %rdx cmp $128, %rdx
jg L(mm_len_128_or_more_forward) ja L(mm_len_128_or_more_forward)
/* Copy [0..128] and return. */ /* Copy [0..128] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -155,13 +155,6 @@ L(mm_len_64_or_more_forward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_128_or_more_forward): L(mm_len_128_or_more_forward):
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_forward)
mov %rsi, %r8 // copy src to r8
mov %rdi, %r9 // copy dst to r9
/* Aligning the address of destination. */ /* Aligning the address of destination. */
/* save first unaligned 64 bytes */ /* save first unaligned 64 bytes */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -169,56 +162,57 @@ L(mm_len_128_or_more_forward):
movdqu 32(%rsi), %xmm2 movdqu 32(%rsi), %xmm2
movdqu 48(%rsi), %xmm3 movdqu 48(%rsi), %xmm3
lea 64(%r9), %rdi lea 64(%rdi), %r8
and $-64, %rdi /* rdi now aligned to next 64 byte boundary */ and $-64, %r8 /* r8 now aligned to next 64 byte boundary */
sub %rdi, %rsi /* rsi = src - dst = diff */
sub %r9, %rsi /* rsi = src - dst = diff */ movdqu (%r8, %rsi), %xmm4
movdqu 16(%r8, %rsi), %xmm5
movdqu 32(%r8, %rsi), %xmm6
movdqu 48(%r8, %rsi), %xmm7
movdqu (%rdi, %rsi), %xmm4 movdqu %xmm0, (%rdi)
movdqu 16(%rdi, %rsi), %xmm5 movdqu %xmm1, 16(%rdi)
movdqu 32(%rdi, %rsi), %xmm6 movdqu %xmm2, 32(%rdi)
movdqu 48(%rdi, %rsi), %xmm7 movdqu %xmm3, 48(%rdi)
movdqa %xmm4, (%r8)
movaps %xmm5, 16(%r8)
movaps %xmm6, 32(%r8)
movaps %xmm7, 48(%r8)
add $64, %r8
movdqu %xmm0, (%r9) lea (%rdi, %rdx), %rbx
movdqu %xmm1, 16(%r9)
movdqu %xmm2, 32(%r9)
movdqu %xmm3, 48(%r9)
movdqa %xmm4, (%rdi)
movdqa %xmm5, 16(%rdi)
movdqa %xmm6, 32(%rdi)
movdqa %xmm7, 48(%rdi)
add $64, %rdi
lea (%r9, %rdx), %rbx
and $-64, %rbx and $-64, %rbx
cmp %r8, %rbx
cmp %rdi, %rbx
jbe L(mm_copy_remaining_forward) jbe L(mm_copy_remaining_forward)
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_loop_forward)
.p2align 4 .p2align 4
L(mm_main_loop_forward): L(mm_main_loop_forward):
prefetcht0 128(%rdi, %rsi) prefetcht0 128(%r8, %rsi)
movdqu (%rdi, %rsi), %xmm0 movdqu (%r8, %rsi), %xmm0
movdqu 16(%rdi, %rsi), %xmm1 movdqu 16(%r8, %rsi), %xmm1
movdqu 32(%rdi, %rsi), %xmm2 movdqu 32(%r8, %rsi), %xmm2
movdqu 48(%rdi, %rsi), %xmm3 movdqu 48(%r8, %rsi), %xmm3
movdqa %xmm0, (%rdi) movdqa %xmm0, (%r8)
movdqa %xmm1, 16(%rdi) movaps %xmm1, 16(%r8)
movdqa %xmm2, 32(%rdi) movaps %xmm2, 32(%r8)
movdqa %xmm3, 48(%rdi) movaps %xmm3, 48(%r8)
lea 64(%rdi), %rdi lea 64(%r8), %r8
cmp %rdi, %rbx cmp %r8, %rbx
ja L(mm_main_loop_forward) ja L(mm_main_loop_forward)
L(mm_copy_remaining_forward): L(mm_copy_remaining_forward):
add %r9, %rdx add %rdi, %rdx
sub %rdi, %rdx sub %r8, %rdx
/* We copied all up till %rdi position in the dst. /* We copied all up till %rdi position in the dst.
In %rdx now is how many bytes are left to copy. In %rdx now is how many bytes are left to copy.
Now we need to advance %r8. */ Now we need to advance %r8. */
lea (%rdi, %rsi), %r8 lea (%r8, %rsi), %r9
L(mm_remaining_0_64_bytes_forward): L(mm_remaining_0_64_bytes_forward):
cmp $32, %rdx cmp $32, %rdx
@ -237,49 +231,49 @@ L(mm_remaining_0_64_bytes_forward):
cmpb $2, %dl cmpb $2, %dl
.p2align 4,,1 .p2align 4,,1
ja L(mm_remaining_3_4_bytes_forward) ja L(mm_remaining_3_4_bytes_forward)
movzbl -1(%r8,%rdx), %esi movzbl -1(%r9,%rdx), %esi
movzbl (%r8), %ebx movzbl (%r9), %ebx
movb %sil, -1(%rdi,%rdx) movb %sil, -1(%r8,%rdx)
movb %bl, (%rdi) movb %bl, (%r8)
jmp L(mm_return) jmp L(mm_return)
L(mm_remaining_33_64_bytes_forward): L(mm_remaining_33_64_bytes_forward):
movdqu (%r8), %xmm0 movdqu (%r9), %xmm0
movdqu 16(%r8), %xmm1 movdqu 16(%r9), %xmm1
movdqu -32(%r8, %rdx), %xmm2 movdqu -32(%r9, %rdx), %xmm2
movdqu -16(%r8, %rdx), %xmm3 movdqu -16(%r9, %rdx), %xmm3
movdqu %xmm0, (%rdi) movdqu %xmm0, (%r8)
movdqu %xmm1, 16(%rdi) movdqu %xmm1, 16(%r8)
movdqu %xmm2, -32(%rdi, %rdx) movdqu %xmm2, -32(%r8, %rdx)
movdqu %xmm3, -16(%rdi, %rdx) movdqu %xmm3, -16(%r8, %rdx)
jmp L(mm_return) jmp L(mm_return)
L(mm_remaining_17_32_bytes_forward): L(mm_remaining_17_32_bytes_forward):
movdqu (%r8), %xmm0 movdqu (%r9), %xmm0
movdqu -16(%r8, %rdx), %xmm1 movdqu -16(%r9, %rdx), %xmm1
movdqu %xmm0, (%rdi) movdqu %xmm0, (%r8)
movdqu %xmm1, -16(%rdi, %rdx) movdqu %xmm1, -16(%r8, %rdx)
jmp L(mm_return)
L(mm_remaining_3_4_bytes_forward):
movzwl -2(%r8,%rdx), %esi
movzwl (%r8), %ebx
movw %si, -2(%rdi,%rdx)
movw %bx, (%rdi)
jmp L(mm_return) jmp L(mm_return)
L(mm_remaining_5_8_bytes_forward): L(mm_remaining_5_8_bytes_forward):
movl (%r8), %esi movl (%r9), %esi
movl -4(%r8,%rdx), %ebx movl -4(%r9,%rdx), %ebx
movl %esi, (%rdi) movl %esi, (%r8)
movl %ebx, -4(%rdi,%rdx) movl %ebx, -4(%r8,%rdx)
jmp L(mm_return) jmp L(mm_return)
L(mm_remaining_9_16_bytes_forward): L(mm_remaining_9_16_bytes_forward):
mov (%r8), %rsi mov (%r9), %rsi
mov -8(%r8, %rdx), %rbx mov -8(%r9, %rdx), %rbx
mov %rsi, (%rdi) mov %rsi, (%r8)
mov %rbx, -8(%rdi, %rdx) mov %rbx, -8(%r8, %rdx)
jmp L(mm_return)
L(mm_remaining_3_4_bytes_forward):
movzwl -2(%r9,%rdx), %esi
movzwl (%r9), %ebx
movw %si, -2(%r8,%rdx)
movw %bx, (%r8)
jmp L(mm_return) jmp L(mm_return)
L(mm_len_0_16_bytes_forward): L(mm_len_0_16_bytes_forward):
@ -321,16 +315,21 @@ L(mm_len_9_16_bytes_forward):
mov %rsi, -8(%rdi, %rdx) mov %rsi, -8(%rdi, %rdx)
jmp L(mm_return) jmp L(mm_return)
L(mm_recalc_len):
/* Compute in %rdx how many bytes are left to copy after
the main loop stops. */
mov %rbx, %rdx
sub %rdi, %rdx
/* The code for copying backwards. */ /* The code for copying backwards. */
L(mm_len_0_or_more_backward): L(mm_len_0_or_more_backward):
/* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128] /* Now do checks for lengths. We do [0..16], [16..32], [32..64], [64..128]
separately. */ separately. */
cmp $16, %rdx cmp $16, %rdx
jbe L(mm_len_0_16_bytes_backward) jbe L(mm_len_0_16_bytes_backward)
cmp $32, %rdx cmp $32, %rdx
jg L(mm_len_32_or_more_backward) ja L(mm_len_32_or_more_backward)
/* Copy [0..32] and return. */ /* Copy [0..32] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -341,7 +340,7 @@ L(mm_len_0_or_more_backward):
L(mm_len_32_or_more_backward): L(mm_len_32_or_more_backward):
cmp $64, %rdx cmp $64, %rdx
jg L(mm_len_64_or_more_backward) ja L(mm_len_64_or_more_backward)
/* Copy [0..64] and return. */ /* Copy [0..64] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -356,7 +355,7 @@ L(mm_len_32_or_more_backward):
L(mm_len_64_or_more_backward): L(mm_len_64_or_more_backward):
cmp $128, %rdx cmp $128, %rdx
jg L(mm_len_128_or_more_backward) ja L(mm_len_128_or_more_backward)
/* Copy [0..128] and return. */ /* Copy [0..128] and return. */
movdqu (%rsi), %xmm0 movdqu (%rsi), %xmm0
@ -378,10 +377,6 @@ L(mm_len_64_or_more_backward):
jmp L(mm_return) jmp L(mm_return)
L(mm_len_128_or_more_backward): L(mm_len_128_or_more_backward):
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_backward)
/* Aligning the address of destination. We need to save /* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */ 16 bits from the source in order not to overwrite them. */
movdqu -16(%rsi, %rdx), %xmm0 movdqu -16(%rsi, %rdx), %xmm0
@ -405,22 +400,19 @@ L(mm_len_128_or_more_backward):
movdqu %xmm2, -48(%rdi, %rdx) movdqu %xmm2, -48(%rdi, %rdx)
movdqu %xmm3, -64(%rdi, %rdx) movdqu %xmm3, -64(%rdi, %rdx)
movdqa %xmm4, -16(%r9) movdqa %xmm4, -16(%r9)
movdqa %xmm5, -32(%r9) movaps %xmm5, -32(%r9)
movdqa %xmm6, -48(%r9) movaps %xmm6, -48(%r9)
movdqa %xmm7, -64(%r9) movaps %xmm7, -64(%r9)
lea -64(%r9), %r9 lea -64(%r9), %r9
lea 64(%rdi), %rbx lea 64(%rdi), %rbx
and $-64, %rbx and $-64, %rbx
/* Compute in %rdx how many bytes are left to copy after
the main loop stops. */
mov %rbx, %rdx
sub %rdi, %rdx
cmp %r9, %rbx cmp %r9, %rbx
jb L(mm_main_loop_backward) jae L(mm_recalc_len)
jmp L(mm_len_0_or_more_backward)
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_loop_backward)
.p2align 4 .p2align 4
L(mm_main_loop_backward): L(mm_main_loop_backward):
@ -432,13 +424,13 @@ L(mm_main_loop_backward):
movdqu -32(%r9, %r8), %xmm2 movdqu -32(%r9, %r8), %xmm2
movdqu -16(%r9, %r8), %xmm3 movdqu -16(%r9, %r8), %xmm3
movdqa %xmm0, -64(%r9) movdqa %xmm0, -64(%r9)
movdqa %xmm1, -48(%r9) movaps %xmm1, -48(%r9)
movdqa %xmm2, -32(%r9) movaps %xmm2, -32(%r9)
movdqa %xmm3, -16(%r9) movaps %xmm3, -16(%r9)
lea -64(%r9), %r9 lea -64(%r9), %r9
cmp %r9, %rbx cmp %r9, %rbx
jb L(mm_main_loop_backward) jb L(mm_main_loop_backward)
jmp L(mm_len_0_or_more_backward) jmp L(mm_recalc_len)
/* Copy [0..16] and return. */ /* Copy [0..16] and return. */
L(mm_len_0_16_bytes_backward): L(mm_len_0_16_bytes_backward):
@ -485,138 +477,23 @@ L(mm_return):
/* Big length copy forward part. */ /* Big length copy forward part. */
L(mm_large_page_forward):
/* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */
mov %rsi, %r8
mov %rdi, %r9
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu 32(%rsi), %xmm2
movdqu 48(%rsi), %xmm3
lea 64(%r9), %rdi
and $-64, %rdi /* rdi = aligned dst */
sub %r9, %rsi /* rsi = diff */
movdqu (%rdi, %rsi), %xmm4
movdqu 16(%rdi, %rsi), %xmm5
movdqu 32(%rdi, %rsi), %xmm6
movdqu 48(%rdi, %rsi), %xmm7
movdqu %xmm0, (%r9)
movdqu %xmm1, 16(%r9)
movdqu %xmm2, 32(%r9)
movdqu %xmm3, 48(%r9)
movntdq %xmm4, (%rdi)
movntdq %xmm5, 16(%rdi)
movntdq %xmm6, 32(%rdi)
movntdq %xmm7, 48(%rdi)
add $64, %rdi
lea (%r9, %rdx), %rbx
and $-128, %rbx
cmp %rdi, %rbx
jbe L(mm_copy_remaining_forward)
.p2align 4 .p2align 4
L(mm_large_page_loop_forward): L(mm_large_page_loop_forward):
movdqu (%rdi, %rsi), %xmm0 movdqu (%r8, %rsi), %xmm0
movdqu 16(%rdi, %rsi), %xmm1 movdqu 16(%r8, %rsi), %xmm1
movdqu 32(%rdi, %rsi), %xmm2 movdqu 32(%r8, %rsi), %xmm2
movdqu 48(%rdi, %rsi), %xmm3 movdqu 48(%r8, %rsi), %xmm3
movdqu 64(%rdi, %rsi), %xmm4 movntdq %xmm0, (%r8)
movdqu 80(%rdi, %rsi), %xmm5 movntdq %xmm1, 16(%r8)
movdqu 96(%rdi, %rsi), %xmm6 movntdq %xmm2, 32(%r8)
movdqu 112(%rdi, %rsi), %xmm7 movntdq %xmm3, 48(%r8)
movntdq %xmm0, (%rdi) lea 64(%r8), %r8
movntdq %xmm1, 16(%rdi) cmp %r8, %rbx
movntdq %xmm2, 32(%rdi)
movntdq %xmm3, 48(%rdi)
movntdq %xmm4, 64(%rdi)
movntdq %xmm5, 80(%rdi)
movntdq %xmm6, 96(%rdi)
movntdq %xmm7, 112(%rdi)
lea 128(%rdi), %rdi
cmp %rdi, %rbx
ja L(mm_large_page_loop_forward) ja L(mm_large_page_loop_forward)
sfence sfence
jmp L(mm_copy_remaining_forward)
add %r9, %rdx
sub %rdi, %rdx
/* We copied all up till %rdi position in the dst.
In %rdx now is how many bytes are left to copy.
Now we need to advance %r8. */
lea (%rdi, %rsi), %r8
cmp $64, %rdx
jb L(mm_remaining_0_64_bytes_forward)
movdqu (%r8), %xmm0
movdqu 16(%r8), %xmm1
movdqu 32(%r8), %xmm2
movdqu 48(%r8), %xmm3
movdqu -64(%r8, %rdx), %xmm4
movdqu -48(%r8, %rdx), %xmm5
movdqu -32(%r8, %rdx), %xmm6
movdqu -16(%r8, %rdx), %xmm7
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, 32(%rdi)
movdqu %xmm3, 48(%rdi)
movdqu %xmm4, -64(%rdi, %rdx)
movdqu %xmm5, -48(%rdi, %rdx)
movdqu %xmm6, -32(%rdi, %rdx)
movdqu %xmm7, -16(%rdi, %rdx)
jmp L(mm_return)
/* Big length copy backward part. */ /* Big length copy backward part. */
L(mm_large_page_backward):
/* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */
movdqu -16(%rsi, %rdx), %xmm0
movdqu -32(%rsi, %rdx), %xmm1
movdqu -48(%rsi, %rdx), %xmm2
movdqu -64(%rsi, %rdx), %xmm3
lea (%rdi, %rdx), %r9
and $-64, %r9
mov %rsi, %r8
sub %rdi, %r8
movdqu -16(%r9, %r8), %xmm4
movdqu -32(%r9, %r8), %xmm5
movdqu -48(%r9, %r8), %xmm6
movdqu -64(%r9, %r8), %xmm7
movdqu %xmm0, -16(%rdi, %rdx)
movdqu %xmm1, -32(%rdi, %rdx)
movdqu %xmm2, -48(%rdi, %rdx)
movdqu %xmm3, -64(%rdi, %rdx)
movntdq %xmm4, -16(%r9)
movntdq %xmm5, -32(%r9)
movntdq %xmm6, -48(%r9)
movntdq %xmm7, -64(%r9)
lea -64(%r9), %r9
lea 128(%rdi), %rbx
and $-64, %rbx
/* Compute in %rdx how many bytes are left to copy after
the main loop stops. */
mov %rbx, %rdx
sub %rdi, %rdx
cmp %r9, %rbx
jae L(mm_len_0_or_more_backward)
.p2align 4 .p2align 4
L(mm_large_page_loop_backward): L(mm_large_page_loop_backward):
movdqu -64(%r9, %r8), %xmm0 movdqu -64(%r9, %r8), %xmm0
@ -630,6 +507,7 @@ L(mm_large_page_loop_backward):
lea -64(%r9), %r9 lea -64(%r9), %r9
cmp %r9, %rbx cmp %r9, %rbx
jb L(mm_large_page_loop_backward) jb L(mm_large_page_loop_backward)
jmp L(mm_len_0_or_more_backward) sfence
jmp L(mm_recalc_len)
END (MEMMOVE) END (MEMMOVE)

29
tests/string_test.cpp
View File

@ -909,6 +909,35 @@ TEST(string, memmove) {
} }
} }
TEST(string, memmove_cache_size) {
size_t len = 600000;
int max_alignment = 31;
int alignments[] = {0, 5, 11, 29, 30};
char* ptr = reinterpret_cast<char*>(malloc(sizeof(char) * len));
char* ptr1 = reinterpret_cast<char*>(malloc(2 * sizeof(char) * len));
char* glob_ptr2 = reinterpret_cast<char*>(malloc(2 * sizeof(char) * len + max_alignment));
size_t pos = 64;
ASSERT_TRUE(ptr != NULL);
ASSERT_TRUE(ptr1 != NULL);
ASSERT_TRUE(glob_ptr2 != NULL);
for (int i = 0; i < 5; i++) {
char* ptr2 = glob_ptr2 + alignments[i];
memset(ptr1, random() & 255, 2 * len);
memset(ptr1, random() & 255, len);
memcpy(ptr2, ptr1, 2 * len);
memcpy(ptr, ptr1, len);
memcpy(ptr1 + pos, ptr, len);
ASSERT_TRUE(memmove(ptr2 + pos, ptr, len) == ptr2 + pos);
ASSERT_EQ(0, memcmp(ptr2, ptr1, 2 * len));
}
free(ptr);
free(ptr1);
free(glob_ptr2);
}
static void verify_memmove(char* src_copy, char* dst, char* src, size_t size) { static void verify_memmove(char* src_copy, char* dst, char* src, size_t size) {
memset(dst, 0, size); memset(dst, 0, size);
memcpy(src, src_copy, size); memcpy(src, src_copy, size);