2be91915dc
This change pulls the memcpy code out into a new file so that the __strcpy_chk and __strcat_chk can use it with an include. The new versions of the two chk functions uses assembly versions of strlen and memcpy to implement this check. This allows near parity with the assembly versions of strcpy/strcat. It also means that as memcpy implementations get faster, so do the chk functions. Other included changes: - Change all of the assembly labels to local labels. The other labels confuse gdb and mess up backtracing. - Add .cfi_startproc and .cfi_endproc directives so that gdb is not confused when falling through from one function to another. - Change all functions to use cfi directives since they are more powerful. - Move the memcpy_chk fail code outside of the memcpy function definition so that backtraces work properly. - Preserve lr before the calls to __fortify_chk_fail so that the backtrace actually works. Testing: - Ran the bionic unit tests. Verified all error messages in logs are set correctly. - Ran libc_test, replacing strcpy with __strcpy_chk and replacing strcat with __strcat_chk. - Ran the debugger on nexus10, nexus4, and old nexus7. Verified that the backtrace is correct for all fortify check failures. Also verify that when falling through from __memcpy_chk to memcpy that the backtrace is still correct. Also verified the same for __memset_chk and bzero. Verified the two different paths in the cortex-a9 memset routine that save variables to the stack still show the backtrace properly. Bug: 9293744 Change-Id: Id5aec8c3cb14101d91bd125eaf3770c9c8aa3f57
207 lines
7.0 KiB
ArmAsm
207 lines
7.0 KiB
ArmAsm
/*
|
|
* Copyright (C) 2008 The Android Open Source Project
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
|
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
|
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
|
|
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
|
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
|
|
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* This code assumes it is running on a processor that supports all arm v7
|
|
* instructions, that supports neon instructions, and that has a 32 byte
|
|
* cache line.
|
|
*/
|
|
|
|
// Check so divider is at least 16 bytes, needed for alignment code.
|
|
cmp r2, #16
|
|
blo 5f
|
|
|
|
|
|
/* check if buffers are aligned. If so, run arm-only version */
|
|
eor r3, r0, r1
|
|
ands r3, r3, #0x3
|
|
beq 11f
|
|
|
|
/* Check the upper size limit for Neon unaligned memory access in memcpy */
|
|
cmp r2, #224
|
|
blo 3f
|
|
|
|
/* align destination to 16 bytes for the write-buffer */
|
|
rsb r3, r0, #0
|
|
ands r3, r3, #0xF
|
|
beq 3f
|
|
|
|
/* copy up to 15-bytes (count in r3) */
|
|
sub r2, r2, r3
|
|
movs ip, r3, lsl #31
|
|
itt mi
|
|
ldrbmi lr, [r1], #1
|
|
strbmi lr, [r0], #1
|
|
itttt cs
|
|
ldrbcs ip, [r1], #1
|
|
ldrbcs lr, [r1], #1
|
|
strbcs ip, [r0], #1
|
|
strbcs lr, [r0], #1
|
|
movs ip, r3, lsl #29
|
|
bge 1f
|
|
// copies 4 bytes, destination 32-bits aligned
|
|
vld1.32 {d0[0]}, [r1]!
|
|
vst1.32 {d0[0]}, [r0, :32]!
|
|
1: bcc 2f
|
|
// copies 8 bytes, destination 64-bits aligned
|
|
vld1.8 {d0}, [r1]!
|
|
vst1.8 {d0}, [r0, :64]!
|
|
2:
|
|
/* preload immediately the next cache line, which we may need */
|
|
pld [r1, #0]
|
|
pld [r1, #(32 * 2)]
|
|
3:
|
|
/* make sure we have at least 64 bytes to copy */
|
|
subs r2, r2, #64
|
|
blo 2f
|
|
|
|
/* preload all the cache lines we need */
|
|
pld [r1, #(32 * 4)]
|
|
pld [r1, #(32 * 6)]
|
|
|
|
1: /* The main loop copies 64 bytes at a time */
|
|
vld1.8 {d0 - d3}, [r1]!
|
|
vld1.8 {d4 - d7}, [r1]!
|
|
pld [r1, #(32 * 6)]
|
|
subs r2, r2, #64
|
|
vst1.8 {d0 - d3}, [r0]!
|
|
vst1.8 {d4 - d7}, [r0]!
|
|
bhs 1b
|
|
|
|
2: /* fix-up the remaining count and make sure we have >= 32 bytes left */
|
|
add r2, r2, #64
|
|
subs r2, r2, #32
|
|
blo 4f
|
|
|
|
3: /* 32 bytes at a time. These cache lines were already preloaded */
|
|
vld1.8 {d0 - d3}, [r1]!
|
|
subs r2, r2, #32
|
|
vst1.8 {d0 - d3}, [r0]!
|
|
bhs 3b
|
|
|
|
4: /* less than 32 left */
|
|
add r2, r2, #32
|
|
tst r2, #0x10
|
|
beq 5f
|
|
// copies 16 bytes, 128-bits aligned
|
|
vld1.8 {d0, d1}, [r1]!
|
|
vst1.8 {d0, d1}, [r0]!
|
|
5: /* copy up to 15-bytes (count in r2) */
|
|
movs ip, r2, lsl #29
|
|
bcc 1f
|
|
vld1.8 {d0}, [r1]!
|
|
vst1.8 {d0}, [r0]!
|
|
1: bge 2f
|
|
vld1.32 {d0[0]}, [r1]!
|
|
vst1.32 {d0[0]}, [r0]!
|
|
2: movs ip, r2, lsl #31
|
|
itt mi
|
|
ldrbmi r3, [r1], #1
|
|
strbmi r3, [r0], #1
|
|
itttt cs
|
|
ldrbcs ip, [r1], #1
|
|
ldrbcs lr, [r1], #1
|
|
strbcs ip, [r0], #1
|
|
strbcs lr, [r0], #1
|
|
|
|
ldmfd sp!, {r0, lr}
|
|
bx lr
|
|
11:
|
|
/* Simple arm-only copy loop to handle aligned copy operations */
|
|
stmfd sp!, {r4, r5, r6, r7, r8}
|
|
pld [r1, #(32 * 4)]
|
|
|
|
/* Check alignment */
|
|
rsb r3, r1, #0
|
|
ands r3, #3
|
|
beq 2f
|
|
|
|
/* align source to 32 bits. We need to insert 2 instructions between
|
|
* a ldr[b|h] and str[b|h] because byte and half-word instructions
|
|
* stall 2 cycles.
|
|
*/
|
|
movs r12, r3, lsl #31
|
|
sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */
|
|
itt mi
|
|
ldrbmi r3, [r1], #1
|
|
strbmi r3, [r0], #1
|
|
itttt cs
|
|
ldrbcs r4, [r1], #1
|
|
ldrbcs r5, [r1], #1
|
|
strbcs r4, [r0], #1
|
|
strbcs r5, [r0], #1
|
|
|
|
2:
|
|
subs r2, r2, #64
|
|
blt 4f
|
|
|
|
3: /* Main copy loop, copying 64 bytes at a time */
|
|
pld [r1, #(32 * 8)]
|
|
ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
subs r2, r2, #64
|
|
bge 3b
|
|
|
|
4: /* Check if there are > 32 bytes left */
|
|
adds r2, r2, #64
|
|
subs r2, r2, #32
|
|
blt 5f
|
|
|
|
/* Copy 32 bytes */
|
|
ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr}
|
|
subs r2, #32
|
|
|
|
5: /* Handle any remaining bytes */
|
|
adds r2, #32
|
|
beq 6f
|
|
|
|
movs r12, r2, lsl #28
|
|
itt cs
|
|
ldmiacs r1!, {r3, r4, r5, r6} /* 16 bytes */
|
|
stmiacs r0!, {r3, r4, r5, r6}
|
|
itt mi
|
|
ldmiami r1!, {r7, r8} /* 8 bytes */
|
|
stmiami r0!, {r7, r8}
|
|
movs r12, r2, lsl #30
|
|
itt cs
|
|
ldrcs r3, [r1], #4 /* 4 bytes */
|
|
strcs r3, [r0], #4
|
|
itt mi
|
|
ldrhmi r4, [r1], #2 /* 2 bytes */
|
|
strhmi r4, [r0], #2
|
|
tst r2, #0x1
|
|
itt ne
|
|
ldrbne r3, [r1] /* last byte */
|
|
strbne r3, [r0]
|
|
6:
|
|
ldmfd sp!, {r4, r5, r6, r7, r8}
|
|
ldmfd sp!, {r0, pc}
|