mirror of
https://github.com/intel/isa-l.git
synced 2025-03-04 07:27:21 +01:00
crc32:Optimize crc32/c for cortex-a72
Change-Id: Ib1658fd4b87b31d8ea6c93f697b50d9b409c186e Signed-off-by: Jerry Yu <jerry.h.yu@arm.com>
This commit is contained in:
Jerry Yu
parent
5e586843eb
commit
0033f42189
@ -46,4 +46,6 @@ lsrc_aarch64 += \
|
||||
crc/aarch64/crc64_jones_refl_pmull.S \
|
||||
crc/aarch64/crc64_jones_norm_pmull.S \
|
||||
crc/aarch64/crc32_mix_neoverse_n1.S \
|
||||
crc/aarch64/crc32c_mix_neoverse_n1.S
|
||||
crc/aarch64/crc32c_mix_neoverse_n1.S \
|
||||
crc/aarch64/crc32_crc_ext_cortex_a72.S \
|
||||
crc/aarch64/crc32c_crc_ext_cortex_a72.S
|
||||
|
||||
135
crc/aarch64/crc32_common_crc_ext_cortex_a72.S
Normal file
135
crc/aarch64/crc32_common_crc_ext_cortex_a72.S
Normal file
@ -0,0 +1,135 @@
|
||||
/**********************************************************************
|
||||
Copyright(c) 2020 Arm Corporation All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Arm Corporation nor the names of its
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
**********************************************************************/
|
||||
|
||||
|
||||
|
||||
|
||||
.macro crc32_hw_common poly_type
|
||||
cbz LEN, .zero_length_ret
|
||||
.ifc \poly_type,crc32
|
||||
mvn wCRC,wCRC
|
||||
.endif
|
||||
tbz BUF, 0, .align_short
|
||||
ldrb wdata,[BUF],1
|
||||
sub LEN,LEN,1
|
||||
crc32_u8 wCRC,wCRC,wdata
|
||||
.align_short:
|
||||
tst BUF,2
|
||||
ccmp LEN,1,0,ne
|
||||
bhi .align_short_2
|
||||
tst BUF,4
|
||||
ccmp LEN,3,0,ne
|
||||
bhi .align_word
|
||||
|
||||
.align_finish:
|
||||
|
||||
cmp LEN, 63
|
||||
bls .loop_16B
|
||||
.loop_64B:
|
||||
ldp data0, data1, [BUF],#16
|
||||
sub LEN,LEN,#64
|
||||
ldp data2, data3, [BUF],#16
|
||||
cmp LEN,#64
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
crc32_u64 wCRC, wCRC, data1
|
||||
ldp data0, data1, [BUF],#16
|
||||
crc32_u64 wCRC, wCRC, data2
|
||||
crc32_u64 wCRC, wCRC, data3
|
||||
ldp data2, data3, [BUF],#16
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
crc32_u64 wCRC, wCRC, data1
|
||||
crc32_u64 wCRC, wCRC, data2
|
||||
crc32_u64 wCRC, wCRC, data3
|
||||
bge .loop_64B
|
||||
|
||||
.loop_16B:
|
||||
cmp LEN, 15
|
||||
bls .less_16B
|
||||
ldp data0, data1, [BUF],#16
|
||||
sub LEN,LEN,#16
|
||||
cmp LEN,15
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
crc32_u64 wCRC, wCRC, data1
|
||||
bls .less_16B
|
||||
ldp data0, data1, [BUF],#16
|
||||
sub LEN,LEN,#16
|
||||
cmp LEN,15
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
crc32_u64 wCRC, wCRC, data1
|
||||
bls .less_16B
|
||||
ldp data0, data1, [BUF],#16
|
||||
sub LEN,LEN,#16 //MUST less than 16B
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
crc32_u64 wCRC, wCRC, data1
|
||||
.less_16B:
|
||||
cmp LEN, 7
|
||||
bls .less_8B
|
||||
ldr data0, [BUF], 8
|
||||
sub LEN, LEN, #8
|
||||
crc32_u64 wCRC, wCRC, data0
|
||||
.less_8B:
|
||||
cmp LEN, 3
|
||||
bls .less_4B
|
||||
ldr wdata, [BUF], 4
|
||||
sub LEN, LEN, #4
|
||||
crc32_u32 wCRC, wCRC, wdata
|
||||
.less_4B:
|
||||
cmp LEN, 1
|
||||
bls .less_2B
|
||||
ldrh wdata, [BUF], 2
|
||||
sub LEN, LEN, #2
|
||||
crc32_u16 wCRC, wCRC, wdata
|
||||
.less_2B:
|
||||
cbz LEN, .finish_exit
|
||||
ldrb wdata, [BUF]
|
||||
crc32_u8 wCRC, wCRC, wdata
|
||||
.finish_exit:
|
||||
.ifc \poly_type,crc32
|
||||
mvn w0, wCRC
|
||||
.else
|
||||
mov w0, wCRC
|
||||
.endif
|
||||
ret
|
||||
.zero_length_ret:
|
||||
mov w0, wCRC
|
||||
ret
|
||||
.align_short_2:
|
||||
ldrh wdata, [BUF], 2
|
||||
sub LEN, LEN, 2
|
||||
tst BUF, 4
|
||||
crc32_u16 wCRC, wCRC, wdata
|
||||
ccmp LEN, 3, 0, ne
|
||||
bls .align_finish
|
||||
.align_word:
|
||||
ldr wdata, [BUF], 4
|
||||
sub LEN, LEN, #4
|
||||
crc32_u32 wCRC, wCRC, wdata
|
||||
b .align_finish
|
||||
|
||||
.endm
|
||||
69
crc/aarch64/crc32_crc_ext_cortex_a72.S
Normal file
69
crc/aarch64/crc32_crc_ext_cortex_a72.S
Normal file
@ -0,0 +1,69 @@
|
||||
/**********************************************************************
|
||||
Copyright(c) 2020 Arm Corporation All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Arm Corporation nor the names of its
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
**********************************************************************/
|
||||
|
||||
.text
|
||||
.align 6
|
||||
.arch armv8-a+crc
|
||||
|
||||
|
||||
#include "crc32_common_crc_ext_cortex_a72.S"
|
||||
|
||||
BUF .req x1
|
||||
LEN .req x2
|
||||
wCRC .req w0
|
||||
data0 .req x4
|
||||
data1 .req x5
|
||||
data2 .req x6
|
||||
data3 .req x7
|
||||
wdata .req w3
|
||||
.macro crc32_u64 dst,src,data
|
||||
crc32x \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u32 dst,src,data
|
||||
crc32w \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u16 dst,src,data
|
||||
crc32h \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u8 dst,src,data
|
||||
crc32b \dst,\src,\data
|
||||
.endm
|
||||
|
||||
/**
|
||||
* uint32_t crc32_crc_ext_cortex_a72(
|
||||
* uint32_t init_crc,
|
||||
* const unsigned char *buf,
|
||||
* uint64_t len);
|
||||
*/
|
||||
.global crc32_crc_ext_cortex_a72
|
||||
.type crc32_crc_ext_cortex_a72, %function
|
||||
crc32_crc_ext_cortex_a72:
|
||||
crc32_hw_common crc32
|
||||
ret
|
||||
.size crc32_crc_ext_cortex_a72, .-crc32_crc_ext_cortex_a72
|
||||
68
crc/aarch64/crc32c_crc_ext_cortex_a72.S
Normal file
68
crc/aarch64/crc32c_crc_ext_cortex_a72.S
Normal file
@ -0,0 +1,68 @@
|
||||
/**********************************************************************
|
||||
Copyright(c) 2020 Arm Corporation All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Arm Corporation nor the names of its
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
**********************************************************************/
|
||||
|
||||
.text
|
||||
.align 6
|
||||
.arch armv8-a+crc
|
||||
|
||||
|
||||
#include "crc32_common_crc_ext_cortex_a72.S"
|
||||
BUF .req x0
|
||||
LEN .req x1
|
||||
wCRC .req w2
|
||||
data0 .req x4
|
||||
data1 .req x5
|
||||
data2 .req x6
|
||||
data3 .req x7
|
||||
wdata .req w3
|
||||
.macro crc32_u64 dst,src,data
|
||||
crc32cx \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u32 dst,src,data
|
||||
crc32cw \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u16 dst,src,data
|
||||
crc32ch \dst,\src,\data
|
||||
.endm
|
||||
.macro crc32_u8 dst,src,data
|
||||
crc32cb \dst,\src,\data
|
||||
.endm
|
||||
|
||||
/**
|
||||
* uint32_t crc32c_crc_ext_cortex_a72(
|
||||
* unsigned char const *buffer,
|
||||
* size_t len,
|
||||
* uint crc32 )
|
||||
*/
|
||||
.global crc32c_crc_ext_cortex_a72
|
||||
.type crc32c_crc_ext_cortex_a72, %function
|
||||
crc32c_crc_ext_cortex_a72:
|
||||
crc32_hw_common crc32c
|
||||
ret
|
||||
.size crc32c_crc_ext_cortex_a72, .-crc32c_crc_ext_cortex_a72
|
||||
@ -62,6 +62,12 @@ DEFINE_INTERFACE_DISPATCHER(crc32_ieee)
|
||||
DEFINE_INTERFACE_DISPATCHER(crc32_iscsi)
|
||||
{
|
||||
unsigned long auxval = getauxval(AT_HWCAP);
|
||||
if (auxval & HWCAP_CRC32) {
|
||||
switch (get_micro_arch_id()) {
|
||||
case MICRO_ARCH_ID(ARM, CORTEX_A72):
|
||||
return PROVIDER_INFO(crc32c_crc_ext_cortex_a72);
|
||||
}
|
||||
}
|
||||
if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) {
|
||||
switch (get_micro_arch_id()) {
|
||||
case MICRO_ARCH_ID(ARM, NEOVERSE_N1):
|
||||
@ -80,6 +86,13 @@ DEFINE_INTERFACE_DISPATCHER(crc32_iscsi)
|
||||
DEFINE_INTERFACE_DISPATCHER(crc32_gzip_refl)
|
||||
{
|
||||
unsigned long auxval = getauxval(AT_HWCAP);
|
||||
|
||||
if (auxval & HWCAP_CRC32) {
|
||||
switch (get_micro_arch_id()) {
|
||||
case MICRO_ARCH_ID(ARM, CORTEX_A72):
|
||||
return PROVIDER_INFO(crc32_crc_ext_cortex_a72);
|
||||
}
|
||||
}
|
||||
if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) {
|
||||
switch (get_micro_arch_id()) {
|
||||
case MICRO_ARCH_ID(ARM, NEOVERSE_N1):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user