generic-library/isa-l
1
0
Fork 0
mirror of https://github.com/intel/isa-l.git synced 2024-12-12 09:23:50 +01:00
Code Issues Projects Releases Wiki Activity

crc: optimize crc with arm64 assembly

Browse Source 
Change-Id: I49166ee06b3ad24babb90aeb0b834d8aacfc2d03
Signed-off-by: Zhiyuan Zhu <zhiyuan.zhu@arm.com>
This commit is contained in:
Zhiyuan Zhu 2019-06-03 09:09:33 +08:00 committed by Richael Zhuang
parent 9c91a18c6e
commit a46da529d9
29 changed files with 4233 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crc/Makefile.am
View File

@ -27,13 +27,14 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
include crc/aarch64/Makefile.am
lsrc += \
crc/crc_base.c \
crc/crc64_base.c
lsrc_base_aliases += crc/crc_base_aliases.c
lsrc_x86_32 += crc/crc_base_aliases.c
lsrc_aarch64 += crc/crc_base_aliases.c
lsrc_x86_64 += \
crc/crc16_t10dif_01.asm \

46
crc/aarch64/Makefile.am Normal file
View File

@ -0,0 +1,46 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
lsrc_aarch64 += \
crc/aarch64/crc_multibinary_arm.S
lsrc_aarch64 += \
crc/aarch64/crc16_t10dif_pmull.S \
crc/aarch64/crc16_t10dif_copy_pmull.S \
crc/aarch64/crc32_iscsi_refl_pmull.S \
crc/aarch64/crc32_iscsi_refl_hw_fold.S \
crc/aarch64/crc32_ieee_norm_pmull.S \
crc/aarch64/crc32_gzip_refl_pmull.S \
crc/aarch64/crc32_gzip_refl_hw_fold.S \
crc/aarch64/crc64_ecma_refl_pmull.S \
crc/aarch64/crc64_ecma_norm_pmull.S \
crc/aarch64/crc64_iso_refl_pmull.S \
crc/aarch64/crc64_iso_norm_pmull.S \
crc/aarch64/crc64_jones_refl_pmull.S \
crc/aarch64/crc64_jones_norm_pmull.S

419
crc/aarch64/crc16_t10dif_copy_pmull.S Normal file
View File

@ -0,0 +1,419 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.arch armv8-a+crc+crypto
.text
.align 3
.global crc16_t10dif_copy_pmull
.type crc16_t10dif_copy_pmull, %function
/* uint16_t crc16_t10dif_pmull(uint16_t seed, uint8_t *buf, uint64_t len) */
/* arguments */
w_seed .req w0
x_dst .req x1
x_src .req x2
x_len .req x3
w_len .req w3
/* returns */
w_ret .req w0
/* these as global temporary registers */
w_tmp .req w6
x_tmp .req x6
x_tmp1 .req x7
x_tmp2 .req x11
d_tmp1 .req d0
d_tmp2 .req d1
q_tmp1 .req q0
q_tmp2 .req q1
v_tmp1 .req v0
v_tmp2 .req v1
/* local variables */
w_counter .req w4
w_crc .req w0
x_crc .req x0
x_counter .req x4
x_crc16tab .req x5
x_src_saved .req x0
x_dst_saved .req x12
crc16_t10dif_copy_pmull:
cmp x_len, 1023
sub sp, sp, #16
uxth w_seed, w_seed
bhi .crc_fold
mov x_tmp, 0
mov w_counter, 0
.crc_table_loop_pre:
cmp x_len, x_tmp
bls .end
sxtw x_counter, w_counter
adrp x_crc16tab, .LANCHOR0
sub x_src, x_src, x_counter
sub x_dst, x_dst, x_counter
add x_crc16tab, x_crc16tab, :lo12:.LANCHOR0
.align 2
.crc_table_loop:
ldrb w_tmp, [x_src, x_counter]
strb w_tmp, [x_dst, x_counter]
add x_counter, x_counter, 1
cmp x_len, x_counter
eor w_tmp, w_tmp, w_crc, lsr 8
ldrh w_tmp, [x_crc16tab, w_tmp, sxtw 1]
eor w_crc, w_tmp, w_crc, lsl 8
uxth w_crc, w_crc
bhi .crc_table_loop
.end:
add sp, sp, 16
ret
/* carry less multiplication, part1 - before loop */
q_x0 .req q2
q_x1 .req q3
q_x2 .req q4
q_x3 .req q5
v_x0 .req v2
v_x1 .req v3
v_x2 .req v4
v_x3 .req v5
d_x0 .req d2
d_x1 .req d3
d_x2 .req d4
d_x3 .req d5
// the following registers only used this part1
d_tmp3 .req d16
v_tmp3 .req v16
.align 3
.crc_fold:
fmov d_tmp1, x_crc
fmov d_tmp2, xzr
dup d_tmp3, v_tmp2.d[0]
shl d_tmp1, d_tmp1, 48
ins v_tmp3.d[1], v_tmp1.d[0]
and x_counter, x_len, -64
sub x_counter, x_counter, #64
cmp x_counter, 63
add x_src_saved, x_src, 64
add x_dst_saved, x_dst, 64
ldr q_x0, [x_src]
ldr q_x1, [x_src, 16]
ldr q_x2, [x_src, 32]
ldr q_x3, [x_src, 48]
str q_x0, [x_dst]
str q_x1, [x_dst, 16]
str q_x2, [x_dst, 32]
str q_x3, [x_dst, 48]
ldr q7, .shuffle_mask
tbl v_tmp1.16b, {v_x0.16b}, v7.16b
eor v_x0.16b, v_tmp3.16b, v_tmp1.16b
tbl v_x1.16b, {v_x1.16b}, v7.16b
tbl v_x2.16b, {v_x2.16b}, v7.16b
tbl v_x3.16b, {v_x3.16b}, v7.16b
bls .crc_fold_loop_end
/* carry less multiplication, part2 - loop */
q_y0 .req q28
q_y1 .req q29
q_y2 .req q30
q_y3 .req q31
v_y0 .req v28
v_y1 .req v29
v_y2 .req v30
v_y3 .req v31
d_x0_h .req d24
d_x0_l .req d2
d_x1_h .req d25
d_x1_l .req d3
d_x2_h .req d26
d_x2_l .req d4
d_x3_h .req d27
d_x3_l .req d5
v_x0_h .req v24
v_x0_l .req v2
v_x1_h .req v25
v_x1_l .req v3
v_x2_h .req v26
v_x2_l .req v4
v_x3_h .req v27
v_x3_l .req v5
v_tmp1_x0 .req v24
v_tmp1_x1 .req v25
v_tmp1_x2 .req v26
v_tmp1_x3 .req v27
d_p4_h .req d19
v_p4_h .req v19
d_p4_l .req d17
v_p4_l .req v17
mov x_tmp, 0x371d0000 /* p4 [1] */
fmov d_p4_h, x_tmp
mov x_tmp, 0x87e70000 /* p4 [0] */
fmov d_p4_l, x_tmp
.align 2
.crc_fold_loop:
add x_src_saved, x_src_saved, 64
add x_dst_saved, x_dst_saved, 64
sub x_counter, x_counter, #64
cmp x_counter, 63
dup d_x0_h, v_x0.d[1]
dup d_x1_h, v_x1.d[1]
dup d_x2_h, v_x2.d[1]
dup d_x3_h, v_x3.d[1]
dup d_x0_l, v_x0.d[0]
dup d_x1_l, v_x1.d[0]
dup d_x2_l, v_x2.d[0]
dup d_x3_l, v_x3.d[0]
ldr q_y0, [x_src_saved, -64]
ldr q_y1, [x_src_saved, -48]
ldr q_y2, [x_src_saved, -32]
ldr q_y3, [x_src_saved, -16]
str q_y0, [x_dst_saved, -64]
str q_y1, [x_dst_saved, -48]
str q_y2, [x_dst_saved, -32]
str q_y3, [x_dst_saved, -16]
pmull v_x0_h.1q, v_x0_h.1d, v_p4_h.1d
pmull v_x0_l.1q, v_x0_l.1d, v_p4_l.1d
pmull v_x1_h.1q, v_x1_h.1d, v_p4_h.1d
pmull v_x1_l.1q, v_x1_l.1d, v_p4_l.1d
pmull v_x2_h.1q, v_x2_h.1d, v_p4_h.1d
pmull v_x2_l.1q, v_x2_l.1d, v_p4_l.1d
pmull v_x3_h.1q, v_x3_h.1d, v_p4_h.1d
pmull v_x3_l.1q, v_x3_l.1d, v_p4_l.1d
tbl v_y0.16b, {v_y0.16b}, v7.16b
tbl v_y1.16b, {v_y1.16b}, v7.16b
tbl v_y2.16b, {v_y2.16b}, v7.16b
tbl v_y3.16b, {v_y3.16b}, v7.16b
eor v_tmp1_x0.16b, v_x0_h.16b, v_x0_l.16b
eor v_tmp1_x1.16b, v_x1_h.16b, v_x1_l.16b
eor v_tmp1_x2.16b, v_x2_h.16b, v_x2_l.16b
eor v_tmp1_x3.16b, v_x3_h.16b, v_x3_l.16b
eor v_x0.16b, v_tmp1_x0.16b, v_y0.16b
eor v_x1.16b, v_tmp1_x1.16b, v_y1.16b
eor v_x2.16b, v_tmp1_x2.16b, v_y2.16b
eor v_x3.16b, v_tmp1_x3.16b, v_y3.16b
bhi .crc_fold_loop
/* carry less multiplication, part3 - after loop */
/* folding 512bit ---> 128bit */
// input parameters:
// v_x0 => v2
// v_x1 => v3
// v_x2 => v4
// v_x3 => v5
// v0, v1, v6, v30, are tmp registers
.crc_fold_loop_end:
mov x_tmp, 0x4c1a0000 /* p1 [1] */
fmov d0, x_tmp
mov x_tmp, 0xfb0b0000 /* p1 [0] */
fmov d1, x_tmp
and w_counter, w_len, -64
sxtw x_tmp, w_counter
add x_src, x_src, x_tmp
add x_dst, x_dst, x_tmp
dup d6, v_x0.d[1]
dup d30, v_x0.d[0]
pmull v6.1q, v6.1d, v0.1d
pmull v30.1q, v30.1d, v1.1d
eor v6.16b, v6.16b, v30.16b
eor v_x1.16b, v6.16b, v_x1.16b
dup d6, v_x1.d[1]
dup d30, v_x1.d[0]
pmull v6.1q, v6.1d, v0.1d
pmull v16.1q, v30.1d, v1.1d
eor v6.16b, v6.16b, v16.16b
eor v_x2.16b, v6.16b, v_x2.16b
dup d_x0, v_x2.d[1]
dup d30, v_x2.d[0]
pmull v0.1q, v_x0.1d, v0.1d
pmull v_x0.1q, v30.1d, v1.1d
eor v1.16b, v0.16b, v_x0.16b
eor v_x0.16b, v1.16b, v_x3.16b
/* carry less multiplication, part3 - after loop */
/* crc16 fold function */
d_16fold_p0_h .req d18
v_16fold_p0_h .req v18
d_16fold_p0_l .req d4
v_16fold_p0_l .req v4
v_16fold_from .req v_x0
d_16fold_from_h .req d3
v_16fold_from_h .req v3
v_16fold_zero .req v7
v_16fold_from1 .req v16
v_16fold_from2 .req v0
d_16fold_from2_h .req d6
v_16fold_from2_h .req v6
v_16fold_tmp .req v0
movi v_16fold_zero.4s, 0
mov x_tmp1, 0x2d560000 /* p0 [1] */
mov x_tmp2, 0x13680000 /* p0 [0] */
ext v_16fold_tmp.16b, v_16fold_zero.16b, v_16fold_from.16b, #8
ext v_16fold_tmp.16b, v0.16b, v_16fold_zero.16b, #4
dup d_16fold_from_h, v_16fold_from.d[1]
fmov d_16fold_p0_h, x_tmp1
pmull v_16fold_from1.1q, v_16fold_from_h.1d, v_16fold_p0_h.1d
eor v_16fold_from2.16b, v_16fold_tmp.16b, v_16fold_from1.16b
dup d_16fold_from2_h, v_16fold_from2.d[1]
fmov d_16fold_p0_l, x_tmp2
pmull v6.1q, v_16fold_from2_h.1d, v_16fold_p0_l.1d
eor v_x0.16b, v0.16b, v6.16b
/* carry less multiplication, part3 - after loop */
/* crc16 barrett reduction function */
// input parameters:
// v_x0: v2
// barrett reduction constant: br[0], br[1]
d_br0 .req d3
v_br0 .req v3
d_br1 .req d5
v_br1 .req v5
mov x_tmp1, 0x57f9 /* br[0] low */
movk x_tmp1, 0xf65a, lsl 16 /* br[0] high */
movk x_tmp1, 0x1, lsl 32
fmov d_br0, x_tmp1
dup d1, v_x0.d[0]
dup d1, v1.d[0]
ext v1.16b, v1.16b, v7.16b, #4
pmull v4.1q, v1.1d, v_br0.1d
ext v1.16b, v4.16b, v7.16b, #4
mov x_tmp1, 0x8bb70000 /* br[1] low */
movk x_tmp1, 0x1, lsl 32 /* br[1] high */
fmov d_br1, x_tmp1
pmull v_br1.1q, v1.1d, v_br1.1d
eor v_x0.16b, v_x0.16b, v_br1.16b
umov x0, v_x0.d[0]
ubfx x0, x0, 16, 16
b .crc_table_loop_pre
.size crc16_t10dif_copy_pmull, .-crc16_t10dif_copy_pmull
.section .rodata
.align 4
.shuffle_mask:
.byte 15, 14, 13, 12, 11, 10, 9, 8
.byte 7, 6, 5, 4, 3, 2, 1, 0
.align 4
.LANCHOR0 = . + 0
.type crc16tab, %object
.size crc16tab, 512
crc16tab:
.hword 0x0000, 0x8bb7, 0x9cd9, 0x176e, 0xb205, 0x39b2, 0x2edc, 0xa56b
.hword 0xEFBD, 0x640a, 0x7364, 0xf8d3, 0x5db8, 0xd60f, 0xc161, 0x4ad6
.hword 0x54CD, 0xdf7a, 0xc814, 0x43a3, 0xe6c8, 0x6d7f, 0x7a11, 0xf1a6
.hword 0xBB70, 0x30c7, 0x27a9, 0xac1e, 0x0975, 0x82c2, 0x95ac, 0x1e1b
.hword 0xA99A, 0x222d, 0x3543, 0xbef4, 0x1b9f, 0x9028, 0x8746, 0x0cf1
.hword 0x4627, 0xcd90, 0xdafe, 0x5149, 0xf422, 0x7f95, 0x68fb, 0xe34c
.hword 0xFD57, 0x76e0, 0x618e, 0xea39, 0x4f52, 0xc4e5, 0xd38b, 0x583c
.hword 0x12EA, 0x995d, 0x8e33, 0x0584, 0xa0ef, 0x2b58, 0x3c36, 0xb781
.hword 0xD883, 0x5334, 0x445a, 0xcfed, 0x6a86, 0xe131, 0xf65f, 0x7de8
.hword 0x373E, 0xbc89, 0xabe7, 0x2050, 0x853b, 0x0e8c, 0x19e2, 0x9255
.hword 0x8C4E, 0x07f9, 0x1097, 0x9b20, 0x3e4b, 0xb5fc, 0xa292, 0x2925
.hword 0x63F3, 0xe844, 0xff2a, 0x749d, 0xd1f6, 0x5a41, 0x4d2f, 0xc698
.hword 0x7119, 0xfaae, 0xedc0, 0x6677, 0xc31c, 0x48ab, 0x5fc5, 0xd472
.hword 0x9EA4, 0x1513, 0x027d, 0x89ca, 0x2ca1, 0xa716, 0xb078, 0x3bcf
.hword 0x25D4, 0xae63, 0xb90d, 0x32ba, 0x97d1, 0x1c66, 0x0b08, 0x80bf
.hword 0xCA69, 0x41de, 0x56b0, 0xdd07, 0x786c, 0xf3db, 0xe4b5, 0x6f02
.hword 0x3AB1, 0xb106, 0xa668, 0x2ddf, 0x88b4, 0x0303, 0x146d, 0x9fda
.hword 0xD50C, 0x5ebb, 0x49d5, 0xc262, 0x6709, 0xecbe, 0xfbd0, 0x7067
.hword 0x6E7C, 0xe5cb, 0xf2a5, 0x7912, 0xdc79, 0x57ce, 0x40a0, 0xcb17
.hword 0x81C1, 0x0a76, 0x1d18, 0x96af, 0x33c4, 0xb873, 0xaf1d, 0x24aa
.hword 0x932B, 0x189c, 0x0ff2, 0x8445, 0x212e, 0xaa99, 0xbdf7, 0x3640
.hword 0x7C96, 0xf721, 0xe04f, 0x6bf8, 0xce93, 0x4524, 0x524a, 0xd9fd
.hword 0xC7E6, 0x4c51, 0x5b3f, 0xd088, 0x75e3, 0xfe54, 0xe93a, 0x628d
.hword 0x285B, 0xa3ec, 0xb482, 0x3f35, 0x9a5e, 0x11e9, 0x0687, 0x8d30
.hword 0xE232, 0x6985, 0x7eeb, 0xf55c, 0x5037, 0xdb80, 0xccee, 0x4759
.hword 0x0D8F, 0x8638, 0x9156, 0x1ae1, 0xbf8a, 0x343d, 0x2353, 0xa8e4
.hword 0xB6FF, 0x3d48, 0x2a26, 0xa191, 0x04fa, 0x8f4d, 0x9823, 0x1394
.hword 0x5942, 0xd2f5, 0xc59b, 0x4e2c, 0xeb47, 0x60f0, 0x779e, 0xfc29
.hword 0x4BA8, 0xc01f, 0xd771, 0x5cc6, 0xf9ad, 0x721a, 0x6574, 0xeec3
.hword 0xA415, 0x2fa2, 0x38cc, 0xb37b, 0x1610, 0x9da7, 0x8ac9, 0x017e
.hword 0x1F65, 0x94d2, 0x83bc, 0x080b, 0xad60, 0x26d7, 0x31b9, 0xba0e
.hword 0xF0D8, 0x7b6f, 0x6c01, 0xe7b6, 0x42dd, 0xc96a, 0xde04, 0x55b3

400
crc/aarch64/crc16_t10dif_pmull.S Normal file
View File

@ -0,0 +1,400 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.arch armv8-a+crc+crypto
.text
.align 3
.global crc16_t10dif_pmull
.type crc16_t10dif_pmull, %function
/* uint16_t crc16_t10dif_pmull(uint16_t seed, uint8_t *buf, uint64_t len) */
/* arguments */
w_seed .req w0
x_buf .req x1
x_len .req x2
w_len .req w2
/* returns */
w_ret .req w0
/* these as global temporary registers */
w_tmp .req w5
x_tmp .req x5
x_tmp1 .req x6
x_tmp2 .req x7
d_tmp1 .req d0
d_tmp2 .req d1
q_tmp1 .req q0
q_tmp2 .req q1
v_tmp1 .req v0
v_tmp2 .req v1
/* local variables */
w_counter .req w3
w_crc .req w0
x_crc .req x0
x_counter .req x3
x_crc16tab .req x4
x_buf_saved .req x0
crc16_t10dif_pmull:
cmp x_len, 1023
sub sp, sp, #16
uxth w_seed, w_seed
bhi .crc_fold
mov x_tmp, 0
mov w_counter, 0
.crc_table_loop_pre:
cmp x_len, x_tmp
bls .end
sxtw x_counter, w_counter
adrp x_crc16tab, .LANCHOR0
sub x_buf, x_buf, x_counter
add x_crc16tab, x_crc16tab, :lo12:.LANCHOR0
.align 2
.crc_table_loop:
ldrb w_tmp, [x_buf, x_counter]
add x_counter, x_counter, 1
cmp x_len, x_counter
eor w_tmp, w_tmp, w_crc, lsr 8
ldrh w_tmp, [x_crc16tab, w_tmp, sxtw 1]
eor w_crc, w_tmp, w_crc, lsl 8
uxth w_crc, w_crc
bhi .crc_table_loop
.end:
add sp, sp, 16
ret
/* carry less multiplication, part1 - before loop */
q_x0 .req q2
q_x1 .req q3
q_x2 .req q4
q_x3 .req q5
v_x0 .req v2
v_x1 .req v3
v_x2 .req v4
v_x3 .req v5
d_x0 .req d2
d_x1 .req d3
d_x2 .req d4
d_x3 .req d5
// the following registers only used this part1
d_tmp3 .req d16
v_tmp3 .req v16
.align 3
.crc_fold:
fmov d_tmp1, x_crc
fmov d_tmp2, xzr
dup d_tmp3, v_tmp2.d[0]
shl d_tmp1, d_tmp1, 48
ins v_tmp3.d[1], v_tmp1.d[0]
and x_counter, x_len, -64
sub x_counter, x_counter, #64
cmp x_counter, 63
add x_buf_saved, x_buf, 64
ldr q_x0, [x_buf]
ldr q_x1, [x_buf, 16]
ldr q_x2, [x_buf, 32]
ldr q_x3, [x_buf, 48]
ldr q7, .shuffle_mask
tbl v_tmp1.16b, {v_x0.16b}, v7.16b
eor v_x0.16b, v_tmp3.16b, v_tmp1.16b
tbl v_x1.16b, {v_x1.16b}, v7.16b
tbl v_x2.16b, {v_x2.16b}, v7.16b
tbl v_x3.16b, {v_x3.16b}, v7.16b
bls .crc_fold_loop_end
/* carry less multiplication, part2 - loop */
q_y0 .req q28
q_y1 .req q29
q_y2 .req q30
q_y3 .req q31
v_y0 .req v28
v_y1 .req v29
v_y2 .req v30
v_y3 .req v31
d_x0_h .req d24
d_x0_l .req d2
d_x1_h .req d25
d_x1_l .req d3
d_x2_h .req d26
d_x2_l .req d4
d_x3_h .req d27
d_x3_l .req d5
v_x0_h .req v24
v_x0_l .req v2
v_x1_h .req v25
v_x1_l .req v3
v_x2_h .req v26
v_x2_l .req v4
v_x3_h .req v27
v_x3_l .req v5
v_tmp1_x0 .req v24
v_tmp1_x1 .req v25
v_tmp1_x2 .req v26
v_tmp1_x3 .req v27
d_p4_h .req d19
v_p4_h .req v19
d_p4_l .req d17
v_p4_l .req v17
mov x_tmp, 0x371d0000 /* p4 [1] */
fmov d_p4_h, x_tmp
mov x_tmp, 0x87e70000 /* p4 [0] */
fmov d_p4_l, x_tmp
.align 2
.crc_fold_loop:
add x_buf_saved, x_buf_saved, 64
sub x_counter, x_counter, #64
cmp x_counter, 63
dup d_x0_h, v_x0.d[1]
dup d_x1_h, v_x1.d[1]
dup d_x2_h, v_x2.d[1]
dup d_x3_h, v_x3.d[1]
dup d_x0_l, v_x0.d[0]
dup d_x1_l, v_x1.d[0]
dup d_x2_l, v_x2.d[0]
dup d_x3_l, v_x3.d[0]
ldr q_y0, [x_buf_saved, -64]
ldr q_y1, [x_buf_saved, -48]
ldr q_y2, [x_buf_saved, -32]
ldr q_y3, [x_buf_saved, -16]
pmull v_x0_h.1q, v_x0_h.1d, v_p4_h.1d
pmull v_x0_l.1q, v_x0_l.1d, v_p4_l.1d
pmull v_x1_h.1q, v_x1_h.1d, v_p4_h.1d
pmull v_x1_l.1q, v_x1_l.1d, v_p4_l.1d
pmull v_x2_h.1q, v_x2_h.1d, v_p4_h.1d
pmull v_x2_l.1q, v_x2_l.1d, v_p4_l.1d
pmull v_x3_h.1q, v_x3_h.1d, v_p4_h.1d
pmull v_x3_l.1q, v_x3_l.1d, v_p4_l.1d
tbl v_y0.16b, {v_y0.16b}, v7.16b
tbl v_y1.16b, {v_y1.16b}, v7.16b
tbl v_y2.16b, {v_y2.16b}, v7.16b
tbl v_y3.16b, {v_y3.16b}, v7.16b
eor v_tmp1_x0.16b, v_x0_h.16b, v_x0_l.16b
eor v_tmp1_x1.16b, v_x1_h.16b, v_x1_l.16b
eor v_tmp1_x2.16b, v_x2_h.16b, v_x2_l.16b
eor v_tmp1_x3.16b, v_x3_h.16b, v_x3_l.16b
eor v_x0.16b, v_tmp1_x0.16b, v_y0.16b
eor v_x1.16b, v_tmp1_x1.16b, v_y1.16b
eor v_x2.16b, v_tmp1_x2.16b, v_y2.16b
eor v_x3.16b, v_tmp1_x3.16b, v_y3.16b
bhi .crc_fold_loop
/* carry less multiplication, part3 - after loop */
/* folding 512bit ---> 128bit */
// input parameters:
// v_x0 => v2
// v_x1 => v3
// v_x2 => v4
// v_x3 => v5
// v0, v1, v6, v30, are tmp registers
.crc_fold_loop_end:
mov x_tmp, 0x4c1a0000 /* p1 [1] */
fmov d0, x_tmp
mov x_tmp, 0xfb0b0000 /* p1 [0] */
fmov d1, x_tmp
and w_counter, w_len, -64
sxtw x_tmp, w_counter
add x_buf, x_buf, x_tmp
dup d6, v_x0.d[1]
dup d30, v_x0.d[0]
pmull v6.1q, v6.1d, v0.1d
pmull v30.1q, v30.1d, v1.1d
eor v6.16b, v6.16b, v30.16b
eor v_x1.16b, v6.16b, v_x1.16b
dup d6, v_x1.d[1]
dup d30, v_x1.d[0]
pmull v6.1q, v6.1d, v0.1d
pmull v16.1q, v30.1d, v1.1d
eor v6.16b, v6.16b, v16.16b
eor v_x2.16b, v6.16b, v_x2.16b
dup d_x0, v_x2.d[1]
dup d30, v_x2.d[0]
pmull v0.1q, v_x0.1d, v0.1d
pmull v_x0.1q, v30.1d, v1.1d
eor v1.16b, v0.16b, v_x0.16b
eor v_x0.16b, v1.16b, v_x3.16b
/* carry less multiplication, part3 - after loop */
/* crc16 fold function */
d_16fold_p0_h .req d18
v_16fold_p0_h .req v18
d_16fold_p0_l .req d4
v_16fold_p0_l .req v4
v_16fold_from .req v_x0
d_16fold_from_h .req d3
v_16fold_from_h .req v3
v_16fold_zero .req v7
v_16fold_from1 .req v16
v_16fold_from2 .req v0
d_16fold_from2_h .req d6
v_16fold_from2_h .req v6
v_16fold_tmp .req v0
movi v_16fold_zero.4s, 0
mov x_tmp1, 0x2d560000 /* p0 [1] */
mov x_tmp2, 0x13680000 /* p0 [0] */
ext v_16fold_tmp.16b, v_16fold_zero.16b, v_16fold_from.16b, #8
ext v_16fold_tmp.16b, v0.16b, v_16fold_zero.16b, #4
dup d_16fold_from_h, v_16fold_from.d[1]
fmov d_16fold_p0_h, x_tmp1
pmull v_16fold_from1.1q, v_16fold_from_h.1d, v_16fold_p0_h.1d
eor v_16fold_from2.16b, v_16fold_tmp.16b, v_16fold_from1.16b
dup d_16fold_from2_h, v_16fold_from2.d[1]
fmov d_16fold_p0_l, x_tmp2
pmull v6.1q, v_16fold_from2_h.1d, v_16fold_p0_l.1d
eor v_x0.16b, v0.16b, v6.16b
/* carry less multiplication, part3 - after loop */
/* crc16 barrett reduction function */
// input parameters:
// v_x0: v2
// barrett reduction constant: br[0], br[1]
d_br0 .req d3
v_br0 .req v3
d_br1 .req d5
v_br1 .req v5
mov x_tmp1, 0x57f9 /* br[0] low */
movk x_tmp1, 0xf65a, lsl 16 /* br[0] high */
movk x_tmp1, 0x1, lsl 32
fmov d_br0, x_tmp1
dup d1, v_x0.d[0]
dup d1, v1.d[0]
ext v1.16b, v1.16b, v7.16b, #4
pmull v4.1q, v1.1d, v_br0.1d
ext v1.16b, v4.16b, v7.16b, #4
mov x_tmp1, 0x8bb70000 /* br[1] low */
movk x_tmp1, 0x1, lsl 32 /* br[1] high */
fmov d_br1, x_tmp1
pmull v_br1.1q, v1.1d, v_br1.1d
eor v_x0.16b, v_x0.16b, v_br1.16b
umov x0, v_x0.d[0]
ubfx x0, x0, 16, 16
b .crc_table_loop_pre
.size crc16_t10dif_pmull, .-crc16_t10dif_pmull
.section .rodata
.align 4
.shuffle_mask:
.byte 15, 14, 13, 12, 11, 10, 9, 8
.byte 7, 6, 5, 4, 3, 2, 1, 0
.align 4
.LANCHOR0 = . + 0
.type crc16tab, %object
.size crc16tab, 512
crc16tab:
.hword 0x0000, 0x8bb7, 0x9cd9, 0x176e, 0xb205, 0x39b2, 0x2edc, 0xa56b
.hword 0xEFBD, 0x640a, 0x7364, 0xf8d3, 0x5db8, 0xd60f, 0xc161, 0x4ad6
.hword 0x54CD, 0xdf7a, 0xc814, 0x43a3, 0xe6c8, 0x6d7f, 0x7a11, 0xf1a6
.hword 0xBB70, 0x30c7, 0x27a9, 0xac1e, 0x0975, 0x82c2, 0x95ac, 0x1e1b
.hword 0xA99A, 0x222d, 0x3543, 0xbef4, 0x1b9f, 0x9028, 0x8746, 0x0cf1
.hword 0x4627, 0xcd90, 0xdafe, 0x5149, 0xf422, 0x7f95, 0x68fb, 0xe34c
.hword 0xFD57, 0x76e0, 0x618e, 0xea39, 0x4f52, 0xc4e5, 0xd38b, 0x583c
.hword 0x12EA, 0x995d, 0x8e33, 0x0584, 0xa0ef, 0x2b58, 0x3c36, 0xb781
.hword 0xD883, 0x5334, 0x445a, 0xcfed, 0x6a86, 0xe131, 0xf65f, 0x7de8
.hword 0x373E, 0xbc89, 0xabe7, 0x2050, 0x853b, 0x0e8c, 0x19e2, 0x9255
.hword 0x8C4E, 0x07f9, 0x1097, 0x9b20, 0x3e4b, 0xb5fc, 0xa292, 0x2925
.hword 0x63F3, 0xe844, 0xff2a, 0x749d, 0xd1f6, 0x5a41, 0x4d2f, 0xc698
.hword 0x7119, 0xfaae, 0xedc0, 0x6677, 0xc31c, 0x48ab, 0x5fc5, 0xd472
.hword 0x9EA4, 0x1513, 0x027d, 0x89ca, 0x2ca1, 0xa716, 0xb078, 0x3bcf
.hword 0x25D4, 0xae63, 0xb90d, 0x32ba, 0x97d1, 0x1c66, 0x0b08, 0x80bf
.hword 0xCA69, 0x41de, 0x56b0, 0xdd07, 0x786c, 0xf3db, 0xe4b5, 0x6f02
.hword 0x3AB1, 0xb106, 0xa668, 0x2ddf, 0x88b4, 0x0303, 0x146d, 0x9fda
.hword 0xD50C, 0x5ebb, 0x49d5, 0xc262, 0x6709, 0xecbe, 0xfbd0, 0x7067
.hword 0x6E7C, 0xe5cb, 0xf2a5, 0x7912, 0xdc79, 0x57ce, 0x40a0, 0xcb17
.hword 0x81C1, 0x0a76, 0x1d18, 0x96af, 0x33c4, 0xb873, 0xaf1d, 0x24aa
.hword 0x932B, 0x189c, 0x0ff2, 0x8445, 0x212e, 0xaa99, 0xbdf7, 0x3640
.hword 0x7C96, 0xf721, 0xe04f, 0x6bf8, 0xce93, 0x4524, 0x524a, 0xd9fd
.hword 0xC7E6, 0x4c51, 0x5b3f, 0xd088, 0x75e3, 0xfe54, 0xe93a, 0x628d
.hword 0x285B, 0xa3ec, 0xb482, 0x3f35, 0x9a5e, 0x11e9, 0x0687, 0x8d30
.hword 0xE232, 0x6985, 0x7eeb, 0xf55c, 0x5037, 0xdb80, 0xccee, 0x4759
.hword 0x0D8F, 0x8638, 0x9156, 0x1ae1, 0xbf8a, 0x343d, 0x2353, 0xa8e4
.hword 0xB6FF, 0x3d48, 0x2a26, 0xa191, 0x04fa, 0x8f4d, 0x9823, 0x1394
.hword 0x5942, 0xd2f5, 0xc59b, 0x4e2c, 0xeb47, 0x60f0, 0x779e, 0xfc29
.hword 0x4BA8, 0xc01f, 0xd771, 0x5cc6, 0xf9ad, 0x721a, 0x6574, 0xeec3
.hword 0xA415, 0x2fa2, 0x38cc, 0xb37b, 0x1610, 0x9da7, 0x8ac9, 0x017e
.hword 0x1F65, 0x94d2, 0x83bc, 0x080b, 0xad60, 0x26d7, 0x31b9, 0xba0e
.hword 0xF0D8, 0x7b6f, 0x6c01, 0xe7b6, 0x42dd, 0xc96a, 0xde04, 0x55b3

176
crc/aarch64/crc32_gzip_refl_hw_fold.S Normal file
View File

@ -0,0 +1,176 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.arch armv8-a+crc+crypto
.text
.align 3
.global crc32_gzip_refl_hw_fold
.type crc32_gzip_refl_hw_fold, %function
/* uint32_t crc32_gzip_refl_hw_fold(uint32_t seed, const unsigned char *buf, uint64_t len) */
w_seed .req w0
w_crc .req w0
x_buf .req x1
x_len .req x2
x_buf_loop_end .req x10
x_buf_iter .req x10
x_tmp .req x15
w_tmp .req w15
d_c0 .req d3
d_c1 .req d1
v_c0 .req v3
v_c1 .req v1
crc32_gzip_refl_hw_fold:
mvn w_seed, w_seed
cmp x_len, 1023
mov x_buf_iter, x_buf
bls .loop_fold_end
sub x_buf_loop_end, x_len, #1024
and x_buf_loop_end, x_buf_loop_end, -1024
add x_buf_loop_end, x_buf_loop_end, 1024
add x_buf_loop_end, x_buf, x_buf_loop_end
mov x_tmp, 0x819b
movk x_tmp, 0xb486, lsl 16
fmov d_c0, x_tmp
mov x_tmp, 0x8617
movk x_tmp, 0x7627, lsl 16
fmov d_c1, x_tmp
x_in64 .req x3
w_crc0 .req w0
w_crc1 .req w4
w_crc2 .req w5
d_crc0 .req d4
d_crc1 .req d5
v_crc0 .req v4
v_crc1 .req v5
.align 3
.loop_fold:
add x9, x_buf, 336
mov x_in64, x_buf
mov w_crc1, 0
mov w_crc2, 0
.align 3
.loop_for:
ldr x8, [x_in64]
ldr x7, [x_in64, 336]
ldr x6, [x_in64, 672]
add x_in64, x_in64, 8
cmp x_in64, x9
crc32x w_crc0, w_crc0, x8
crc32x w_crc1, w_crc1, x7
crc32x w_crc2, w_crc2, x6
bne .loop_for
uxtw x_tmp, w_crc0
fmov d_crc0, x_tmp
pmull v_crc0.1q, v_crc0.1d, v_c0.1d
uxtw x_tmp, w_crc1
fmov d_crc1, x_tmp
pmull v_crc1.1q, v_crc1.1d, v_c1.1d
ldr x_tmp, [x_buf, 1008]
crc32x w_crc2, w_crc2, x_tmp
fmov x_tmp, d_crc0
crc32x w_crc0, wzr, x_tmp
fmov x_tmp, d_crc1
crc32x w_crc1, wzr, x_tmp
eor w_crc0, w_crc0, w_crc1
eor w_crc0, w_crc0, w_crc2
ldr x_tmp, [x_buf, 1016]
crc32x w_crc0, w_crc0, x_tmp
add x_buf, x_buf, 1024
cmp x_buf_loop_end, x_buf
bne .loop_fold
and x_len, x_len, 1023
x_buf_loop_size8_end .req x3
.loop_fold_end:
cmp x_len, 7
bls .size_4
sub x_buf_loop_size8_end, x_len, #8
and x_buf_loop_size8_end, x_buf_loop_size8_end, -8
add x_buf_loop_size8_end, x_buf_loop_size8_end, 8
add x_buf_loop_size8_end, x_buf_iter, x_buf_loop_size8_end
.align 3
.loop_size_8:
ldr x_tmp, [x_buf_iter], 8
crc32x w_crc, w_crc, x_tmp
cmp x_buf_iter, x_buf_loop_size8_end
bne .loop_size_8
and x_len, x_len, 7
.size_4:
cmp x_len, 3
bls .size_2
ldr w_tmp, [x_buf_iter], 4
crc32w w_crc, w_crc, w_tmp
sub x_len, x_len, #4
.size_2:
cmp x_len, 1
bls .size_1
ldrh w_tmp, [x_buf_iter], 2
crc32h w_crc, w_crc, w_tmp
sub x_len, x_len, #2
.size_1:
cbz x_len, .done
ldrb w_tmp, [x_buf_iter]
crc32b w_crc, w_crc, w_tmp
.done:
mvn w_crc, w_crc
ret
.size crc32_gzip_refl_hw_fold, .-crc32_gzip_refl_hw_fold

33
crc/aarch64/crc32_gzip_refl_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc32_gzip_refl_pmull.h"
#include "crc32_refl_common_pmull.h"
crc32_refl_func crc32_gzip_refl_pmull

87
crc/aarch64/crc32_gzip_refl_pmull.h Normal file
View File

@ -0,0 +1,87 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0x2d95
.equ p4_low_b1, 0x8f35
.equ p4_high_b0, 0x13d7
.equ p4_high_b1, 0x1d95
.equ p1_low_b0, 0x9191
.equ p1_low_b1, 0xae68
.equ p1_high_b0, 0x009e
.equ p1_high_b1, 0xccaa
.equ p0_low_b0, 0x6765
.equ p0_low_b1, 0xb8bc
.equ p0_high_b0, p1_high_b0
.equ p0_high_b1, p1_high_b1
.equ br_low_b0, 0x0641
.equ br_low_b1, 0xdb71
.equ br_low_b2, 0x1
.equ br_high_b0, 0x1641
.equ br_high_b1, 0xf701
.equ br_high_b2, 0x1
.text
.section .rodata
.align 4
.set .LANCHOR0,. + 0
.type crc32_table_gzip_refl, %object
.size crc32_table_gzip_refl, 1024
crc32_table_gzip_refl:
.word 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3
.word 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91
.word 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7
.word 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5
.word 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b
.word 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59
.word 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f
.word 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d
.word 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433
.word 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01
.word 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457
.word 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65
.word 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb
.word 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9
.word 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f
.word 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad
.word 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683
.word 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1
.word 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7
.word 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5
.word 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b
.word 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79
.word 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f
.word 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d
.word 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713
.word 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21
.word 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777
.word 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45
.word 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db
.word 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9
.word 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf
.word 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d

33
crc/aarch64/crc32_ieee_norm_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc32_ieee_norm_pmull.h"
#include "crc32_norm_common_pmull.h"
crc32_norm_func crc32_ieee_norm_pmull

87
crc/aarch64/crc32_ieee_norm_pmull.h Normal file
View File

@ -0,0 +1,87 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0x8b11
.equ p4_low_b1, 0xe622
.equ p4_high_b0, 0x794c
.equ p4_high_b1, 0x8833
.equ p1_low_b0, 0x5605
.equ p1_low_b1, 0xe8a4
.equ p1_high_b0, 0xcd4c
.equ p1_high_b1, 0xc5b9
.equ p0_low_b0, 0x678d
.equ p0_low_b1, 0x490d
.equ p0_high_b0, 0xaa66
.equ p0_high_b1, 0xf200
.equ br_low_b0, 0x01df
.equ br_low_b1, 0x04d1
.equ br_low_b2, 0x1
.equ br_high_b0, 0x1db7
.equ br_high_b1, 0x04c1
.equ br_high_b2, 0x1
.text
.section .rodata
.align 4
.set .LANCHOR0,. + 0
.type crc32_table_ieee_norm, %object
.size crc32_table_ieee_norm, 1024
crc32_table_ieee_norm:
.word 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005
.word 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd
.word 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75
.word 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd
.word 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5
.word 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d
.word 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95
.word 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d
.word 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072
.word 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca
.word 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02
.word 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba
.word 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692
.word 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a
.word 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2
.word 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a
.word 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb
.word 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53
.word 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b
.word 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623
.word 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b
.word 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3
.word 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b
.word 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3
.word 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c
.word 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24
.word 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec
.word 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654
.word 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c
.word 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4
.word 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c
.word 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4

172
crc/aarch64/crc32_iscsi_refl_hw_fold.S Normal file
View File

@ -0,0 +1,172 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.arch armv8-a+crc+crypto
.text
.align 3
.global crc32_iscsi_refl_hw_fold
.type crc32_iscsi_refl_hw_fold, %function
/* unsigned int crc32_iscsi_refl_hw_fold(unsigned char *buffer, int len, unsigned int crc_init) */
x_buffer .req x0
w_len .req w1
w_crc_init .req w2
w_crc .req w2
w_len_loop_end .req w9
x_buf_loop_end .req x9
x_buf_iter .req x9
x_tmp .req x15
w_tmp .req w15
w_crc_ret .req w0
crc32_iscsi_refl_hw_fold:
cmp w_len, 1023
mov x_buf_iter, x_buffer
ble .loop_fold_end
sub w10, w_len, #1024
lsr w12, w10, 10
lsl w_len_loop_end, w12, 10
add x_buf_loop_end, x_buf_loop_end, 1024
add x_buf_loop_end, x_buffer, x_buf_loop_end
mov x_tmp, 0xf38a
movk x_tmp, 0xe417, lsl 16
fmov d3, x_tmp
mov x_tmp, 0x8014
movk x_tmp, 0x8f15, lsl 16
fmov d1, x_tmp
x_in64 .req x1
w_crc0 .req w2
w_crc1 .req w3
w_crc2 .req w4
.align 3
.loop_fold:
add x8, x_buffer, 336
mov x_in64, x_buffer
mov w_crc1, 0
mov w_crc2, 0
.align 3
.loop_for:
ldr x7, [x_in64]
ldr x6, [x_in64, 336]
ldr x5, [x_in64, 672]
add x_in64, x_in64, 8
cmp x_in64, x8
crc32cx w_crc0, w_crc0, x7
crc32cx w_crc1, w_crc1, x6
crc32cx w_crc2, w_crc2, x5
bne .loop_for
uxtw x_tmp, w_crc0
fmov d4, x_tmp
pmull v2.1q, v4.1d, v3.1d
uxtw x_tmp, w_crc1
fmov d5, x_tmp
pmull v5.1q, v5.1d, v1.1d
fmov x_tmp, d2
crc32cx w_crc0, wzr, x_tmp
fmov x_tmp, d5
crc32cx w_crc1, wzr, x_tmp
ldr x_tmp, [x_buffer, 1008]
crc32cx w_crc2, w_crc2, x_tmp
eor w_crc1, w_crc1, w_crc0
eor w_crc1, w_crc1, w_crc2
ldr x_tmp, [x_buffer, 1016]
crc32cx w_crc0, w_crc1, x_tmp
add x_buffer, x_buffer, 1024
cmp x_buf_loop_end, x_buffer
bne .loop_fold
sub w_len, w10, w12, lsl 10
x_buf_loop_size8_end .req x3
.loop_fold_end:
cmp w_len, 7
ble .size_4
sub w_len, w_len, #8
lsr w4, w_len, 3
lsl w3, w4, 3
add x_buf_loop_size8_end, x_buf_loop_size8_end, 8
add x_buf_loop_size8_end, x_buf_iter, x_buf_loop_size8_end
.align 3
.loop_size_8:
ldr x_tmp, [x_buf_iter], 8
crc32cx w_crc, w_crc, x_tmp
cmp x_buf_iter, x_buf_loop_size8_end
bne .loop_size_8
sub w_len, w_len, w4, lsl 3
.size_4:
cmp w_len, 3
ble .size_2
ldr w_tmp, [x_buf_iter], 4
crc32cw w_crc, w_crc, w_tmp
sub w_len, w_len, #4
.size_2:
cmp w_len, 1
ble .size_1
ldrh w_tmp, [x_buf_iter], 2
crc32ch w_crc, w_crc, w_tmp
sub w_len, w_len, #2
.size_1:
mov w_crc_ret, w_crc
cmp w_len, 1
bne .done
ldrb w_tmp, [x_buf_iter]
crc32cb w_crc_ret, w_crc, w_tmp
.done:
ret
.size crc32_iscsi_refl_hw_fold, .-crc32_iscsi_refl_hw_fold

53
crc/aarch64/crc32_iscsi_refl_pmull.S Normal file
View File

@ -0,0 +1,53 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc32_iscsi_refl_pmull.h"
#include "crc32_refl_common_pmull.h"
crc32_refl_func crc32_iscsi_refl_pmull_internal
.arch armv8-a+crc+crypto
.text
.align 3
.global crc32_iscsi_refl_pmull
.type crc32_iscsi_refl_pmull, %function
crc32_iscsi_refl_pmull:
stp x29, x30, [sp, -32]!
mov x29, sp
mov w7, w2
sxtw x2, w1
mov x1, x0
mov w0, w7
mvn w0, w0
bl crc32_iscsi_refl_pmull_internal
mvn w0, w0
ldp x29, x30, [sp], 32
ret

87
crc/aarch64/crc32_iscsi_refl_pmull.h Normal file
View File

@ -0,0 +1,87 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0xef02
.equ p4_low_b1, 0x740e
.equ p4_high_b0, 0xddf8
.equ p4_high_b1, 0x9e4a
.equ p1_low_b0, 0x0dfe
.equ p1_low_b1, 0xf20c
.equ p1_high_b0, 0x7d27
.equ p1_high_b1, 0x493c
.equ p0_low_b0, 0xaab8
.equ p0_low_b1, 0xdd45
.equ p0_high_b0, p1_high_b0
.equ p0_high_b1, p1_high_b1
.equ br_low_b0, 0x76f1
.equ br_low_b1, 0x05ec
.equ br_low_b2, 0x1
.equ br_high_b0, 0x13f1
.equ br_high_b1, 0xdea7
.equ br_high_b2, 0x0
.text
.section .rodata
.align 4
.set .LANCHOR0,. + 0
.type crc32_table_iscsi_refl, %object
.size crc32_table_iscsi_refl, 1024
crc32_table_iscsi_refl:
.word 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB
.word 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24
.word 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384
.word 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B
.word 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35
.word 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA
.word 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A
.word 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595
.word 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957
.word 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198
.word 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38
.word 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7
.word 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789
.word 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46
.word 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6
.word 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829
.word 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93
.word 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C
.word 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC
.word 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033
.word 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D
.word 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982
.word 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622
.word 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED
.word 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F
.word 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0
.word 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540
.word 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F
.word 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1
.word 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E
.word 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E
.word 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351

316
crc/aarch64/crc32_norm_common_pmull.h Normal file
View File

@ -0,0 +1,316 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.macro crc32_norm_func name:req
.arch armv8-a+crc+crypto
.text
.align 3
.global \name
.type \name, %function
/* crc32_norm_func(uint32_t seed, uint8_t * buf, uint64_t len) */
// constant
.equ FOLD_SIZE, 1024
// parameter
w_seed .req w0
x_seed .req x0
x_buf .req x1
x_len .req x2
x_buf_tmp .req x0
// crc32 normal function entry
\name\():
mvn w_seed, w_seed
mov x3, 0
mov w4, 0
cmp x_len, (FOLD_SIZE - 1)
uxtw x_seed, w_seed
bhi .crc32_clmul_pre
.crc32_norm_tab_pre:
cmp x_len, x3
bls .done
sxtw x4, w4
adrp x5, .LANCHOR0
sub x_buf, x_buf, x4
add x5, x5, :lo12:.LANCHOR0
.align 3
.loop_crc32_norm_tab:
ldrb w3, [x_buf, x4]
add x4, x4, 1
cmp x_len, x4
eor x3, x3, x0, lsr 24
and x3, x3, 255
ldr w3, [x5, x3, lsl 2]
eor x0, x3, x0, lsl 8
bhi .loop_crc32_norm_tab
.done:
mvn w_seed, w_seed
ret
// crcc32 clmul prepare
x_buf_end .req x3
q_shuffle .req q3
v_shuffle .req v3
q_x0_tmp .req q5
q_x1 .req q6
q_x2 .req q4
q_x3 .req q1
v_x0_tmp .req v5
v_x0 .req v2
v_x1 .req v6
v_x2 .req v4
v_x3 .req v1
d_p4_high .req d7
d_p4_low .req d5
.align 2
.crc32_clmul_pre:
and x3, x_len, -64
cmp x3, 63
bls .clmul_end
lsl x_seed, x_seed, 32
movi v2.4s, 0
ins v2.d[1], x_seed
adrp x4, .shuffle
ldr q_shuffle, [x4, #:lo12:.shuffle]
sub x4, x3, #64
cmp x4, 63
ldr q_x0_tmp, [x_buf]
ldr q_x1, [x_buf, 16]
ldr q_x2, [x_buf, 32]
ldr q_x3, [x_buf, 48]
add x_buf_tmp, x_buf, 64
tbl v_x0_tmp.16b, {v_x0_tmp.16b}, v_shuffle.16b
tbl v_x1.16b, {v_x1.16b}, v_shuffle.16b
tbl v_x2.16b, {v_x2.16b}, v_shuffle.16b
tbl v_x3.16b, {v_x3.16b}, v_shuffle.16b
eor v_x0.16b, v_x0.16b, v_x0_tmp.16b
bls .clmul_loop_end
add x_buf_end, x_buf_tmp, x4
mov x4, p4_high_b0
movk x4, p4_high_b1, lsl 16
fmov d_p4_high, x4
mov x4, p4_low_b0
movk x4, p4_low_b1, lsl 16
fmov d_p4_low, x4
// crc32 clmul loop
//v_x0 .req v2
//v_x1 .req v6
//v_x2 .req v4
//v_x3 .req v1
d_x0_high .req d22
d_x1_high .req d20
d_x2_high .req d18
d_x3_high .req d16
v_x0_high .req v22
v_x1_high .req v20
v_x2_high .req v18
v_x3_high .req v16
q_y0_high .req q23
q_y1_high .req q21
q_y2_high .req q19
q_y3_high .req q17
v_y0_high .req v23
v_y1_high .req v21
v_y2_high .req v19
v_y3_high .req v17
v_p4_high .req v7
v_p4_low .req v5
//v_shuffle .req v3
.align 3
.clmul_loop:
dup d_x0_high, v_x0.d[1]
dup d_x1_high, v_x1.d[1]
dup d_x2_high, v_x2.d[1]
dup d_x3_high, v_x3.d[1]
add x_buf_tmp, x_buf_tmp, 64
ldr q_y0_high, [x_buf_tmp, -64]
ldr q_y1_high, [x_buf_tmp, -48]
ldr q_y2_high, [x_buf_tmp, -32]
ldr q_y3_high, [x_buf_tmp, -16]
cmp x_buf_tmp, x_buf_end
pmull v_x0.1q, v_x0.1d, v_p4_low.1d
pmull v_x1.1q, v_x1.1d, v_p4_low.1d
pmull v_x2.1q, v_x2.1d, v_p4_low.1d
pmull v_x3.1q, v_x3.1d, v_p4_low.1d
pmull v_x0_high.1q, v_x0_high.1d, v_p4_high.1d
pmull v_x1_high.1q, v_x1_high.1d, v_p4_high.1d
pmull v_x2_high.1q, v_x2_high.1d, v_p4_high.1d
pmull v_x3_high.1q, v_x3_high.1d, v_p4_high.1d
eor v_x0.16b, v_x0_high.16b, v_x0.16b
eor v_x1.16b, v_x1_high.16b, v_x1.16b
eor v_x2.16b, v_x2_high.16b, v_x2.16b
eor v_x3.16b, v_x3_high.16b, v_x3.16b
tbl v_y0_high.16b, {v_y0_high.16b}, v_shuffle.16b
tbl v_y1_high.16b, {v_y1_high.16b}, v_shuffle.16b
tbl v_y2_high.16b, {v_y2_high.16b}, v_shuffle.16b
tbl v_y3_high.16b, {v_y3_high.16b}, v_shuffle.16b
eor v_x0.16b, v_x0.16b, v_y0_high.16b
eor v_x1.16b, v_x1.16b, v_y1_high.16b
eor v_x2.16b, v_x2.16b, v_y2_high.16b
eor v_x3.16b, v_x3.16b, v_y3_high.16b
bne .clmul_loop
//v_x0 .req v2
//v_x1 .req v6
//v_x2 .req v4
//v_x3 .req v1
d_p1_high .req d7
d_p1_low .req d5
v_p1_high .req v7
v_p1_low .req v5
.clmul_loop_end:
// folding 512bit --> 128bit
mov x0, p1_high_b0
movk x0, p1_high_b1, lsl 16
fmov d_p1_high, x0
mov x0, p1_low_b0
movk x0, p1_low_b1, lsl 16
fmov d_p1_low, x0
dup d16, v_x0.d[1]
pmull v_x0.1q, v_x0.1d, v_p1_low.1d
pmull v16.1q, v16.1d, v_p1_high.1d
eor v_x0.16b, v16.16b, v_x0.16b
eor v_x1.16b, v_x0.16b, v_x1.16b
dup d17, v_x1.d[1]
pmull v_x1.1q, v_x1.1d, v_p1_low.1d
pmull v17.1q, v17.1d, v_p1_high.1d
eor v_x1.16b, v17.16b, v_x1.16b
eor v_x2.16b, v_x1.16b, v_x2.16b
dup d0, v_x2.d[1]
pmull v_x2.1q, v_x2.1d, v_p1_low.1d
pmull v0.1q, v0.1d, v_p1_high.1d
eor v_x2.16b, v0.16b, v_x2.16b
eor v_x3.16b, v_x2.16b, v_x3.16b
//v_x0 .req v2
//v_x3 .req v1
d_x0 .req d2
v_zero .req v3
// fold 64b
movi v_zero.4s, 0
mov x5, p0_high_b0
movk x5, p0_high_b1, lsl 16
mov x0, p0_low_b0
movk x0, p0_low_b1, lsl 16
dup d_x0, v_x3.d[1]
ext v0.16b, v_zero.16b, v_x3.16b, #8
fmov d16, x5
pmull v_x0.1q, v_x0.1d, v16.1d
fmov d17, x0
ext v0.16b, v0.16b, v_zero.16b, #4
eor v0.16b, v0.16b, v_x0.16b
dup d_x0, v0.d[1]
pmull v_x0.1q, v_x0.1d, v17.1d
// barrett reduction
d_br_low .req d16
d_br_high .req d17
v_br_low .req v16
v_br_high .req v17
mov x4, br_low_b0
movk x4, br_low_b1, lsl 16
movk x4, br_low_b2, lsl 32
mov x3, br_high_b0
movk x3, br_high_b1, lsl 16
movk x3, br_high_b2, lsl 32
fmov d_br_low, x4
eor v0.16b, v0.16b, v2.16b
umov x0, v0.d[0]
fmov d2, x0
ext v2.16b, v2.16b, v3.16b, #4
pmull v2.1q, v2.1d, v_br_low.1d
fmov d_br_high, x3
ext v2.16b, v2.16b, v3.16b, #4
pmull v2.1q, v2.1d, v_br_high.1d
eor v0.16b, v0.16b, v2.16b
umov x_seed, v0.d[0]
.clmul_end:
and w4, w2, -64
sxtw x3, w4
add x1, x1, x3
b .crc32_norm_tab_pre
.size \name, .-\name
.section .rodata.cst16,"aM",@progbits,16
.align 4
.shuffle:
.byte 15, 14, 13, 12, 11, 10, 9
.byte 8, 7, 6, 5, 4, 3, 2, 1, 0
.endm

280
crc/aarch64/crc32_refl_common_pmull.h Normal file
View File

@ -0,0 +1,280 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.macro crc32_refl_func name:req
.arch armv8-a+crc+crypto
.text
.align 3
.global \name
.type \name, %function
/* crc32_refl_func(uint32_t seed, uint8_t * buf, uint64_t len) */
// constant
.equ FOLD_SIZE, 1024
// paramter
w_seed .req w0
x_seed .req x0
x_buf .req x1
x_len .req x2
x_buf_tmp .req x0
// crc32 refl function entry
\name\():
mvn w_seed, w_seed
mov x3, 0
mov w4, 0
cmp x_len, (FOLD_SIZE - 1)
bhi .crc32_clmul_pre
.crc32_refl_tab_pre:
cmp x_len, x3
bls .done
sxtw x4, w4
adrp x5, .LANCHOR0
sub x_buf, x_buf, x4
add x5, x5, :lo12:.LANCHOR0
.align 3
.loop_crc32_refl_tab:
ldrb w3, [x_buf, x4]
add x4, x4, 1
cmp x_len, x4
eor x3, x_seed, x3
and x3, x3, 255
ldr w3, [x5, x3, lsl 2]
eor w_seed, w3, w_seed, lsr 8
bhi .loop_crc32_refl_tab
.done:
mvn w_seed, w_seed
ret
d_y0_tmp .req d0
v_y0_tmp .req v0
q_x0_tmp .req q3
v_x0_tmp .req v3
v_x0 .req v0
q_x1 .req q2
q_x2 .req q4
q_x3 .req q1
d_p4_low .req d17
d_p4_high .req d19
x_buf_end .req x3
.align 2
.crc32_clmul_pre:
and x4, x_len, -64
uxtw x_seed, w_seed
cmp x4, 63
bls .clmul_end
fmov d_y0_tmp, x_seed
ins v_y0_tmp.d[1], x3
ldr q_x0_tmp, [x_buf]
ldr q_x1, [x_buf, 16]
ldr q_x2, [x_buf, 32]
ldr q_x3, [x_buf, 48]
eor v_x0.16b, v_y0_tmp.16b, v_x0_tmp.16b
sub x5, x4, #64
cmp x5, 63
add x_buf_tmp, x_buf, 64
bls .clmul_loop_end
mov x4, p4_high_b0
movk x4, p4_high_b1, lsl 16
fmov d_p4_high, x4
mov x4, p4_low_b0
movk x4, p4_low_b1, lsl 16
fmov d_p4_low, x4
add x_buf_end, x_buf_tmp, x5
v_p4_low .req v17
v_p4_high .req v19
// v_x0 .req v0
v_x1 .req v2
v_x2 .req v4
v_x3 .req v1
q_y0 .req q7
q_y1 .req q5
q_y2 .req q3
q_y3 .req q21
v_y0 .req v7
v_y1 .req v5
v_y2 .req v3
v_y3 .req v21
d_x0_h .req d22
d_x1_h .req d20
d_x2_h .req d18
d_x3_h .req d6
v_x0_h .req v22
v_x1_h .req v20
v_x2_h .req v18
v_x3_h .req v6
.align 3
.clmul_loop:
add x_buf_tmp, x_buf_tmp, 64
cmp x_buf_tmp, x_buf_end
dup d_x0_h, v_x0.d[1]
dup d_x1_h, v_x1.d[1]
dup d_x2_h, v_x2.d[1]
dup d_x3_h, v_x3.d[1]
ldr q_y0, [x_buf_tmp, -64]
ldr q_y1, [x_buf_tmp, -48]
ldr q_y2, [x_buf_tmp, -32]
ldr q_y3, [x_buf_tmp, -16]
pmull v_x0.1q, v_x0.1d, v_p4_low.1d
pmull v_x1.1q, v_x1.1d, v_p4_low.1d
pmull v_x2.1q, v_x2.1d, v_p4_low.1d
pmull v_x3.1q, v_x3.1d, v_p4_low.1d
pmull v_x0_h.1q, v_x0_h.1d, v_p4_high.1d
pmull v_x1_h.1q, v_x1_h.1d, v_p4_high.1d
pmull v_x2_h.1q, v_x2_h.1d, v_p4_high.1d
pmull v_x3_h.1q, v_x3_h.1d, v_p4_high.1d
eor v_y0.16b, v_y0.16b, v22.16b
eor v_y1.16b, v_y1.16b, v20.16b
eor v_y2.16b, v_y2.16b, v18.16b
eor v_y3.16b, v_y3.16b, v6.16b
eor v_x0.16b, v_y0.16b, v_x0.16b
eor v_x1.16b, v_y1.16b, v_x1.16b
eor v_x2.16b, v_y2.16b, v_x2.16b
eor v_x3.16b, v_y3.16b, v_x3.16b
bne .clmul_loop
// v_x0 .req v0
// v_x1 .req v2
// v_x2 .req v4
// v_x3 .req v1
d_x0 .req d0
d_p1_high .req d7
d_p1_low .req d17
v_p1_high .req v7
v_p1_low .req v17
.clmul_loop_end:
// fold 128b
mov x0, p1_high_b0
movk x0, p1_high_b1, lsl 16
fmov d_p1_high, x0
mov x0, p1_low_b0
movk x0, p1_low_b1, lsl 16
fmov d_p1_low, x0
dup d6, v_x0.d[1]
pmull v_x0.1q, v_x0.1d, v_p1_low.1d
pmull v6.1q, v6.1d, v_p1_high.1d
eor v6.16b, v6.16b, v_x0.16b
eor v_x1.16b, v6.16b, v_x1.16b
dup d6, v_x1.d[1]
pmull v_x1.1q, v_x1.1d, v_p1_low.1d
pmull v6.1q, v6.1d, v_p1_high.1d
eor v6.16b, v6.16b, v_x1.16b
eor v_x2.16b, v6.16b, v_x2.16b
dup d_x0, v_x2.d[1] // d_x0 temparory saved v_x2 high
pmull v_x2.1q, v_x2.1d, v_p1_low.1d
pmull v_x0.1q, v_x0.1d, v_p1_high.1d
eor v_x0.16b, v_x0.16b, v_x2.16b
eor v_x0.16b, v_x0.16b, v_x3.16b
// all
mov x0, 4294967295
fmov d3, x0
movi v5.4s, 0
// fold 64b
mov x4, p0_low_b0
movk x4, p0_low_b1, lsl 16
fmov d1, x4
dup d2, v0.d[0]
ext v0.16b, v0.16b, v5.16b, #8
pmull v2.1q, v2.1d, v7.1d
eor v0.16b, v0.16b, v2.16b
and v2.16b, v3.16b, v0.16b
ext v0.16b, v0.16b, v5.16b, #4
pmull v2.1q, v2.1d, v1.1d
// barrett reduction
mov x3, br_high_b0
movk x3, br_high_b1, lsl 16
movk x3, br_high_b2, lsl 32
fmov d1, x3
eor v0.16b, v0.16b, v2.16b
and v2.16b, v0.16b, v3.16b
pmull v2.1q, v2.1d, v1.1d
mov x0, br_low_b0
movk x0, br_low_b1, lsl 16
movk x0, br_low_b2, lsl 32
fmov d1, x0
and v2.16b, v3.16b, v2.16b
pmull v2.1q, v2.1d, v1.1d
eor v0.16b, v0.16b, v2.16b
umov w_seed, v0.s[1]
uxtw x_seed, w_seed
.clmul_end:
and w4, w2, -64
sxtw x3, w4
add x_buf, x_buf, x3
b .crc32_refl_tab_pre
.size \name, .-\name
.endm

33
crc/aarch64/crc64_ecma_norm_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_ecma_norm_pmull.h"
#include "crc64_norm_common_pmull.h"
crc64_norm_func crc64_ecma_norm_pmull

200
crc/aarch64/crc64_ecma_norm_pmull.h Normal file
View File

@ -0,0 +1,200 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, (0xf020)
.equ p4_low_b1, 0x540d
.equ p4_low_b2, 0x43ca
.equ p4_low_b3, 0x5f68
.equ p4_high_b0, 0xb83f
.equ p4_high_b1, 0x1205
.equ p4_high_b2, 0xb698
.equ p4_high_b3, 0xddf4
.equ p1_low_b0, (0xfab6)
.equ p1_low_b1, 0xeb52
.equ p1_low_b2, 0xc3c7
.equ p1_low_b3, 0x05f5
.equ p1_high_b0, 0x740e
.equ p1_high_b1, 0xd257
.equ p1_high_b2, 0x38a7
.equ p1_high_b3, 0x4eb9
.equ p0_low_b0, (0xfab6)
.equ p0_low_b1, 0xeb52
.equ p0_low_b2, 0xc3c7
.equ p0_low_b3, 0x05f5
.equ p0_high_b0, 0x0
.equ p0_high_b1, 0x0
.equ p0_high_b2, 0x0
.equ p0_high_b3, 0x0
.equ br_low_b0, (0xf872)
.equ br_low_b1, 0x6cc4
.equ br_low_b2, 0x29d0
.equ br_low_b3, 0x578d
.equ br_high_b0, 0x3693
.equ br_high_b1, 0xa9ea
.equ br_high_b2, 0xe1eb
.equ br_high_b3, 0x42f0
.text
.section .rodata
.align 4
.set lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0x42f0e1eba9ea3693
.xword 0x85e1c3d753d46d26, 0xc711223cfa3e5bb5
.xword 0x493366450e42ecdf, 0x0bc387aea7a8da4c
.xword 0xccd2a5925d9681f9, 0x8e224479f47cb76a
.xword 0x9266cc8a1c85d9be, 0xd0962d61b56fef2d
.xword 0x17870f5d4f51b498, 0x5577eeb6e6bb820b
.xword 0xdb55aacf12c73561, 0x99a54b24bb2d03f2
.xword 0x5eb4691841135847, 0x1c4488f3e8f96ed4
.xword 0x663d78ff90e185ef, 0x24cd9914390bb37c
.xword 0xe3dcbb28c335e8c9, 0xa12c5ac36adfde5a
.xword 0x2f0e1eba9ea36930, 0x6dfeff5137495fa3
.xword 0xaaefdd6dcd770416, 0xe81f3c86649d3285
.xword 0xf45bb4758c645c51, 0xb6ab559e258e6ac2
.xword 0x71ba77a2dfb03177, 0x334a9649765a07e4
.xword 0xbd68d2308226b08e, 0xff9833db2bcc861d
.xword 0x388911e7d1f2dda8, 0x7a79f00c7818eb3b
.xword 0xcc7af1ff21c30bde, 0x8e8a101488293d4d
.xword 0x499b3228721766f8, 0x0b6bd3c3dbfd506b
.xword 0x854997ba2f81e701, 0xc7b97651866bd192
.xword 0x00a8546d7c558a27, 0x4258b586d5bfbcb4
.xword 0x5e1c3d753d46d260, 0x1cecdc9e94ace4f3
.xword 0xdbfdfea26e92bf46, 0x990d1f49c77889d5
.xword 0x172f5b3033043ebf, 0x55dfbadb9aee082c
.xword 0x92ce98e760d05399, 0xd03e790cc93a650a
.xword 0xaa478900b1228e31, 0xe8b768eb18c8b8a2
.xword 0x2fa64ad7e2f6e317, 0x6d56ab3c4b1cd584
.xword 0xe374ef45bf6062ee, 0xa1840eae168a547d
.xword 0x66952c92ecb40fc8, 0x2465cd79455e395b
.xword 0x3821458aada7578f, 0x7ad1a461044d611c
.xword 0xbdc0865dfe733aa9, 0xff3067b657990c3a
.xword 0x711223cfa3e5bb50, 0x33e2c2240a0f8dc3
.xword 0xf4f3e018f031d676, 0xb60301f359dbe0e5
.xword 0xda050215ea6c212f, 0x98f5e3fe438617bc
.xword 0x5fe4c1c2b9b84c09, 0x1d14202910527a9a
.xword 0x93366450e42ecdf0, 0xd1c685bb4dc4fb63
.xword 0x16d7a787b7faa0d6, 0x5427466c1e109645
.xword 0x4863ce9ff6e9f891, 0x0a932f745f03ce02
.xword 0xcd820d48a53d95b7, 0x8f72eca30cd7a324
.xword 0x0150a8daf8ab144e, 0x43a04931514122dd
.xword 0x84b16b0dab7f7968, 0xc6418ae602954ffb
.xword 0xbc387aea7a8da4c0, 0xfec89b01d3679253
.xword 0x39d9b93d2959c9e6, 0x7b2958d680b3ff75
.xword 0xf50b1caf74cf481f, 0xb7fbfd44dd257e8c
.xword 0x70eadf78271b2539, 0x321a3e938ef113aa
.xword 0x2e5eb66066087d7e, 0x6cae578bcfe24bed
.xword 0xabbf75b735dc1058, 0xe94f945c9c3626cb
.xword 0x676dd025684a91a1, 0x259d31cec1a0a732
.xword 0xe28c13f23b9efc87, 0xa07cf2199274ca14
.xword 0x167ff3eacbaf2af1, 0x548f120162451c62
.xword 0x939e303d987b47d7, 0xd16ed1d631917144
.xword 0x5f4c95afc5edc62e, 0x1dbc74446c07f0bd
.xword 0xdaad56789639ab08, 0x985db7933fd39d9b
.xword 0x84193f60d72af34f, 0xc6e9de8b7ec0c5dc
.xword 0x01f8fcb784fe9e69, 0x43081d5c2d14a8fa
.xword 0xcd2a5925d9681f90, 0x8fdab8ce70822903
.xword 0x48cb9af28abc72b6, 0x0a3b7b1923564425
.xword 0x70428b155b4eaf1e, 0x32b26afef2a4998d
.xword 0xf5a348c2089ac238, 0xb753a929a170f4ab
.xword 0x3971ed50550c43c1, 0x7b810cbbfce67552
.xword 0xbc902e8706d82ee7, 0xfe60cf6caf321874
.xword 0xe224479f47cb76a0, 0xa0d4a674ee214033
.xword 0x67c58448141f1b86, 0x253565a3bdf52d15
.xword 0xab1721da49899a7f, 0xe9e7c031e063acec
.xword 0x2ef6e20d1a5df759, 0x6c0603e6b3b7c1ca
.xword 0xf6fae5c07d3274cd, 0xb40a042bd4d8425e
.xword 0x731b26172ee619eb, 0x31ebc7fc870c2f78
.xword 0xbfc9838573709812, 0xfd39626eda9aae81
.xword 0x3a28405220a4f534, 0x78d8a1b9894ec3a7
.xword 0x649c294a61b7ad73, 0x266cc8a1c85d9be0
.xword 0xe17dea9d3263c055, 0xa38d0b769b89f6c6
.xword 0x2daf4f0f6ff541ac, 0x6f5faee4c61f773f
.xword 0xa84e8cd83c212c8a, 0xeabe6d3395cb1a19
.xword 0x90c79d3fedd3f122, 0xd2377cd44439c7b1
.xword 0x15265ee8be079c04, 0x57d6bf0317edaa97
.xword 0xd9f4fb7ae3911dfd, 0x9b041a914a7b2b6e
.xword 0x5c1538adb04570db, 0x1ee5d94619af4648
.xword 0x02a151b5f156289c, 0x4051b05e58bc1e0f
.xword 0x87409262a28245ba, 0xc5b073890b687329
.xword 0x4b9237f0ff14c443, 0x0962d61b56fef2d0
.xword 0xce73f427acc0a965, 0x8c8315cc052a9ff6
.xword 0x3a80143f5cf17f13, 0x7870f5d4f51b4980
.xword 0xbf61d7e80f251235, 0xfd913603a6cf24a6
.xword 0x73b3727a52b393cc, 0x31439391fb59a55f
.xword 0xf652b1ad0167feea, 0xb4a25046a88dc879
.xword 0xa8e6d8b54074a6ad, 0xea16395ee99e903e
.xword 0x2d071b6213a0cb8b, 0x6ff7fa89ba4afd18
.xword 0xe1d5bef04e364a72, 0xa3255f1be7dc7ce1
.xword 0x64347d271de22754, 0x26c49cccb40811c7
.xword 0x5cbd6cc0cc10fafc, 0x1e4d8d2b65facc6f
.xword 0xd95caf179fc497da, 0x9bac4efc362ea149
.xword 0x158e0a85c2521623, 0x577eeb6e6bb820b0
.xword 0x906fc95291867b05, 0xd29f28b9386c4d96
.xword 0xcedba04ad0952342, 0x8c2b41a1797f15d1
.xword 0x4b3a639d83414e64, 0x09ca82762aab78f7
.xword 0x87e8c60fded7cf9d, 0xc51827e4773df90e
.xword 0x020905d88d03a2bb, 0x40f9e43324e99428
.xword 0x2cffe7d5975e55e2, 0x6e0f063e3eb46371
.xword 0xa91e2402c48a38c4, 0xebeec5e96d600e57
.xword 0x65cc8190991cb93d, 0x273c607b30f68fae
.xword 0xe02d4247cac8d41b, 0xa2dda3ac6322e288
.xword 0xbe992b5f8bdb8c5c, 0xfc69cab42231bacf
.xword 0x3b78e888d80fe17a, 0x7988096371e5d7e9
.xword 0xf7aa4d1a85996083, 0xb55aacf12c735610
.xword 0x724b8ecdd64d0da5, 0x30bb6f267fa73b36
.xword 0x4ac29f2a07bfd00d, 0x08327ec1ae55e69e
.xword 0xcf235cfd546bbd2b, 0x8dd3bd16fd818bb8
.xword 0x03f1f96f09fd3cd2, 0x41011884a0170a41
.xword 0x86103ab85a2951f4, 0xc4e0db53f3c36767
.xword 0xd8a453a01b3a09b3, 0x9a54b24bb2d03f20
.xword 0x5d45907748ee6495, 0x1fb5719ce1045206
.xword 0x919735e51578e56c, 0xd367d40ebc92d3ff
.xword 0x1476f63246ac884a, 0x568617d9ef46bed9
.xword 0xe085162ab69d5e3c, 0xa275f7c11f7768af
.xword 0x6564d5fde549331a, 0x279434164ca30589
.xword 0xa9b6706fb8dfb2e3, 0xeb46918411358470
.xword 0x2c57b3b8eb0bdfc5, 0x6ea7525342e1e956
.xword 0x72e3daa0aa188782, 0x30133b4b03f2b111
.xword 0xf7021977f9cceaa4, 0xb5f2f89c5026dc37
.xword 0x3bd0bce5a45a6b5d, 0x79205d0e0db05dce
.xword 0xbe317f32f78e067b, 0xfcc19ed95e6430e8
.xword 0x86b86ed5267cdbd3, 0xc4488f3e8f96ed40
.xword 0x0359ad0275a8b6f5, 0x41a94ce9dc428066
.xword 0xcf8b0890283e370c, 0x8d7be97b81d4019f
.xword 0x4a6acb477bea5a2a, 0x089a2aacd2006cb9
.xword 0x14dea25f3af9026d, 0x562e43b4931334fe
.xword 0x913f6188692d6f4b, 0xd3cf8063c0c759d8
.xword 0x5dedc41a34bbeeb2, 0x1f1d25f19d51d821
.xword 0xd80c07cd676f8394, 0x9afce626ce85b507

33
crc/aarch64/crc64_ecma_refl_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_ecma_refl_pmull.h"
#include "crc64_refl_common_pmull.h"
crc64_refl_func crc64_ecma_refl_pmull

196
crc/aarch64/crc64_ecma_refl_pmull.h Normal file
View File

@ -0,0 +1,196 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0x41f3
.equ p4_low_b1, 0x9dd4
.equ p4_low_b2, 0xefbb
.equ p4_low_b3, 0x6ae3
.equ p4_high_b0, 0x2df4
.equ p4_high_b1, 0xa784
.equ p4_high_b2, 0x6054
.equ p4_high_b3, 0x081f
.equ p1_low_b0, 0x3ae4
.equ p1_low_b1, 0xca39
.equ p1_low_b2, 0xd497
.equ p1_low_b3, 0xe05d
.equ p1_high_b0, 0x5f40
.equ p1_high_b1, 0xc787
.equ p1_high_b2, 0x95af
.equ p1_high_b3, 0xdabe
.equ p0_low_b0, 0x5f40
.equ p0_low_b1, 0xc787
.equ p0_low_b2, 0x95af
.equ p0_low_b3, 0xdabe
.equ br_low_b0, 0x63d5
.equ br_low_b1, 0x1729
.equ br_low_b2, 0x466c
.equ br_low_b3, 0x9c3e
.equ br_high_b0, 0x1e85
.equ br_high_b1, 0xaf0e
.equ br_high_b2, 0xaf2b
.equ br_high_b3, 0x92d8
.text
.section .rodata
.align 4
.set .lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0xb32e4cbe03a75f6f
.xword 0xf4843657a840a05b, 0x47aa7ae9abe7ff34
.xword 0x7bd0c384ff8f5e33, 0xc8fe8f3afc28015c
.xword 0x8f54f5d357cffe68, 0x3c7ab96d5468a107
.xword 0xf7a18709ff1ebc66, 0x448fcbb7fcb9e309
.xword 0x0325b15e575e1c3d, 0xb00bfde054f94352
.xword 0x8c71448d0091e255, 0x3f5f08330336bd3a
.xword 0x78f572daa8d1420e, 0xcbdb3e64ab761d61
.xword 0x7d9ba13851336649, 0xceb5ed8652943926
.xword 0x891f976ff973c612, 0x3a31dbd1fad4997d
.xword 0x064b62bcaebc387a, 0xb5652e02ad1b6715
.xword 0xf2cf54eb06fc9821, 0x41e11855055bc74e
.xword 0x8a3a2631ae2dda2f, 0x39146a8fad8a8540
.xword 0x7ebe1066066d7a74, 0xcd905cd805ca251b
.xword 0xf1eae5b551a2841c, 0x42c4a90b5205db73
.xword 0x056ed3e2f9e22447, 0xb6409f5cfa457b28
.xword 0xfb374270a266cc92, 0x48190ecea1c193fd
.xword 0x0fb374270a266cc9, 0xbc9d3899098133a6
.xword 0x80e781f45de992a1, 0x33c9cd4a5e4ecdce
.xword 0x7463b7a3f5a932fa, 0xc74dfb1df60e6d95
.xword 0x0c96c5795d7870f4, 0xbfb889c75edf2f9b
.xword 0xf812f32ef538d0af, 0x4b3cbf90f69f8fc0
.xword 0x774606fda2f72ec7, 0xc4684a43a15071a8
.xword 0x83c230aa0ab78e9c, 0x30ec7c140910d1f3
.xword 0x86ace348f355aadb, 0x3582aff6f0f2f5b4
.xword 0x7228d51f5b150a80, 0xc10699a158b255ef
.xword 0xfd7c20cc0cdaf4e8, 0x4e526c720f7dab87
.xword 0x09f8169ba49a54b3, 0xbad65a25a73d0bdc
.xword 0x710d64410c4b16bd, 0xc22328ff0fec49d2
.xword 0x85895216a40bb6e6, 0x36a71ea8a7ace989
.xword 0x0adda7c5f3c4488e, 0xb9f3eb7bf06317e1
.xword 0xfe5991925b84e8d5, 0x4d77dd2c5823b7ba
.xword 0x64b62bcaebc387a1, 0xd7986774e864d8ce
.xword 0x90321d9d438327fa, 0x231c512340247895
.xword 0x1f66e84e144cd992, 0xac48a4f017eb86fd
.xword 0xebe2de19bc0c79c9, 0x58cc92a7bfab26a6
.xword 0x9317acc314dd3bc7, 0x2039e07d177a64a8
.xword 0x67939a94bc9d9b9c, 0xd4bdd62abf3ac4f3
.xword 0xe8c76f47eb5265f4, 0x5be923f9e8f53a9b
.xword 0x1c4359104312c5af, 0xaf6d15ae40b59ac0
.xword 0x192d8af2baf0e1e8, 0xaa03c64cb957be87
.xword 0xeda9bca512b041b3, 0x5e87f01b11171edc
.xword 0x62fd4976457fbfdb, 0xd1d305c846d8e0b4
.xword 0x96797f21ed3f1f80, 0x2557339fee9840ef
.xword 0xee8c0dfb45ee5d8e, 0x5da24145464902e1
.xword 0x1a083bacedaefdd5, 0xa9267712ee09a2ba
.xword 0x955cce7fba6103bd, 0x267282c1b9c65cd2
.xword 0x61d8f8281221a3e6, 0xd2f6b4961186fc89
.xword 0x9f8169ba49a54b33, 0x2caf25044a02145c
.xword 0x6b055fede1e5eb68, 0xd82b1353e242b407
.xword 0xe451aa3eb62a1500, 0x577fe680b58d4a6f
.xword 0x10d59c691e6ab55b, 0xa3fbd0d71dcdea34
.xword 0x6820eeb3b6bbf755, 0xdb0ea20db51ca83a
.xword 0x9ca4d8e41efb570e, 0x2f8a945a1d5c0861
.xword 0x13f02d374934a966, 0xa0de61894a93f609
.xword 0xe7741b60e174093d, 0x545a57dee2d35652
.xword 0xe21ac88218962d7a, 0x5134843c1b317215
.xword 0x169efed5b0d68d21, 0xa5b0b26bb371d24e
.xword 0x99ca0b06e7197349, 0x2ae447b8e4be2c26
.xword 0x6d4e3d514f59d312, 0xde6071ef4cfe8c7d
.xword 0x15bb4f8be788911c, 0xa6950335e42fce73
.xword 0xe13f79dc4fc83147, 0x521135624c6f6e28
.xword 0x6e6b8c0f1807cf2f, 0xdd45c0b11ba09040
.xword 0x9aefba58b0476f74, 0x29c1f6e6b3e0301b
.xword 0xc96c5795d7870f42, 0x7a421b2bd420502d
.xword 0x3de861c27fc7af19, 0x8ec62d7c7c60f076
.xword 0xb2bc941128085171, 0x0192d8af2baf0e1e
.xword 0x4638a2468048f12a, 0xf516eef883efae45
.xword 0x3ecdd09c2899b324, 0x8de39c222b3eec4b
.xword 0xca49e6cb80d9137f, 0x7967aa75837e4c10
.xword 0x451d1318d716ed17, 0xf6335fa6d4b1b278
.xword 0xb199254f7f564d4c, 0x02b769f17cf11223
.xword 0xb4f7f6ad86b4690b, 0x07d9ba1385133664
.xword 0x4073c0fa2ef4c950, 0xf35d8c442d53963f
.xword 0xcf273529793b3738, 0x7c0979977a9c6857
.xword 0x3ba3037ed17b9763, 0x888d4fc0d2dcc80c
.xword 0x435671a479aad56d, 0xf0783d1a7a0d8a02
.xword 0xb7d247f3d1ea7536, 0x04fc0b4dd24d2a59
.xword 0x3886b22086258b5e, 0x8ba8fe9e8582d431
.xword 0xcc0284772e652b05, 0x7f2cc8c92dc2746a
.xword 0x325b15e575e1c3d0, 0x8175595b76469cbf
.xword 0xc6df23b2dda1638b, 0x75f16f0cde063ce4
.xword 0x498bd6618a6e9de3, 0xfaa59adf89c9c28c
.xword 0xbd0fe036222e3db8, 0x0e21ac88218962d7
.xword 0xc5fa92ec8aff7fb6, 0x76d4de52895820d9
.xword 0x317ea4bb22bfdfed, 0x8250e80521188082
.xword 0xbe2a516875702185, 0x0d041dd676d77eea
.xword 0x4aae673fdd3081de, 0xf9802b81de97deb1
.xword 0x4fc0b4dd24d2a599, 0xfceef8632775faf6
.xword 0xbb44828a8c9205c2, 0x086ace348f355aad
.xword 0x34107759db5dfbaa, 0x873e3be7d8faa4c5
.xword 0xc094410e731d5bf1, 0x73ba0db070ba049e
.xword 0xb86133d4dbcc19ff, 0x0b4f7f6ad86b4690
.xword 0x4ce50583738cb9a4, 0xffcb493d702be6cb
.xword 0xc3b1f050244347cc, 0x709fbcee27e418a3
.xword 0x3735c6078c03e797, 0x841b8ab98fa4b8f8
.xword 0xadda7c5f3c4488e3, 0x1ef430e13fe3d78c
.xword 0x595e4a08940428b8, 0xea7006b697a377d7
.xword 0xd60abfdbc3cbd6d0, 0x6524f365c06c89bf
.xword 0x228e898c6b8b768b, 0x91a0c532682c29e4
.xword 0x5a7bfb56c35a3485, 0xe955b7e8c0fd6bea
.xword 0xaeffcd016b1a94de, 0x1dd181bf68bdcbb1
.xword 0x21ab38d23cd56ab6, 0x9285746c3f7235d9
.xword 0xd52f0e859495caed, 0x6601423b97329582
.xword 0xd041dd676d77eeaa, 0x636f91d96ed0b1c5
.xword 0x24c5eb30c5374ef1, 0x97eba78ec690119e
.xword 0xab911ee392f8b099, 0x18bf525d915feff6
.xword 0x5f1528b43ab810c2, 0xec3b640a391f4fad
.xword 0x27e05a6e926952cc, 0x94ce16d091ce0da3
.xword 0xd3646c393a29f297, 0x604a2087398eadf8
.xword 0x5c3099ea6de60cff, 0xef1ed5546e415390
.xword 0xa8b4afbdc5a6aca4, 0x1b9ae303c601f3cb
.xword 0x56ed3e2f9e224471, 0xe5c372919d851b1e
.xword 0xa26908783662e42a, 0x114744c635c5bb45
.xword 0x2d3dfdab61ad1a42, 0x9e13b115620a452d
.xword 0xd9b9cbfcc9edba19, 0x6a978742ca4ae576
.xword 0xa14cb926613cf817, 0x1262f598629ba778
.xword 0x55c88f71c97c584c, 0xe6e6c3cfcadb0723
.xword 0xda9c7aa29eb3a624, 0x69b2361c9d14f94b
.xword 0x2e184cf536f3067f, 0x9d36004b35545910
.xword 0x2b769f17cf112238, 0x9858d3a9ccb67d57
.xword 0xdff2a94067518263, 0x6cdce5fe64f6dd0c
.xword 0x50a65c93309e7c0b, 0xe388102d33392364
.xword 0xa4226ac498dedc50, 0x170c267a9b79833f
.xword 0xdcd7181e300f9e5e, 0x6ff954a033a8c131
.xword 0x28532e49984f3e05, 0x9b7d62f79be8616a
.xword 0xa707db9acf80c06d, 0x14299724cc279f02
.xword 0x5383edcd67c06036, 0xe0ada17364673f59

33
crc/aarch64/crc64_iso_norm_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_iso_norm_pmull.h"
#include "crc64_norm_common_pmull.h"
crc64_norm_func crc64_iso_norm_pmull

201
crc/aarch64/crc64_iso_norm_pmull.h Normal file
View File

@ -0,0 +1,201 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, (0x0101)
.equ p4_low_b1, 0x0100
.equ p4_low_b2, 0x0001
.equ p4_low_b3, 0x0000
.equ p4_high_b0, 0x1b1b
.equ p4_high_b1, 0x1b00
.equ p4_high_b2, 0x001b
.equ p4_high_b3, 0x0000
.equ p1_low_b0, (0x0145)
.equ p1_low_b1, 0x0000
.equ p1_low_b2, 0x0000
.equ p1_low_b3, 0x0000
.equ p1_high_b0, 0x1db7
.equ p1_high_b1, 0x0000
.equ p1_high_b2, 0x0000
.equ p1_high_b3, 0x0000
.equ p0_low_b0, (0x0145)
.equ p0_low_b1, 0x0000
.equ p0_low_b2, 0x0000
.equ p0_low_b3, 0x0000
.equ p0_high_b0, 0x0000
.equ p0_high_b1, 0x0000
.equ p0_high_b2, 0x0000
.equ p0_high_b3, 0x0000
.equ br_low_b0, (0x001b)
.equ br_low_b1, 0x0000
.equ br_low_b2, 0x0000
.equ br_low_b3, 0x0000
.equ br_high_b0, 0x001b
.equ br_high_b1, 0x0000
.equ br_high_b2, 0x0000
.equ br_high_b3, 0x0000
.text
.section .rodata
.align 4
.set lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0x000000000000001b
.xword 0x0000000000000036, 0x000000000000002d
.xword 0x000000000000006c, 0x0000000000000077
.xword 0x000000000000005a, 0x0000000000000041
.xword 0x00000000000000d8, 0x00000000000000c3
.xword 0x00000000000000ee, 0x00000000000000f5
.xword 0x00000000000000b4, 0x00000000000000af
.xword 0x0000000000000082, 0x0000000000000099
.xword 0x00000000000001b0, 0x00000000000001ab
.xword 0x0000000000000186, 0x000000000000019d
.xword 0x00000000000001dc, 0x00000000000001c7
.xword 0x00000000000001ea, 0x00000000000001f1
.xword 0x0000000000000168, 0x0000000000000173
.xword 0x000000000000015e, 0x0000000000000145
.xword 0x0000000000000104, 0x000000000000011f
.xword 0x0000000000000132, 0x0000000000000129
.xword 0x0000000000000360, 0x000000000000037b
.xword 0x0000000000000356, 0x000000000000034d
.xword 0x000000000000030c, 0x0000000000000317
.xword 0x000000000000033a, 0x0000000000000321
.xword 0x00000000000003b8, 0x00000000000003a3
.xword 0x000000000000038e, 0x0000000000000395
.xword 0x00000000000003d4, 0x00000000000003cf
.xword 0x00000000000003e2, 0x00000000000003f9
.xword 0x00000000000002d0, 0x00000000000002cb
.xword 0x00000000000002e6, 0x00000000000002fd
.xword 0x00000000000002bc, 0x00000000000002a7
.xword 0x000000000000028a, 0x0000000000000291
.xword 0x0000000000000208, 0x0000000000000213
.xword 0x000000000000023e, 0x0000000000000225
.xword 0x0000000000000264, 0x000000000000027f
.xword 0x0000000000000252, 0x0000000000000249
.xword 0x00000000000006c0, 0x00000000000006db
.xword 0x00000000000006f6, 0x00000000000006ed
.xword 0x00000000000006ac, 0x00000000000006b7
.xword 0x000000000000069a, 0x0000000000000681
.xword 0x0000000000000618, 0x0000000000000603
.xword 0x000000000000062e, 0x0000000000000635
.xword 0x0000000000000674, 0x000000000000066f
.xword 0x0000000000000642, 0x0000000000000659
.xword 0x0000000000000770, 0x000000000000076b
.xword 0x0000000000000746, 0x000000000000075d
.xword 0x000000000000071c, 0x0000000000000707
.xword 0x000000000000072a, 0x0000000000000731
.xword 0x00000000000007a8, 0x00000000000007b3
.xword 0x000000000000079e, 0x0000000000000785
.xword 0x00000000000007c4, 0x00000000000007df
.xword 0x00000000000007f2, 0x00000000000007e9
.xword 0x00000000000005a0, 0x00000000000005bb
.xword 0x0000000000000596, 0x000000000000058d
.xword 0x00000000000005cc, 0x00000000000005d7
.xword 0x00000000000005fa, 0x00000000000005e1
.xword 0x0000000000000578, 0x0000000000000563
.xword 0x000000000000054e, 0x0000000000000555
.xword 0x0000000000000514, 0x000000000000050f
.xword 0x0000000000000522, 0x0000000000000539
.xword 0x0000000000000410, 0x000000000000040b
.xword 0x0000000000000426, 0x000000000000043d
.xword 0x000000000000047c, 0x0000000000000467
.xword 0x000000000000044a, 0x0000000000000451
.xword 0x00000000000004c8, 0x00000000000004d3
.xword 0x00000000000004fe, 0x00000000000004e5
.xword 0x00000000000004a4, 0x00000000000004bf
.xword 0x0000000000000492, 0x0000000000000489
.xword 0x0000000000000d80, 0x0000000000000d9b
.xword 0x0000000000000db6, 0x0000000000000dad
.xword 0x0000000000000dec, 0x0000000000000df7
.xword 0x0000000000000dda, 0x0000000000000dc1
.xword 0x0000000000000d58, 0x0000000000000d43
.xword 0x0000000000000d6e, 0x0000000000000d75
.xword 0x0000000000000d34, 0x0000000000000d2f
.xword 0x0000000000000d02, 0x0000000000000d19
.xword 0x0000000000000c30, 0x0000000000000c2b
.xword 0x0000000000000c06, 0x0000000000000c1d
.xword 0x0000000000000c5c, 0x0000000000000c47
.xword 0x0000000000000c6a, 0x0000000000000c71
.xword 0x0000000000000ce8, 0x0000000000000cf3
.xword 0x0000000000000cde, 0x0000000000000cc5
.xword 0x0000000000000c84, 0x0000000000000c9f
.xword 0x0000000000000cb2, 0x0000000000000ca9
.xword 0x0000000000000ee0, 0x0000000000000efb
.xword 0x0000000000000ed6, 0x0000000000000ecd
.xword 0x0000000000000e8c, 0x0000000000000e97
.xword 0x0000000000000eba, 0x0000000000000ea1
.xword 0x0000000000000e38, 0x0000000000000e23
.xword 0x0000000000000e0e, 0x0000000000000e15
.xword 0x0000000000000e54, 0x0000000000000e4f
.xword 0x0000000000000e62, 0x0000000000000e79
.xword 0x0000000000000f50, 0x0000000000000f4b
.xword 0x0000000000000f66, 0x0000000000000f7d
.xword 0x0000000000000f3c, 0x0000000000000f27
.xword 0x0000000000000f0a, 0x0000000000000f11
.xword 0x0000000000000f88, 0x0000000000000f93
.xword 0x0000000000000fbe, 0x0000000000000fa5
.xword 0x0000000000000fe4, 0x0000000000000fff
.xword 0x0000000000000fd2, 0x0000000000000fc9
.xword 0x0000000000000b40, 0x0000000000000b5b
.xword 0x0000000000000b76, 0x0000000000000b6d
.xword 0x0000000000000b2c, 0x0000000000000b37
.xword 0x0000000000000b1a, 0x0000000000000b01
.xword 0x0000000000000b98, 0x0000000000000b83
.xword 0x0000000000000bae, 0x0000000000000bb5
.xword 0x0000000000000bf4, 0x0000000000000bef
.xword 0x0000000000000bc2, 0x0000000000000bd9
.xword 0x0000000000000af0, 0x0000000000000aeb
.xword 0x0000000000000ac6, 0x0000000000000add
.xword 0x0000000000000a9c, 0x0000000000000a87
.xword 0x0000000000000aaa, 0x0000000000000ab1
.xword 0x0000000000000a28, 0x0000000000000a33
.xword 0x0000000000000a1e, 0x0000000000000a05
.xword 0x0000000000000a44, 0x0000000000000a5f
.xword 0x0000000000000a72, 0x0000000000000a69
.xword 0x0000000000000820, 0x000000000000083b
.xword 0x0000000000000816, 0x000000000000080d
.xword 0x000000000000084c, 0x0000000000000857
.xword 0x000000000000087a, 0x0000000000000861
.xword 0x00000000000008f8, 0x00000000000008e3
.xword 0x00000000000008ce, 0x00000000000008d5
.xword 0x0000000000000894, 0x000000000000088f
.xword 0x00000000000008a2, 0x00000000000008b9
.xword 0x0000000000000990, 0x000000000000098b
.xword 0x00000000000009a6, 0x00000000000009bd
.xword 0x00000000000009fc, 0x00000000000009e7
.xword 0x00000000000009ca, 0x00000000000009d1
.xword 0x0000000000000948, 0x0000000000000953
.xword 0x000000000000097e, 0x0000000000000965
.xword 0x0000000000000924, 0x000000000000093f
.xword 0x0000000000000912, 0x0000000000000909

33
crc/aarch64/crc64_iso_refl_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_iso_refl_pmull.h"
#include "crc64_refl_common_pmull.h"
crc64_refl_func crc64_iso_refl_pmull

197
crc/aarch64/crc64_iso_refl_pmull.h Normal file
View File

@ -0,0 +1,197 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0x0001
.equ p4_low_b1, 0xb000
.equ p4_low_b2, 0x01b1
.equ p4_low_b3, 0x01b0
.equ p4_high_b0, 0x0001
.equ p4_high_b1, 0x0000
.equ p4_high_b2, 0x0101
.equ p4_high_b3, 0xb100
.equ p1_low_b0, 0x0001
.equ p1_low_b1, 0x0000
.equ p1_low_b2, 0x0000
.equ p1_low_b3, 0x6b70
.equ p1_high_b0, 0x0001
.equ p1_high_b1, 0x0000
.equ p1_high_b2, 0x0000
.equ p1_high_b3, 0xf500
.equ p0_low_b0, 0x0001
.equ p0_low_b1, 0x0000
.equ p0_low_b2, 0x0000
.equ p0_low_b3, 0xf500
.equ br_low_b0, 0x0001
.equ br_low_b1, 0x0000
.equ br_low_b2, 0x0000
.equ br_low_b3, 0xb000
.equ br_high_b0, 0x0001
.equ br_high_b1, 0x0000
.equ br_high_b2, 0x0000
.equ br_high_b3, 0xb000
.text
.section .rodata
.align 4
.set .lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0x01b0000000000000
.xword 0x0360000000000000, 0x02d0000000000000
.xword 0x06c0000000000000, 0x0770000000000000
.xword 0x05a0000000000000, 0x0410000000000000
.xword 0x0d80000000000000, 0x0c30000000000000
.xword 0x0ee0000000000000, 0x0f50000000000000
.xword 0x0b40000000000000, 0x0af0000000000000
.xword 0x0820000000000000, 0x0990000000000000
.xword 0x1b00000000000000, 0x1ab0000000000000
.xword 0x1860000000000000, 0x19d0000000000000
.xword 0x1dc0000000000000, 0x1c70000000000000
.xword 0x1ea0000000000000, 0x1f10000000000000
.xword 0x1680000000000000, 0x1730000000000000
.xword 0x15e0000000000000, 0x1450000000000000
.xword 0x1040000000000000, 0x11f0000000000000
.xword 0x1320000000000000, 0x1290000000000000
.xword 0x3600000000000000, 0x37b0000000000000
.xword 0x3560000000000000, 0x34d0000000000000
.xword 0x30c0000000000000, 0x3170000000000000
.xword 0x33a0000000000000, 0x3210000000000000
.xword 0x3b80000000000000, 0x3a30000000000000
.xword 0x38e0000000000000, 0x3950000000000000
.xword 0x3d40000000000000, 0x3cf0000000000000
.xword 0x3e20000000000000, 0x3f90000000000000
.xword 0x2d00000000000000, 0x2cb0000000000000
.xword 0x2e60000000000000, 0x2fd0000000000000
.xword 0x2bc0000000000000, 0x2a70000000000000
.xword 0x28a0000000000000, 0x2910000000000000
.xword 0x2080000000000000, 0x2130000000000000
.xword 0x23e0000000000000, 0x2250000000000000
.xword 0x2640000000000000, 0x27f0000000000000
.xword 0x2520000000000000, 0x2490000000000000
.xword 0x6c00000000000000, 0x6db0000000000000
.xword 0x6f60000000000000, 0x6ed0000000000000
.xword 0x6ac0000000000000, 0x6b70000000000000
.xword 0x69a0000000000000, 0x6810000000000000
.xword 0x6180000000000000, 0x6030000000000000
.xword 0x62e0000000000000, 0x6350000000000000
.xword 0x6740000000000000, 0x66f0000000000000
.xword 0x6420000000000000, 0x6590000000000000
.xword 0x7700000000000000, 0x76b0000000000000
.xword 0x7460000000000000, 0x75d0000000000000
.xword 0x71c0000000000000, 0x7070000000000000
.xword 0x72a0000000000000, 0x7310000000000000
.xword 0x7a80000000000000, 0x7b30000000000000
.xword 0x79e0000000000000, 0x7850000000000000
.xword 0x7c40000000000000, 0x7df0000000000000
.xword 0x7f20000000000000, 0x7e90000000000000
.xword 0x5a00000000000000, 0x5bb0000000000000
.xword 0x5960000000000000, 0x58d0000000000000
.xword 0x5cc0000000000000, 0x5d70000000000000
.xword 0x5fa0000000000000, 0x5e10000000000000
.xword 0x5780000000000000, 0x5630000000000000
.xword 0x54e0000000000000, 0x5550000000000000
.xword 0x5140000000000000, 0x50f0000000000000
.xword 0x5220000000000000, 0x5390000000000000
.xword 0x4100000000000000, 0x40b0000000000000
.xword 0x4260000000000000, 0x43d0000000000000
.xword 0x47c0000000000000, 0x4670000000000000
.xword 0x44a0000000000000, 0x4510000000000000
.xword 0x4c80000000000000, 0x4d30000000000000
.xword 0x4fe0000000000000, 0x4e50000000000000
.xword 0x4a40000000000000, 0x4bf0000000000000
.xword 0x4920000000000000, 0x4890000000000000
.xword 0xd800000000000000, 0xd9b0000000000000
.xword 0xdb60000000000000, 0xdad0000000000000
.xword 0xdec0000000000000, 0xdf70000000000000
.xword 0xdda0000000000000, 0xdc10000000000000
.xword 0xd580000000000000, 0xd430000000000000
.xword 0xd6e0000000000000, 0xd750000000000000
.xword 0xd340000000000000, 0xd2f0000000000000
.xword 0xd020000000000000, 0xd190000000000000
.xword 0xc300000000000000, 0xc2b0000000000000
.xword 0xc060000000000000, 0xc1d0000000000000
.xword 0xc5c0000000000000, 0xc470000000000000
.xword 0xc6a0000000000000, 0xc710000000000000
.xword 0xce80000000000000, 0xcf30000000000000
.xword 0xcde0000000000000, 0xcc50000000000000
.xword 0xc840000000000000, 0xc9f0000000000000
.xword 0xcb20000000000000, 0xca90000000000000
.xword 0xee00000000000000, 0xefb0000000000000
.xword 0xed60000000000000, 0xecd0000000000000
.xword 0xe8c0000000000000, 0xe970000000000000
.xword 0xeba0000000000000, 0xea10000000000000
.xword 0xe380000000000000, 0xe230000000000000
.xword 0xe0e0000000000000, 0xe150000000000000
.xword 0xe540000000000000, 0xe4f0000000000000
.xword 0xe620000000000000, 0xe790000000000000
.xword 0xf500000000000000, 0xf4b0000000000000
.xword 0xf660000000000000, 0xf7d0000000000000
.xword 0xf3c0000000000000, 0xf270000000000000
.xword 0xf0a0000000000000, 0xf110000000000000
.xword 0xf880000000000000, 0xf930000000000000
.xword 0xfbe0000000000000, 0xfa50000000000000
.xword 0xfe40000000000000, 0xfff0000000000000
.xword 0xfd20000000000000, 0xfc90000000000000
.xword 0xb400000000000000, 0xb5b0000000000000
.xword 0xb760000000000000, 0xb6d0000000000000
.xword 0xb2c0000000000000, 0xb370000000000000
.xword 0xb1a0000000000000, 0xb010000000000000
.xword 0xb980000000000000, 0xb830000000000000
.xword 0xbae0000000000000, 0xbb50000000000000
.xword 0xbf40000000000000, 0xbef0000000000000
.xword 0xbc20000000000000, 0xbd90000000000000
.xword 0xaf00000000000000, 0xaeb0000000000000
.xword 0xac60000000000000, 0xadd0000000000000
.xword 0xa9c0000000000000, 0xa870000000000000
.xword 0xaaa0000000000000, 0xab10000000000000
.xword 0xa280000000000000, 0xa330000000000000
.xword 0xa1e0000000000000, 0xa050000000000000
.xword 0xa440000000000000, 0xa5f0000000000000
.xword 0xa720000000000000, 0xa690000000000000
.xword 0x8200000000000000, 0x83b0000000000000
.xword 0x8160000000000000, 0x80d0000000000000
.xword 0x84c0000000000000, 0x8570000000000000
.xword 0x87a0000000000000, 0x8610000000000000
.xword 0x8f80000000000000, 0x8e30000000000000
.xword 0x8ce0000000000000, 0x8d50000000000000
.xword 0x8940000000000000, 0x88f0000000000000
.xword 0x8a20000000000000, 0x8b90000000000000
.xword 0x9900000000000000, 0x98b0000000000000
.xword 0x9a60000000000000, 0x9bd0000000000000
.xword 0x9fc0000000000000, 0x9e70000000000000
.xword 0x9ca0000000000000, 0x9d10000000000000
.xword 0x9480000000000000, 0x9530000000000000
.xword 0x97e0000000000000, 0x9650000000000000
.xword 0x9240000000000000, 0x93f0000000000000
.xword 0x9120000000000000, 0x9090000000000000

33
crc/aarch64/crc64_jones_norm_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_jones_norm_pmull.h"
#include "crc64_norm_common_pmull.h"
crc64_norm_func crc64_jones_norm_pmull

200
crc/aarch64/crc64_jones_norm_pmull.h Normal file
View File

@ -0,0 +1,200 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, (0xd25e)
.equ p4_low_b1, 0xca43
.equ p4_low_b2, 0x1e58
.equ p4_low_b3, 0x4e50
.equ p4_high_b0, 0xf643
.equ p4_high_b1, 0x8f27
.equ p4_high_b2, 0x6158
.equ p4_high_b3, 0x13c9
.equ p1_low_b0, (0x7038)
.equ p1_low_b1, 0x5001
.equ p1_low_b2, 0xed27
.equ p1_low_b3, 0x4445
.equ p1_high_b0, 0xd736
.equ p1_high_b1, 0x7cfb
.equ p1_high_b2, 0x7415
.equ p1_high_b3, 0x698b
.equ p0_low_b0, (0x7038)
.equ p0_low_b1, 0x5001
.equ p0_low_b2, 0xed27
.equ p0_low_b3, 0x4445
.equ p0_high_b0, 0x0000
.equ p0_high_b1, 0x0000
.equ p0_high_b2, 0x0000
.equ p0_high_b3, 0x0000
.equ br_low_b0, (0x6cf8)
.equ br_low_b1, 0x98be
.equ br_low_b2, 0xeeb2
.equ br_low_b3, 0xddf3
.equ br_high_b0, 0x35a9
.equ br_high_b1, 0x94c9
.equ br_high_b2, 0xd235
.equ br_high_b3, 0xad93
.text
.section .rodata
.align 4
.set lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0xad93d23594c935a9
.xword 0xf6b4765ebd5b5efb, 0x5b27a46b29926b52
.xword 0x40fb3e88ee7f885f, 0xed68ecbd7ab6bdf6
.xword 0xb64f48d65324d6a4, 0x1bdc9ae3c7ede30d
.xword 0x81f67d11dcff10be, 0x2c65af2448362517
.xword 0x77420b4f61a44e45, 0xdad1d97af56d7bec
.xword 0xc10d4399328098e1, 0x6c9e91aca649ad48
.xword 0x37b935c78fdbc61a, 0x9a2ae7f21b12f3b3
.xword 0xae7f28162d3714d5, 0x03ecfa23b9fe217c
.xword 0x58cb5e48906c4a2e, 0xf5588c7d04a57f87
.xword 0xee84169ec3489c8a, 0x4317c4ab5781a923
.xword 0x183060c07e13c271, 0xb5a3b2f5eadaf7d8
.xword 0x2f895507f1c8046b, 0x821a8732650131c2
.xword 0xd93d23594c935a90, 0x74aef16cd85a6f39
.xword 0x6f726b8f1fb78c34, 0xc2e1b9ba8b7eb99d
.xword 0x99c61dd1a2ecd2cf, 0x3455cfe43625e766
.xword 0xf16d8219cea71c03, 0x5cfe502c5a6e29aa
.xword 0x07d9f44773fc42f8, 0xaa4a2672e7357751
.xword 0xb196bc9120d8945c, 0x1c056ea4b411a1f5
.xword 0x4722cacf9d83caa7, 0xeab118fa094aff0e
.xword 0x709bff0812580cbd, 0xdd082d3d86913914
.xword 0x862f8956af035246, 0x2bbc5b633bca67ef
.xword 0x3060c180fc2784e2, 0x9df313b568eeb14b
.xword 0xc6d4b7de417cda19, 0x6b4765ebd5b5efb0
.xword 0x5f12aa0fe39008d6, 0xf281783a77593d7f
.xword 0xa9a6dc515ecb562d, 0x04350e64ca026384
.xword 0x1fe994870def8089, 0xb27a46b29926b520
.xword 0xe95de2d9b0b4de72, 0x44ce30ec247debdb
.xword 0xdee4d71e3f6f1868, 0x7377052baba62dc1
.xword 0x2850a14082344693, 0x85c3737516fd733a
.xword 0x9e1fe996d1109037, 0x338c3ba345d9a59e
.xword 0x68ab9fc86c4bcecc, 0xc5384dfdf882fb65
.xword 0x4f48d60609870daf, 0xe2db04339d4e3806
.xword 0xb9fca058b4dc5354, 0x146f726d201566fd
.xword 0x0fb3e88ee7f885f0, 0xa2203abb7331b059
.xword 0xf9079ed05aa3db0b, 0x54944ce5ce6aeea2
.xword 0xcebeab17d5781d11, 0x632d792241b128b8
.xword 0x380add49682343ea, 0x95990f7cfcea7643
.xword 0x8e45959f3b07954e, 0x23d647aaafcea0e7
.xword 0x78f1e3c1865ccbb5, 0xd56231f41295fe1c
.xword 0xe137fe1024b0197a, 0x4ca42c25b0792cd3
.xword 0x1783884e99eb4781, 0xba105a7b0d227228
.xword 0xa1ccc098cacf9125, 0x0c5f12ad5e06a48c
.xword 0x5778b6c67794cfde, 0xfaeb64f3e35dfa77
.xword 0x60c18301f84f09c4, 0xcd5251346c863c6d
.xword 0x9675f55f4514573f, 0x3be6276ad1dd6296
.xword 0x203abd891630819b, 0x8da96fbc82f9b432
.xword 0xd68ecbd7ab6bdf60, 0x7b1d19e23fa2eac9
.xword 0xbe25541fc72011ac, 0x13b6862a53e92405
.xword 0x489122417a7b4f57, 0xe502f074eeb27afe
.xword 0xfede6a97295f99f3, 0x534db8a2bd96ac5a
.xword 0x086a1cc99404c708, 0xa5f9cefc00cdf2a1
.xword 0x3fd3290e1bdf0112, 0x9240fb3b8f1634bb
.xword 0xc9675f50a6845fe9, 0x64f48d65324d6a40
.xword 0x7f281786f5a0894d, 0xd2bbc5b36169bce4
.xword 0x899c61d848fbd7b6, 0x240fb3eddc32e21f
.xword 0x105a7c09ea170579, 0xbdc9ae3c7ede30d0
.xword 0xe6ee0a57574c5b82, 0x4b7dd862c3856e2b
.xword 0x50a1428104688d26, 0xfd3290b490a1b88f
.xword 0xa61534dfb933d3dd, 0x0b86e6ea2dfae674
.xword 0x91ac011836e815c7, 0x3c3fd32da221206e
.xword 0x671877468bb34b3c, 0xca8ba5731f7a7e95
.xword 0xd1573f90d8979d98, 0x7cc4eda54c5ea831
.xword 0x27e349ce65ccc363, 0x8a709bfbf105f6ca
.xword 0x9e91ac0c130e1b5e, 0x33027e3987c72ef7
.xword 0x6825da52ae5545a5, 0xc5b608673a9c700c
.xword 0xde6a9284fd719301, 0x73f940b169b8a6a8
.xword 0x28dee4da402acdfa, 0x854d36efd4e3f853
.xword 0x1f67d11dcff10be0, 0xb2f403285b383e49
.xword 0xe9d3a74372aa551b, 0x44407576e66360b2
.xword 0x5f9cef95218e83bf, 0xf20f3da0b547b616
.xword 0xa92899cb9cd5dd44, 0x04bb4bfe081ce8ed
.xword 0x30ee841a3e390f8b, 0x9d7d562faaf03a22
.xword 0xc65af24483625170, 0x6bc9207117ab64d9
.xword 0x7015ba92d04687d4, 0xdd8668a7448fb27d
.xword 0x86a1cccc6d1dd92f, 0x2b321ef9f9d4ec86
.xword 0xb118f90be2c61f35, 0x1c8b2b3e760f2a9c
.xword 0x47ac8f555f9d41ce, 0xea3f5d60cb547467
.xword 0xf1e3c7830cb9976a, 0x5c7015b69870a2c3
.xword 0x0757b1ddb1e2c991, 0xaac463e8252bfc38
.xword 0x6ffc2e15dda9075d, 0xc26ffc20496032f4
.xword 0x9948584b60f259a6, 0x34db8a7ef43b6c0f
.xword 0x2f07109d33d68f02, 0x8294c2a8a71fbaab
.xword 0xd9b366c38e8dd1f9, 0x7420b4f61a44e450
.xword 0xee0a5304015617e3, 0x43998131959f224a
.xword 0x18be255abc0d4918, 0xb52df76f28c47cb1
.xword 0xaef16d8cef299fbc, 0x0362bfb97be0aa15
.xword 0x58451bd25272c147, 0xf5d6c9e7c6bbf4ee
.xword 0xc1830603f09e1388, 0x6c10d43664572621
.xword 0x3737705d4dc54d73, 0x9aa4a268d90c78da
.xword 0x8178388b1ee19bd7, 0x2cebeabe8a28ae7e
.xword 0x77cc4ed5a3bac52c, 0xda5f9ce03773f085
.xword 0x40757b122c610336, 0xede6a927b8a8369f
.xword 0xb6c10d4c913a5dcd, 0x1b52df7905f36864
.xword 0x008e459ac21e8b69, 0xad1d97af56d7bec0
.xword 0xf63a33c47f45d592, 0x5ba9e1f1eb8ce03b
.xword 0xd1d97a0a1a8916f1, 0x7c4aa83f8e402358
.xword 0x276d0c54a7d2480a, 0x8afede61331b7da3
.xword 0x91224482f4f69eae, 0x3cb196b7603fab07
.xword 0x679632dc49adc055, 0xca05e0e9dd64f5fc
.xword 0x502f071bc676064f, 0xfdbcd52e52bf33e6
.xword 0xa69b71457b2d58b4, 0x0b08a370efe46d1d
.xword 0x10d4399328098e10, 0xbd47eba6bcc0bbb9
.xword 0xe6604fcd9552d0eb, 0x4bf39df8019be542
.xword 0x7fa6521c37be0224, 0xd2358029a377378d
.xword 0x891224428ae55cdf, 0x2481f6771e2c6976
.xword 0x3f5d6c94d9c18a7b, 0x92cebea14d08bfd2
.xword 0xc9e91aca649ad480, 0x647ac8fff053e129
.xword 0xfe502f0deb41129a, 0x53c3fd387f882733
.xword 0x08e45953561a4c61, 0xa5778b66c2d379c8
.xword 0xbeab1185053e9ac5, 0x1338c3b091f7af6c
.xword 0x481f67dbb865c43e, 0xe58cb5ee2cacf197
.xword 0x20b4f813d42e0af2, 0x8d272a2640e73f5b
.xword 0xd6008e4d69755409, 0x7b935c78fdbc61a0
.xword 0x604fc69b3a5182ad, 0xcddc14aeae98b704
.xword 0x96fbb0c5870adc56, 0x3b6862f013c3e9ff
.xword 0xa142850208d11a4c, 0x0cd157379c182fe5
.xword 0x57f6f35cb58a44b7, 0xfa6521692143711e
.xword 0xe1b9bb8ae6ae9213, 0x4c2a69bf7267a7ba
.xword 0x170dcdd45bf5cce8, 0xba9e1fe1cf3cf941
.xword 0x8ecbd005f9191e27, 0x235802306dd02b8e
.xword 0x787fa65b444240dc, 0xd5ec746ed08b7575
.xword 0xce30ee8d17669678, 0x63a33cb883afa3d1
.xword 0x388498d3aa3dc883, 0x95174ae63ef4fd2a
.xword 0x0f3dad1425e60e99, 0xa2ae7f21b12f3b30
.xword 0xf989db4a98bd5062, 0x541a097f0c7465cb
.xword 0x4fc6939ccb9986c6, 0xe25541a95f50b36f
.xword 0xb972e5c276c2d83d, 0x14e137f7e20bed94

33
crc/aarch64/crc64_jones_refl_pmull.S Normal file
View File

@ -0,0 +1,33 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "crc64_jones_refl_pmull.h"
#include "crc64_refl_common_pmull.h"
crc64_refl_func crc64_jones_refl_pmull

196
crc/aarch64/crc64_jones_refl_pmull.h Normal file
View File

@ -0,0 +1,196 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.equ p4_low_b0, 0xb4fb
.equ p4_low_b1, 0x6d9a
.equ p4_low_b2, 0xefb1
.equ p4_low_b3, 0xaf86
.equ p4_high_b0, 0x14e4
.equ p4_high_b1, 0x34f0
.equ p4_high_b2, 0x84a6
.equ p4_high_b3, 0xf497
.equ p1_low_b0, 0xa32c
.equ p1_low_b1, 0x505d
.equ p1_low_b2, 0xbe7d
.equ p1_low_b3, 0xd9d7
.equ p1_high_b0, 0x4444
.equ p1_high_b1, 0xc96f
.equ p1_high_b2, 0x0015
.equ p1_high_b3, 0x381d
.equ p0_low_b0, 0x4444
.equ p0_low_b1, 0xc96f
.equ p0_low_b2, 0x0015
.equ p0_low_b3, 0x381d
.equ br_low_b0, 0x9f77
.equ br_low_b1, 0x9aef
.equ br_low_b2, 0xfa32
.equ br_low_b3, 0x3e6c
.equ br_high_b0, 0x936b
.equ br_high_b1, 0x5897
.equ br_high_b2, 0x2653
.equ br_high_b3, 0x2b59
.text
.section .rodata
.align 4
.set .lanchor_crc64_tab,. + 0
.type crc64_tab, %object
.size crc64_tab, 2048
crc64_tab:
.xword 0x0000000000000000, 0x7ad870c830358979
.xword 0xf5b0e190606b12f2, 0x8f689158505e9b8b
.xword 0xc038e5739841b68f, 0xbae095bba8743ff6
.xword 0x358804e3f82aa47d, 0x4f50742bc81f2d04
.xword 0xab28ecb46814fe75, 0xd1f09c7c5821770c
.xword 0x5e980d24087fec87, 0x24407dec384a65fe
.xword 0x6b1009c7f05548fa, 0x11c8790fc060c183
.xword 0x9ea0e857903e5a08, 0xe478989fa00bd371
.xword 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8
.xword 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a
.xword 0xbd301a4810ffd90e, 0xc7e86a8020ca5077
.xword 0x4880fbd87094cbfc, 0x32588b1040a14285
.xword 0xd620138fe0aa91f4, 0xacf86347d09f188d
.xword 0x2390f21f80c18306, 0x594882d7b0f40a7f
.xword 0x1618f6fc78eb277b, 0x6cc0863448deae02
.xword 0xe3a8176c18803589, 0x997067a428b5bcf0
.xword 0xfa11fe77117cdf02, 0x80c98ebf2149567b
.xword 0x0fa11fe77117cdf0, 0x75796f2f41224489
.xword 0x3a291b04893d698d, 0x40f16bccb908e0f4
.xword 0xcf99fa94e9567b7f, 0xb5418a5cd963f206
.xword 0x513912c379682177, 0x2be1620b495da80e
.xword 0xa489f35319033385, 0xde51839b2936bafc
.xword 0x9101f7b0e12997f8, 0xebd98778d11c1e81
.xword 0x64b116208142850a, 0x1e6966e8b1770c73
.xword 0x8719014c99c2b083, 0xfdc17184a9f739fa
.xword 0x72a9e0dcf9a9a271, 0x08719014c99c2b08
.xword 0x4721e43f0183060c, 0x3df994f731b68f75
.xword 0xb29105af61e814fe, 0xc849756751dd9d87
.xword 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f
.xword 0xd9810c6891bd5c04, 0xa3597ca0a188d57d
.xword 0xec09088b6997f879, 0x96d1784359a27100
.xword 0x19b9e91b09fcea8b, 0x636199d339c963f2
.xword 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416
.xword 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4
.xword 0x1f423fcee22f9be0, 0x659a4f06d21a1299
.xword 0xeaf2de5e82448912, 0x902aae96b271006b
.xword 0x74523609127ad31a, 0x0e8a46c1224f5a63
.xword 0x81e2d7997211c1e8, 0xfb3aa75142244891
.xword 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec
.xword 0x41da32eaea507767, 0x3b024222da65fe1e
.xword 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97
.xword 0x57c2c41692bb501c, 0x2d1ab4dea28ed965
.xword 0x624ac0f56a91f461, 0x1892b03d5aa47d18
.xword 0x97fa21650afae693, 0xed2251ad3acf6fea
.xword 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2
.xword 0xfcea28a2faafae69, 0x8632586aca9a2710
.xword 0xc9622c4102850a14, 0xb3ba5c8932b0836d
.xword 0x3cd2cdd162ee18e6, 0x460abd1952db919f
.xword 0x256b24ca6b12f26d, 0x5fb354025b277b14
.xword 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6
.xword 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b
.xword 0x10e3202993385610, 0x6a3b50e1a30ddf69
.xword 0x8e43c87e03060c18, 0xf49bb8b633338561
.xword 0x7bf329ee636d1eea, 0x012b592653589793
.xword 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee
.xword 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c
.xword 0x5863dbf1e3ac9dec, 0x22bbab39d3991495
.xword 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667
.xword 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a
.xword 0x6debdf121b863991, 0x1733afda2bb3b0e8
.xword 0xf34b37458bb86399, 0x8993478dbb8deae0
.xword 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812
.xword 0x3373d23613f9d516, 0x49aba2fe23cc5c6f
.xword 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d
.xword 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc
.xword 0x601c72b9cc20db47, 0x1ac40271fc15523e
.xword 0x5594765a340a7f3a, 0x2f4c0692043ff643
.xword 0xa02497ca54616dc8, 0xdafce7026454e4b1
.xword 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9
.xword 0xcb349e0da4342532, 0xb1eceec59401ac4b
.xword 0xfebc9aee5c1e814f, 0x8464ea266c2b0836
.xword 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4
.xword 0xe8a46c1224f5a634, 0x927c1cda14c02f4d
.xword 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf
.xword 0x289c8961bcb410bb, 0x5244f9a98c8199c2
.xword 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30
.xword 0x438c80a64ce15841, 0x3954f06e7cd4d138
.xword 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca
.xword 0x83b465d5d4a0eece, 0xf96c151de49567b7
.xword 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545
.xword 0x6fbd6d5ebd3716b7, 0x15651d968d029fce
.xword 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c
.xword 0xaf85882d2576a038, 0xd55df8e515432941
.xword 0x5a3569bd451db2ca, 0x20ed197575283bb3
.xword 0xc49581ead523e8c2, 0xbe4df122e51661bb
.xword 0x3125607ab548fa30, 0x4bfd10b2857d7349
.xword 0x04ad64994d625e4d, 0x7e7514517d57d734
.xword 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6
.xword 0x12b5926535897936, 0x686de2ad05bcf04f
.xword 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd
.xword 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0
.xword 0x273d9686cda3dd4b, 0x5de5e64efd965432
.xword 0xb99d7ed15d9d8743, 0xc3450e196da80e3a
.xword 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8
.xword 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5
.xword 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47
.xword 0x4ad64994d625e4da, 0x300e395ce6106da3
.xword 0xbf66a804b64ef628, 0xc5bed8cc867b7f51
.xword 0x8aeeace74e645255, 0xf036dc2f7e51db2c
.xword 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de
.xword 0xe1fea520be311aaf, 0x9b26d5e88e0493d6
.xword 0x144e44b0de5a085d, 0x6e963478ee6f8124
.xword 0x21c640532670ac20, 0x5b1e309b16452559
.xword 0xd476a1c3461bbed2, 0xaeaed10b762e37ab
.xword 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222
.xword 0xc26e573f3ef099a9, 0xb8b627f70ec510d0
.xword 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad
.xword 0x0256b24ca6b12f26, 0x788ec2849684a65f
.xword 0x9cf65a1b368f752e, 0xe62e2ad306bafc57
.xword 0x6946bb8b56e467dc, 0x139ecb4366d1eea5
.xword 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8
.xword 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a
.xword 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1
.xword 0x45775673a732292a, 0x3faf26bb9707a053
.xword 0x70ff52905f188d57, 0x0a2722586f2d042e
.xword 0x854fb3003f739fa5, 0xff97c3c80f4616dc
.xword 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4
.xword 0xee5fbac7cf26d75f, 0x9487ca0fff135e26
.xword 0xdbd7be24370c7322, 0xa10fceec0739fa5b
.xword 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9
.xword 0xcdcf48d84fe75459, 0xb71738107fd2dd20
.xword 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2
.xword 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf
.xword 0xf8474c3bb7cdf024, 0x829f3cf387f8795d
.xword 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355
.xword 0x935745fc4798b8de, 0xe98f353477ad31a7
.xword 0xa6df411fbfb21ca3, 0xdc0731d78f8795da
.xword 0x536fa08fdfd90e51, 0x29b7d047efec8728

310
crc/aarch64/crc64_norm_common_pmull.h Normal file
View File

@ -0,0 +1,310 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.macro crc64_norm_func name:req
.arch armv8-a+crc+crypto
.text
.align 3
.global \name
.type \name, %function
/* uint64_t crc64_norm_func(uint64_t seed, const uint8_t * buf, uint64_t len) */
// parameter
x_seed .req x0
x_buf .req x1
x_len .req x2
// return
x_crc_ret .req x0
// constant
.equ FOLD_SIZE, 1024
// global variables
x_buf_end .req x3 // buffer address with truncated
x_counter .req x4
x_buf_iter .req x5
x_crc64_tab_addr .req x9
w_tmp .req w6
x_tmp .req x6
w_tmp1 .req w7
x_tmp1 .req x7
\name\():
mvn x_seed, x_seed
mov x_counter, 0
cmp x_len, (FOLD_SIZE-1)
bhi .crc64_clmul_pre
.crc64_tab_pre:
cmp x_len, x_counter
bls .done
adrp x_tmp, lanchor_crc64_tab
add x_buf_iter, x_buf, x_counter
add x_buf, x_buf, x_len
add x_crc64_tab_addr, x_tmp, :lo12:lanchor_crc64_tab
.align 3
.loop_crc64_tab:
ldrb w_tmp, [x_buf_iter], 1
cmp x_buf, x_buf_iter
eor x_tmp, x_tmp, x_seed, lsr 56
ldr x_tmp, [x_crc64_tab_addr, x_tmp, lsl 3]
eor x_seed, x_tmp, x_seed, lsl 8
bne .loop_crc64_tab
.done:
mvn x_crc_ret, x_seed
ret
d_tmp .req d3
q_tmp .req q3
v_tmp .req v3
q_x0 .req q2
q_x1 .req q16
q_x2 .req q6
q_x3 .req q4
v_x0 .req v2
v_x1 .req v16
v_x2 .req v6
v_x3 .req v4
d_p4_low .req d3
d_p4_high .req d5
v_p4_low .req v3
v_p4_high .req v5
q_shuffle .req q1
v_shuffle .req v1
.align 2
.crc64_clmul_pre:
adrp x_tmp, .shuffle_data
ldr q_shuffle, [x_tmp, #:lo12:.shuffle_data]
and x_counter, x_len, -64
sub x_tmp1, x_counter, #64
cmp x_tmp1, 63 // align and the truncated buffer size
movi v_x0.4s, 0
ins v_x0.d[1], x_seed
add x_buf_iter, x_buf, 64
ldr q_tmp, [x_buf]
ldr q_x1, [x_buf, 16]
ldr q_x2, [x_buf, 32]
ldr q_x3, [x_buf, 48]
tbl v_tmp.16b, {v_tmp.16b}, v_shuffle.16b
tbl v_x1.16b, {v_x1.16b}, v_shuffle.16b
tbl v_x2.16b, {v_x2.16b}, v_shuffle.16b
tbl v_x3.16b, {v_x3.16b}, v_shuffle.16b
eor v_x0.16b, v_x0.16b, v_tmp.16b
bls .clmul_loop_end
mov x_tmp, p4_high_b0
movk x_tmp, p4_high_b1, lsl 16
movk x_tmp, p4_high_b2, lsl 32
movk x_tmp, p4_high_b3, lsl 48
fmov d_p4_high, x_tmp
mov x_tmp, p4_low_b0
movk x_tmp, p4_low_b1, lsl 16
movk x_tmp, p4_low_b2, lsl 32
movk x_tmp, p4_low_b3, lsl 48
fmov d_p4_low, x_tmp
add x_buf_end, x_buf_iter, x_tmp1
// 1024bit --> 512bit loop
// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t)
d_x0_high .req d24
d_x1_high .req d22
d_x2_high .req d20
d_x3_high .req d18
v_x0_high .req v24
v_x1_high .req v22
v_x2_high .req v20
v_x3_high .req v18
q_y0 .req q17
q_y1 .req q19
q_y2 .req q7
q_y3 .req q0
v_y0 .req v17
v_y1 .req v19
v_y2 .req v7
v_y3 .req v0
.align 3
.clmul_loop:
dup d_x0_high, v_x0.d[1]
dup d_x1_high, v_x1.d[1]
dup d_x2_high, v_x2.d[1]
dup d_x3_high, v_x3.d[1]
add x_buf_iter, x_buf_iter, 64
cmp x_buf_iter, x_buf_end
ldr q_y0, [x_buf_iter, -64]
ldr q_y1, [x_buf_iter, -48]
ldr q_y2, [x_buf_iter, -32]
ldr q_y3, [x_buf_iter, -16]
pmull v_x0.1q, v_x0.1d, v_p4_low.1d
pmull v_x1.1q, v_x1.1d, v_p4_low.1d
pmull v_x2.1q, v_x2.1d, v_p4_low.1d
pmull v_x3.1q, v_x3.1d, v_p4_low.1d
pmull v_x0_high.1q, v_x0_high.1d, v_p4_high.1d
pmull v_x1_high.1q, v_x1_high.1d, v_p4_high.1d
pmull v_x2_high.1q, v_x2_high.1d, v_p4_high.1d
pmull v_x3_high.1q, v_x3_high.1d, v_p4_high.1d
tbl v_y0.16b, {v_y0.16b}, v_shuffle.16b
tbl v_y1.16b, {v_y1.16b}, v_shuffle.16b
tbl v_y2.16b, {v_y2.16b}, v_shuffle.16b
tbl v_y3.16b, {v_y3.16b}, v_shuffle.16b
eor v_x0_high.16b, v_x0_high.16b, v_x0.16b
eor v_x1_high.16b, v_x1_high.16b, v_x1.16b
eor v_x2_high.16b, v_x2_high.16b, v_x2.16b
eor v_x3_high.16b, v_x3_high.16b, v_x3.16b
eor v_x0.16b, v_x0_high.16b, v_y0.16b
eor v_x1.16b, v_x1_high.16b, v_y1.16b
eor v_x2.16b, v_x2_high.16b, v_y2.16b
eor v_x3.16b, v_x3_high.16b, v_y3.16b
bne .clmul_loop
// folding 512bit --> 128bit
// merge x0, x1, x2, x3 => x0 (uint64x2_t)
d_tmp1 .req d18
v_tmp1 .req v18
d_p1_high .req d5
v_p1_high .req v5
d_p1_low .req d3
v_p1_low .req v3
.clmul_loop_end:
mov x_tmp, p1_high_b0
movk x_tmp, p1_high_b1, lsl 16
movk x_tmp, p1_high_b2, lsl 32
movk x_tmp, p1_high_b3, lsl 48
fmov d_p1_high, x_tmp
mov x_tmp, p1_low_b0
movk x_tmp, p1_low_b1, lsl 16
movk x_tmp, p1_low_b2, lsl 32
movk x_tmp, p1_low_b3, lsl 48
fmov d_p1_low, x_tmp
dup d_tmp1, v_x0.d[1]
pmull v_x0.1q, v_x0.1d, v_p1_low.1d
pmull v_tmp1.1q, v_tmp1.1d, v_p1_high.1d
eor v_x0.16b, v_tmp1.16b, v_x0.16b
eor v_x1.16b, v_x0.16b, v_x1.16b
dup d_tmp1, v_x1.d[1]
pmull v_x1.1q, v_x1.1d, v_p1_low.1d
pmull v_tmp1.1q, v_tmp1.1d, v_p1_high.1d
eor v_tmp1.16b, v_tmp1.16b, v_x1.16b
eor v_x2.16b, v_tmp1.16b, v_x2.16b
dup d_tmp1, v_x2.d[1]
pmull v_x2.1q, v_x2.1d, v_p1_low.1d
pmull v_tmp1.1q, v_tmp1.1d, v_p1_high.1d
eor v_x2.16b, v_tmp1.16b, v_x2.16b
eor v_x3.16b, v_x2.16b, v_x3.16b
// fold 64b
d_p0_low .req d3
v_p0_low .req v3
d_x3_high1 .req d2
v_x3_high1 .req v2
mov x_tmp, p0_low_b0
movk x_tmp, p0_low_b1, lsl 16
movk x_tmp, p0_low_b2, lsl 32
movk x_tmp, p0_low_b3, lsl 48
fmov d_p0_low, x_tmp
dup d_x3_high1, v_x3.d[1]
movi v0.4s, 0
ext v0.16b, v0.16b, v_x3.16b, #8
pmull v_x3_high1.1q, v_x3_high1.1d, v_p0_low.1d
eor v0.16b, v0.16b, v_x3_high1.16b
// barrett reduction
mov x_tmp, br_low_b0
movk x_tmp, br_low_b1, lsl 16
movk x_tmp, br_low_b2, lsl 32
movk x_tmp, br_low_b3, lsl 48
movi v1.4s, 0x0
mov x_tmp1, -1
ins v1.d[1], x_tmp1
and v2.16b, v1.16b, v0.16b
fmov d1, x_tmp
dup d4, v0.d[1]
pmull v4.1q, v4.1d, v1.1d
mov x_tmp, br_high_b0
movk x_tmp, br_high_b1, lsl 16
movk x_tmp, br_high_b2, lsl 32
movk x_tmp, br_high_b3, lsl 48
fmov d1, x_tmp
eor v2.16b, v2.16b, v4.16b
dup d2, v2.d[1]
pmull v2.1q, v2.1d, v1.1d
eor v0.16b, v0.16b, v2.16b
umov x_seed, v0.d[0]
b .crc64_tab_pre
.size \name, .-\name
.section .rodata.cst16,"aM",@progbits,16
.align 4
.shuffle_data:
.byte 15, 14, 13, 12, 11, 10, 9, 8
.byte 7, 6, 5, 4, 3, 2, 1, 0
.endm

302
crc/aarch64/crc64_refl_common_pmull.h Normal file
View File

@ -0,0 +1,302 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
.macro crc64_refl_func name:req
.arch armv8-a+crc+crypto
.text
.align 3
.global \name
.type \name, %function
// parameter
x_seed .req x0
x_buf .req x1
x_len .req x2
// return
x_crc_ret .req x0
// constant
.equ FOLD_SIZE, 1024
// global variable
x_buf_end .req x3
x_counter .req x4
x_buf_iter .req x5
x_crc64_tab_addr .req x6
w_tmp .req w7
x_tmp .req x7
// crc64 refl function entry
\name\():
// crc64 for table
mvn x_seed, x_seed
mov x_counter, 0
cmp x_len, (FOLD_SIZE-1)
bhi .crc64_clmul_pre
.crc64_tab_pre:
cmp x_len, x_counter
bls .done
adrp x_tmp, .lanchor_crc64_tab
add x_buf_iter, x_buf, x_counter
add x_buf, x_buf, x_len
add x_crc64_tab_addr, x_tmp, :lo12:.lanchor_crc64_tab
.align 3
.loop_crc64_tab:
ldrb w_tmp, [x_buf_iter], 1
eor w_tmp, w_tmp, w0
cmp x_buf, x_buf_iter
and x_tmp, x_tmp, 255
ldr x_tmp, [x_crc64_tab_addr, x_tmp, lsl 3]
eor x_seed, x_tmp, x_seed, lsr 8
bne .loop_crc64_tab
.done:
mvn x_crc_ret, x_crc_ret
ret
// clmul prepare
q_x0 .req q0
q_x1 .req q4
q_x2 .req q6
q_x3 .req q1
v_x0 .req v0
v_x1 .req v4
v_x2 .req v6
v_x3 .req v1
d_p4_high .req d17
d_p4_low .req d7
v_p4_high .req v17
v_p4_low .req v7
d_y0_tmp .req d0
v_y0_tmp .req v0
q_tmp .req q2
v_tmp .req v2
.align 2
.crc64_clmul_pre:
ldr q_tmp, [x_buf]
ldr q_x1, [x_buf, 16]
ldr q_x2, [x_buf, 32]
ldr q_x3, [x_buf, 48]
and x_counter, x_len, -64
sub x_tmp, x_counter, #64
cmp x_tmp, 63
fmov d_y0_tmp, x_seed // save crc to d0
eor v_x0.16b, v_y0_tmp.16b, v_tmp.16b
add x_buf_iter, x_buf, 64
bls .clmul_loop_end
add x_buf_end, x_buf_iter, x_tmp
mov x_tmp, p4_high_b0
movk x_tmp, p4_high_b1, lsl 16
movk x_tmp, p4_high_b2, lsl 32
movk x_tmp, p4_high_b3, lsl 48
fmov d_p4_high, x_tmp
mov x_tmp, p4_low_b0
movk x_tmp, p4_low_b1, lsl 16
movk x_tmp, p4_low_b2, lsl 32
movk x_tmp, p4_low_b3, lsl 48
fmov d_p4_low, x_tmp
// 1024bit --> 512bit loop
// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t)
d_x0_high .req d24
d_x1_high .req d22
d_x2_high .req d20
d_x3_high .req d16
v_x0_high .req v24
v_x1_high .req v22
v_x2_high .req v20
v_x3_high .req v16
q_x0_tmp .req q2
q_x1_tmp .req q5
q_x2_tmp .req q3
q_x3_tmp .req q18
v_x0_tmp .req v2
v_x1_tmp .req v5
v_x2_tmp .req v3
v_x3_tmp .req v18
q_x0_tmp .req q2
q_x1_tmp .req q5
q_x2_tmp .req q3
q_x3_tmp .req q18
.align 3
.clmul_loop:
add x_buf_iter, x_buf_iter, 64
cmp x_buf_iter, x_buf_end
dup d_x0_high, v_x0.d[1]
dup d_x1_high, v_x1.d[1]
dup d_x2_high, v_x2.d[1]
dup d_x3_high, v_x3.d[1]
pmull v_x0_high.1q, v_x0_high.1d, v_p4_high.1d
pmull v_x1_high.1q, v_x1_high.1d, v_p4_high.1d
pmull v_x2_high.1q, v_x2_high.1d, v_p4_high.1d
pmull v_x3_high.1q, v_x3_high.1d, v_p4_high.1d
pmull v_x0.1q, v_x0.1d, v_p4_low.1d
pmull v_x1.1q, v_x1.1d, v_p4_low.1d
pmull v_x2.1q, v_x2.1d, v_p4_low.1d
pmull v_x3.1q, v_x3.1d, v_p4_low.1d
ldr q_x0_tmp, [x_buf_iter, -64]
ldr q_x1_tmp, [x_buf_iter, -48]
ldr q_x2_tmp, [x_buf_iter, -32]
ldr q_x3_tmp, [x_buf_iter, -16]
eor v_x0_tmp.16b, v_x0_tmp.16b, v_x0_high.16b
eor v_x1_tmp.16b, v_x1_tmp.16b, v_x1_high.16b
eor v_x2_tmp.16b, v_x2_tmp.16b, v_x2_high.16b
eor v_x3_tmp.16b, v_x3_tmp.16b, v_x3_high.16b
eor v_x0.16b, v_x0_tmp.16b, v_x0.16b
eor v_x1.16b, v_x1_tmp.16b, v_x1.16b
eor v_x2.16b, v_x2_tmp.16b, v_x2.16b
eor v_x3.16b, v_x3_tmp.16b, v_x3.16b
bne .clmul_loop
// folding 512bit --> 128bit
// merge x0, x1, x2, x3 => x3 (uint64x2_t)
// input: x0 -> v_x0, x1 -> v_x1, x2 -> v_x2, x3 -> v_x3
// output: v_x3
d_p1_high .req d5
d_p1_low .req d3
v_p1_high .req v5
v_p1_low .req v3
d_tmp_high .req d16
d_tmp_low .req d2
v_tmp_high .req v16
v_tmp_low .req v2
.clmul_loop_end:
mov x_tmp, p1_high_b0
movk x_tmp, p1_high_b1, lsl 16
movk x_tmp, p1_high_b2, lsl 32
movk x_tmp, p1_high_b3, lsl 48
fmov d_p1_high, x_tmp
mov x_tmp, p1_low_b0
movk x_tmp, p1_low_b1, lsl 16
movk x_tmp, p1_low_b2, lsl 32
movk x_tmp, p1_low_b3, lsl 48
fmov d_p1_low, x_tmp
dup d_tmp_high, v_x0.d[1]
dup d_tmp_low, v_x0.d[0]
pmull v_tmp_high.1q, v_tmp_high.1d, v_p1_high.1d
pmull v_tmp_low.1q, v_tmp_low.1d, v_p1_low.1d
eor v_tmp_high.16b, v_tmp_high.16b, v_tmp_low.16b
eor v_x1.16b, v_tmp_high.16b, v_x1.16b
dup d_tmp_high, v_x1.d[1]
pmull v_x1.1q, v_x1.1d, v_p1_low.1d
pmull v_tmp_high.1q, v_tmp_high.1d, v_p1_high.1d
eor v_tmp_high.16b, v_tmp_high.16b, v_x1.16b
eor v_x2.16b, v_tmp_high.16b, v_x2.16b
dup d_tmp_high, v_x2.d[1]
pmull v_x2.1q, v_x2.1d, v_p1_low.1d
pmull v_tmp_high.1q, v_tmp_high.1d, v_p1_high.1d
eor v_tmp_high.16b, v_tmp_high.16b, v_x2.16b
eor v_x3.16b, v_tmp_high.16b, v_x3.16b
// fold 64b
// input: v_x3
// output: v_x3
d_p0_low .req d3
v_p0_low .req v3
d_x3_low_fold_64b .req d2
v_x3_low_fold_64b .req v2
v_zero_fold_64b .req v0
mov x_tmp, p0_low_b0
movk x_tmp, p0_low_b1, lsl 16
movk x_tmp, p0_low_b2, lsl 32
movk x_tmp, p0_low_b3, lsl 48
fmov d_p0_low, x_tmp
dup d_x3_low_fold_64b, v_x3.d[0]
movi v_zero_fold_64b.4s, 0
ext v_x3.16b, v_x3.16b, v0.16b, #8
pmull v_x3_low_fold_64b.1q, v_x3_low_fold_64b.1d, v_p0_low.1d
eor v_x3.16b, v_x3.16b, v_x3_low_fold_64b.16b
// barrett reduction
// input: v_x3
// output: x0
d_br_low .req d3
d_br_high .req d5
v_br_low .req v3
v_br_high .req v5
mov x0, br_low_b0
movk x0, br_low_b1, lsl 16
movk x0, br_low_b2, lsl 32
movk x0, br_low_b3, lsl 48
fmov d_br_low, x0
mov x0, br_high_b0
movk x0, br_high_b1, lsl 16
movk x0, br_high_b2, lsl 32
movk x0, br_high_b3, lsl 48
fmov d_br_high, x0
dup d2, v_x3.d[0]
pmull v2.1q, v2.1d, v_br_low.1d
pmull v4.1q, v2.1d, v_br_high.1d
ext v0.16b, v0.16b, v2.16b, #8
eor v0.16b, v0.16b, v4.16b
eor v0.16b, v0.16b, v_x3.16b
umov x0, v0.d[1]
b .crc64_tab_pre
.size \name, .-\name
.endm

42
crc/aarch64/crc_multibinary_arm.S Normal file
View File

@ -0,0 +1,42 @@
########################################################################
# Copyright(c) 2019 Arm Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Arm Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################
#include "multibinary_arm.h"
mbin_dispatch crc32_iscsi crc32_iscsi_refl_pmull
mbin_dispatch crc16_t10dif crc16_t10dif_pmull
mbin_dispatch crc16_t10dif_copy crc16_t10dif_copy_pmull
mbin_dispatch crc32_ieee crc32_ieee_norm_pmull
mbin_dispatch crc32_gzip_refl crc32_gzip_refl_pmull
mbin_dispatch crc64_ecma_refl crc64_ecma_refl_pmull
mbin_dispatch crc64_ecma_norm crc64_ecma_norm_pmull
mbin_dispatch crc64_iso_refl crc64_iso_refl_pmull
mbin_dispatch crc64_iso_norm crc64_iso_norm_pmull
mbin_dispatch crc64_jones_refl crc64_jones_refl_pmull
mbin_dispatch crc64_jones_norm crc64_jones_norm_pmull