######################################################################## # Copyright (c) 2019 Microsoft Corporation. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # * Neither the name of Microsoft Corporation nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ######################################################################### #include "../include/aarch64_label.h" // parameters #define w_seed w0 #define x_seed x0 #define x_buf x1 #define w_len w2 #define x_len x2 // return #define w_crc_ret w0 #define x_crc_ret x0 // constant #define FOLD_SIZE 64 // global variables #define x_buf_end x3 #define w_counter w4 #define x_counter x4 #define x_buf_iter x5 #define x_crc_tab_addr x6 #define x_tmp2 x6 #define w_tmp w7 #define x_tmp x7 #define v_x0 v0 #define d_x0 d0 #define s_x0 s0 #define q_x1 q1 #define v_x1 v1 #define q_x2 q2 #define v_x2 v2 #define q_x3 q3 #define v_x3 v3 #define d_x3 d3 #define s_x3 s3 #define q_y0 q4 #define v_y0 v4 #define v_tmp_high v4 #define d_tmp_high d4 #define q_y1 q5 #define v_y1 v5 #define v_tmp_low v5 #define q_y2 q6 #define v_y2 v6 #define q_y3 q7 #define v_y3 v7 #define q_x0_tmp q30 #define v_x0_tmp v30 #define d_p4_high v30.d[1] #define d_p4_low d30 #define v_p4 v30 #define d_p1_high v30.d[1] #define d_p1_low d30 #define v_p1 v30 #define d_p0_high v30.d[1] #define d_p0_low d30 #define v_p0 v30 #define d_br_low d30 #define d_br_low2 v30.d[1] #define v_br_low v30 #define q_shuffle q31 #define v_shuffle v31 #define d_br_high d31 #define d_br_high2 v31.d[1] #define v_br_high v31 #define d_p0_low2 d31 #define d_p0_high2 v31.d[1] #define v_p02 v31 #define v_x0_high v16 #define v_x1_high v17 #define v_x2_high v18 #define v_x3_high v19 .macro crc_refl_load_first_block ldr q_x0_tmp, [x_buf] ldr q_x1, [x_buf, 16] ldr q_x2, [x_buf, 32] ldr q_x3, [x_buf, 48] and x_counter, x_len, -64 sub x_tmp, x_counter, #64 cmp x_tmp, 63 add x_buf_iter, x_buf, 64 eor v_x0.16b, v_x0.16b, v_x0_tmp.16b .endm .macro crc_norm_load_first_block #ifndef __APPLE__ adrp x_tmp, .shuffle_data ldr q_shuffle, [x_tmp, #:lo12:.shuffle_data] #else adrp x_tmp, .shuffle_data@PAGE ldr q_shuffle, [x_tmp, #.shuffle_data@PAGEOFF] #endif ldr q_x0_tmp, [x_buf] ldr q_x1, [x_buf, 16] ldr q_x2, [x_buf, 32] ldr q_x3, [x_buf, 48] and x_counter, x_len, -64 sub x_tmp, x_counter, #64 cmp x_tmp, 63 add x_buf_iter, x_buf, 64 tbl v_x0_tmp.16b, {v_x0_tmp.16b}, v_shuffle.16b tbl v_x1.16b, {v_x1.16b}, v_shuffle.16b tbl v_x2.16b, {v_x2.16b}, v_shuffle.16b tbl v_x3.16b, {v_x3.16b}, v_shuffle.16b eor v_x0.16b, v_x0.16b, v_x0_tmp.16b .endm .macro crc32_load_p4 add x_buf_end, x_buf_iter, x_tmp mov x_tmp, p4_low_b0 movk x_tmp, p4_low_b1, lsl 16 fmov d_p4_low, x_tmp mov x_tmp2, p4_high_b0 movk x_tmp2, p4_high_b1, lsl 16 fmov d_p4_high, x_tmp2 .endm .macro crc64_load_p4 add x_buf_end, x_buf_iter, x_tmp mov x_tmp, p4_low_b0 movk x_tmp, p4_low_b1, lsl 16 movk x_tmp, p4_low_b2, lsl 32 movk x_tmp, p4_low_b3, lsl 48 fmov d_p4_low, x_tmp mov x_tmp2, p4_high_b0 movk x_tmp2, p4_high_b1, lsl 16 movk x_tmp2, p4_high_b2, lsl 32 movk x_tmp2, p4_high_b3, lsl 48 fmov d_p4_high, x_tmp2 .endm .macro crc_refl_loop .align 3 .clmul_loop: // interleave ldr and pmull(2) for arch which can only issue quadword load every // other cycle (i.e. A55) ldr q_y0, [x_buf_iter] pmull2 v_x0_high.1q, v_x0.2d, v_p4.2d ldr q_y1, [x_buf_iter, 16] pmull2 v_x1_high.1q, v_x1.2d, v_p4.2d ldr q_y2, [x_buf_iter, 32] pmull2 v_x2_high.1q, v_x2.2d, v_p4.2d ldr q_y3, [x_buf_iter, 48] pmull2 v_x3_high.1q, v_x3.2d, v_p4.2d pmull v_x0.1q, v_x0.1d, v_p4.1d add x_buf_iter, x_buf_iter, 64 pmull v_x1.1q, v_x1.1d, v_p4.1d cmp x_buf_iter, x_buf_end pmull v_x2.1q, v_x2.1d, v_p4.1d pmull v_x3.1q, v_x3.1d, v_p4.1d eor v_x0.16b, v_x0_high.16b, v_x0.16b eor v_x0.16b, v_x0.16b, v_y0.16b eor v_x1.16b, v_x1_high.16b, v_x1.16b eor v_x1.16b, v_x1.16b, v_y1.16b eor v_x2.16b, v_x2_high.16b, v_x2.16b eor v_x2.16b, v_x2.16b, v_y2.16b eor v_x3.16b, v_x3_high.16b, v_x3.16b eor v_x3.16b, v_x3.16b, v_y3.16b bne .clmul_loop .endm .macro crc_norm_loop .align 3 .clmul_loop: // interleave ldr and pmull(2) for arch which can only issue quadword load every // other cycle (i.e. A55) ldr q_y0, [x_buf_iter] pmull2 v_x0_high.1q, v_x0.2d, v_p4.2d ldr q_y1, [x_buf_iter, 16] pmull2 v_x1_high.1q, v_x1.2d, v_p4.2d ldr q_y2, [x_buf_iter, 32] pmull2 v_x2_high.1q, v_x2.2d, v_p4.2d ldr q_y3, [x_buf_iter, 48] pmull2 v_x3_high.1q, v_x3.2d, v_p4.2d pmull v_x0.1q, v_x0.1d, v_p4.1d add x_buf_iter, x_buf_iter, 64 pmull v_x1.1q, v_x1.1d, v_p4.1d cmp x_buf_iter, x_buf_end pmull v_x2.1q, v_x2.1d, v_p4.1d pmull v_x3.1q, v_x3.1d, v_p4.1d tbl v_y0.16b, {v_y0.16b}, v_shuffle.16b tbl v_y1.16b, {v_y1.16b}, v_shuffle.16b tbl v_y2.16b, {v_y2.16b}, v_shuffle.16b tbl v_y3.16b, {v_y3.16b}, v_shuffle.16b eor v_x0.16b, v_x0.16b, v_x0_high.16b eor v_x1.16b, v_x1.16b, v_x1_high.16b eor v_x2.16b, v_x2.16b, v_x2_high.16b eor v_x3.16b, v_x3.16b, v_x3_high.16b eor v_x0.16b, v_x0.16b, v_y0.16b eor v_x1.16b, v_x1.16b, v_y1.16b eor v_x2.16b, v_x2.16b, v_y2.16b eor v_x3.16b, v_x3.16b, v_y3.16b bne .clmul_loop .endm .macro crc32_fold_512b_to_128b mov x_tmp, p1_low_b0 movk x_tmp, p1_low_b1, lsl 16 fmov d_p1_low, x_tmp mov x_tmp2, p1_high_b0 movk x_tmp2, p1_high_b1, lsl 16 fmov d_p1_high, x_tmp2 pmull2 v_tmp_high.1q, v_x0.2d, v_p1.2d pmull v_tmp_low.1q, v_x0.1d, v_p1.1d eor v_x1.16b, v_x1.16b, v_tmp_high.16b eor v_x1.16b, v_x1.16b, v_tmp_low.16b pmull2 v_tmp_high.1q, v_x1.2d, v_p1.2d pmull v_tmp_low.1q, v_x1.1d, v_p1.1d eor v_x2.16b, v_x2.16b, v_tmp_high.16b eor v_x2.16b, v_x2.16b, v_tmp_low.16b pmull2 v_tmp_high.1q, v_x2.2d, v_p1.2d pmull v_tmp_low.1q, v_x2.1d, v_p1.1d eor v_x3.16b, v_x3.16b, v_tmp_high.16b eor v_x3.16b, v_x3.16b, v_tmp_low.16b .endm .macro crc64_fold_512b_to_128b mov x_tmp, p1_low_b0 movk x_tmp, p1_low_b1, lsl 16 movk x_tmp, p1_low_b2, lsl 32 movk x_tmp, p1_low_b3, lsl 48 fmov d_p1_low, x_tmp mov x_tmp2, p1_high_b0 movk x_tmp2, p1_high_b1, lsl 16 movk x_tmp2, p1_high_b2, lsl 32 movk x_tmp2, p1_high_b3, lsl 48 fmov d_p1_high, x_tmp2 pmull2 v_tmp_high.1q, v_x0.2d, v_p1.2d pmull v_tmp_low.1q, v_x0.1d, v_p1.1d eor v_x1.16b, v_x1.16b, v_tmp_high.16b eor v_x1.16b, v_x1.16b, v_tmp_low.16b pmull2 v_tmp_high.1q, v_x1.2d, v_p1.2d pmull v_tmp_low.1q, v_x1.1d, v_p1.1d eor v_x2.16b, v_x2.16b, v_tmp_high.16b eor v_x2.16b, v_x2.16b, v_tmp_low.16b pmull2 v_tmp_high.1q, v_x2.2d, v_p1.2d pmull v_tmp_low.1q, v_x2.1d, v_p1.1d eor v_x3.16b, v_x3.16b, v_tmp_high.16b eor v_x3.16b, v_x3.16b, v_tmp_low.16b .endm