generic-library/isa-l
1
0
Fork 0
mirror of https://github.com/intel/isa-l.git synced 2024-12-12 09:23:50 +01:00
Code Issues Projects Releases Wiki Activity
7ebc65baa7
isa-l/igzip/encode_df_06.asm
H.J. Lu cd888f01a4 x86: Add ENDBR32/ENDBR64 at function entries for Intel CET 
To support Intel CET, all indirect branch targets must start with
ENDBR32/ENDBR64.  Here is a patch to define endbranch and add it to
function entries in x86 assembly codes which are indirect branch
targets as discovered by running testsuite on Intel CET machine and
visual inspection.

Verified with

$ CC="gcc -Wl,-z,cet-report=error -fcf-protection" CXX="g++ -Wl,-z,cet-report=error -fcf-protection" .../configure x86_64-linux
$ make -j8
$ make -j8 check

with both nasm and yasm on both CET and non-CET machines.

Change-Id: I9822578e7294fb5043a64ab7de5c41de81a7d337
Signed-off-by: H.J. Lu <hjl.tools@gmail.com>
2020-05-26 09:16:49 -07:00

625 lines
15 KiB
NASM
Raw Blame History

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Copyright(c) 2011-2018 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
; are met:
; * Redistributions of source code must retain the above copyright
; notice, this list of conditions and the following disclaimer.
; * Redistributions in binary form must reproduce the above copyright
; notice, this list of conditions and the following disclaimer in
; the documentation and/or other materials provided with the
; distribution.
; * Neither the name of Intel Corporation nor the names of its
; contributors may be used to endorse or promote products derived
; from this software without specific prior written permission.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%include "reg_sizes.asm"
%include "lz0a_const.asm"
%include "data_struct2.asm"
%include "stdmac.asm"
%ifdef HAVE_AS_KNOWS_AVX512
%define ARCH 06
%define USE_HSWNI
; tree entry is 4 bytes:
; lit/len tree (513 entries)
; | 3 | 2 | 1 | 0 |
; | len | code |
;
; dist tree
; | 3 | 2 | 1 | 0 |
; |eblen:codlen| code |
; token format:
; DIST_OFFSET:0 : lit/len
; 31:(DIST_OFFSET + 5) : dist Extra Bits
; (DIST_OFFSET + 5):DIST_OFFSET : dist code
; lit/len: 0-256 (literal)
; 257-512 (dist + 254)
; returns final token pointer
; equal to token_end if successful
; uint32_t* encode_df(uint32_t *token_start, uint32_t *token_end,
; BitBuf *out_buf, uint32_t *trees);
%ifidn __OUTPUT_FORMAT__, win64
%define arg1 rcx
%define arg2 rdx
%define arg3 r8
%define arg4 r9
%define sym rsi
%define dsym rdi
%define hufftables r9
%define ptr r11
%else
; Linux
%define arg1 rdi
%define arg2 rsi
%define arg3 rdx
%define arg4 rcx
%define sym r9
%define dsym r8
%define hufftables r11
%define ptr rdi
%endif
%define in_buf_end arg2
%define bitbuf arg3
%define out_buf bitbuf
; bit_count is rcx
%define bits rax
%define data r12
%define tmp rbx
%define len dsym
%define tmp2 r10
%define end_ptr rbp
%define LIT_MASK ((0x1 << LIT_LEN_BIT_COUNT) - 1)
%define DIST_MASK ((0x1 << DIST_LIT_BIT_COUNT) - 1)
%define codes1 zmm1
%define code_lens1 zmm2
%define codes2 zmm3
%define code_lens2 zmm4
%define codes3 zmm5
%define ztmp zmm5
%define code_lens3 zmm6
%define codes4 zmm7
%define syms zmm7
%define code_lens4 zmm8
%define dsyms zmm8
%define zbits_count_q zmm8
%define codes_lookup1 zmm9
%define codes_lookup2 zmm10
%define datas zmm11
%define zbits zmm12
%define zbits_count zmm13
%define zoffset_mask zmm14
%define znotoffset_mask zmm23
%define zq_64 zmm15
%define zlit_mask zmm16
%define zdist_mask zmm17
%define zlit_icr_mask zmm18
%define zeb_icr_mask zmm19
%define zmax_write zmm20
%define zrot_perm zmm21
%define zq_8 zmm22
%define VECTOR_SIZE 0x40
%define VECTOR_LOOP_PROCESSED (2 * VECTOR_SIZE)
%define VECTOR_SLOP 0x40 - 8
gpr_save_mem_offset equ 0
gpr_save_mem_size equ 8 * 6
xmm_save_mem_offset equ gpr_save_mem_offset + gpr_save_mem_size
xmm_save_mem_size equ 10 * 16
bitbuf_mem_offset equ xmm_save_mem_offset + xmm_save_mem_size
bitbuf_mem_size equ 8
stack_size equ gpr_save_mem_size + xmm_save_mem_size + bitbuf_mem_size
%macro FUNC_SAVE 0
sub rsp, stack_size
mov [rsp + gpr_save_mem_offset + 0*8], rbx
mov [rsp + gpr_save_mem_offset + 1*8], rbp
mov [rsp + gpr_save_mem_offset + 2*8], r12
%ifidn __OUTPUT_FORMAT__, win64
mov [rsp + gpr_save_mem_offset + 3*8], rsi
mov [rsp + gpr_save_mem_offset + 4*8], rdi
MOVDQU [rsp + xmm_save_mem_offset + 0*8], xmm6
MOVDQU [rsp + xmm_save_mem_offset + 1*8], xmm7
MOVDQU [rsp + xmm_save_mem_offset + 2*8], xmm8
MOVDQU [rsp + xmm_save_mem_offset + 3*8], xmm9
MOVDQU [rsp + xmm_save_mem_offset + 4*8], xmm10
MOVDQU [rsp + xmm_save_mem_offset + 5*8], xmm11
MOVDQU [rsp + xmm_save_mem_offset + 6*8], xmm12
MOVDQU [rsp + xmm_save_mem_offset + 7*8], xmm13
MOVDQU [rsp + xmm_save_mem_offset + 8*8], xmm14
MOVDQU [rsp + xmm_save_mem_offset + 9*8], xmm15
%endif
%endm
%macro FUNC_RESTORE 0
mov rbx, [rsp + gpr_save_mem_offset + 0*8]
mov rbp, [rsp + gpr_save_mem_offset + 1*8]
mov r12, [rsp + gpr_save_mem_offset + 2*8]
%ifidn __OUTPUT_FORMAT__, win64
mov rsi, [rsp + gpr_save_mem_offset + 3*8]
mov rdi, [rsp + gpr_save_mem_offset + 4*8]
MOVDQU xmm6, [rsp + xmm_save_mem_offset + 0*8]
MOVDQU xmm7, [rsp + xmm_save_mem_offset + 1*8]
MOVDQU xmm8, [rsp + xmm_save_mem_offset + 2*8]
MOVDQU xmm9, [rsp + xmm_save_mem_offset + 3*8]
MOVDQU xmm10, [rsp + xmm_save_mem_offset + 4*8]
MOVDQU xmm11, [rsp + xmm_save_mem_offset + 5*8]
MOVDQU xmm12, [rsp + xmm_save_mem_offset + 6*8]
MOVDQU xmm13, [rsp + xmm_save_mem_offset + 7*8]
MOVDQU xmm14, [rsp + xmm_save_mem_offset + 8*8]
MOVDQU xmm15, [rsp + xmm_save_mem_offset + 9*8]
%endif
add rsp, stack_size
%endmacro
default rel
section .text
global encode_deflate_icf_ %+ ARCH
encode_deflate_icf_ %+ ARCH:
endbranch
FUNC_SAVE
%ifnidn ptr, arg1
mov ptr, arg1
%endif
%ifnidn hufftables, arg4
mov hufftables, arg4
%endif
mov [rsp + bitbuf_mem_offset], bitbuf
mov bits, [bitbuf + _m_bits]
mov ecx, [bitbuf + _m_bit_count]
mov end_ptr, [bitbuf + _m_out_end]
mov out_buf, [bitbuf + _m_out_buf] ; clobbers bitbuf
sub end_ptr, VECTOR_SLOP
sub in_buf_end, VECTOR_LOOP_PROCESSED
cmp ptr, in_buf_end
jge .finish
kxorq k0, k0, k0
kmovq k1, [k_mask_1]
kmovq k2, [k_mask_2]
kmovq k3, [k_mask_3]
kmovq k4, [k_mask_4]
kmovq k5, [k_mask_5]
vmovdqa64 zrot_perm, [rot_perm]
vbroadcasti64x2 zq_64, [q_64]
vbroadcasti64x2 zq_8, [q_8]
vpbroadcastq zoffset_mask, [offset_mask]
vpternlogd znotoffset_mask, znotoffset_mask, zoffset_mask, 0x55
vpbroadcastd zlit_mask, [lit_mask]
vpbroadcastd zdist_mask, [dist_mask]
vpbroadcastd zlit_icr_mask, [lit_icr_mask]
vpbroadcastd zeb_icr_mask, [eb_icr_mask]
vpbroadcastd zmax_write, [max_write_d]
knotq k6, k0
vmovdqu64 datas, [ptr]
vpandd syms, datas, zlit_mask
vpgatherdd codes_lookup1 {k6}, [hufftables + _lit_len_table + 4 * syms]
knotq k7, k0
vpsrld dsyms, datas, DIST_OFFSET
vpandd dsyms, dsyms, zdist_mask
vpgatherdd codes_lookup2 {k7}, [hufftables + _dist_table + 4 * dsyms]
vmovq zbits %+ x, bits
vmovq zbits_count %+ x, rcx
.main_loop:
;; Sets codes1 to contain lit/len codes andcode_lens1 the corresponding lengths
vpsrld code_lens1, codes_lookup1, 24
vpandd codes1, codes_lookup1, zlit_icr_mask
;; Sets codes2 to contain dist codes, code_lens2 the corresponding lengths,
;; and code_lens3 the extra bit counts
vmovdqu16 codes2 {k1}{z}, codes_lookup2 ;Bits 8 and above of zbits are 0
vpsrld code_lens2, codes_lookup2, 24
vpsrld code_lens3, codes_lookup2, 16
vpandd code_lens3, code_lens3, zeb_icr_mask
;; Set codes3 to contain the extra bits
vpsrld codes3, datas, EXTRA_BITS_OFFSET
cmp out_buf, end_ptr
ja .main_loop_exit
;; Start code lookups for next iteration
knotq k6, k0
add ptr, VECTOR_SIZE
vmovdqu64 datas, [ptr]
vpandd syms, datas, zlit_mask
vpgatherdd codes_lookup1 {k6}, [hufftables + _lit_len_table + 4 * syms]
knotq k7, k0
vpsrld dsyms, datas, DIST_OFFSET
vpandd dsyms, dsyms, zdist_mask
vpgatherdd codes_lookup2 {k7}, [hufftables + _dist_table + 4 * dsyms]
;; Merge dist code with extra bits
vpsllvd codes3, codes3, code_lens2
vpxord codes2, codes2, codes3
vpaddd code_lens2, code_lens2, code_lens3
;; Check for long codes
vpaddd code_lens3, code_lens1, code_lens2
vpcmpgtd k6, code_lens3, zmax_write
ktestd k6, k6
jnz .long_codes
;; Merge dist and len codes
vpsllvd codes2, codes2, code_lens1
vpxord codes1, codes1, codes2
vmovdqa32 codes3 {k1}{z}, codes1
vpsrlq codes1, codes1, 32
vpsrlq code_lens1, code_lens3, 32
vmovdqa32 code_lens3 {k1}{z}, code_lens3
;; Merge bitbuf bits
vpsllvq codes3, codes3, zbits_count
vpxord codes3, codes3, zbits
vpaddq code_lens3, code_lens3, zbits_count
;; Merge two symbols into qwords
vpsllvq codes1, codes1, code_lens3
vpxord codes1, codes1, codes3
vpaddq code_lens1, code_lens1, code_lens3
;; Determine total bits at end of each qword
vpermq zbits_count {k5}{z}, zrot_perm, code_lens1
vpaddq code_lens2, zbits_count, code_lens1
vshufi64x2 zbits_count {k3}{z}, code_lens2, code_lens2, 0x90
vpaddq code_lens2, code_lens2, zbits_count
vshufi64x2 zbits_count {k2}{z}, code_lens2, code_lens2, 0x40
vpaddq code_lens2, code_lens2, zbits_count
;; Bit align quadwords
vpandd zbits_count, code_lens2, zoffset_mask
vpermq zbits_count_q {k5}{z}, zrot_perm, zbits_count
vpsllvq codes1, codes1, zbits_count_q
;; Check whether any of the last bytes overlap
vpcmpq k6 {k5}, code_lens1, zbits_count, 1
;; Get last byte in each qword
vpsrlq code_lens2, code_lens2, 3
vpaddq code_lens1, code_lens1, zbits_count_q
vpandq code_lens1, code_lens1, znotoffset_mask
vpsrlvq codes3, codes1, code_lens1
;; Branch to handle overlapping last bytes
ktestd k6, k6
jnz .small_codes
.small_codes_next:
;; Save off zbits and zbits_count for next loop
knotq k7, k5
vpermq zbits {k7}{z}, zrot_perm, codes3
vpermq zbits_count {k7}{z}, zrot_perm, zbits_count
;; Merge last byte in each qword with the next qword
vpermq codes3 {k5}{z}, zrot_perm, codes3
vpxord codes1, codes1, codes3
;; Determine total bytes written
vextracti64x2 code_lens1 %+ x, code_lens2, 3
vpextrq tmp2, code_lens1 %+ x, 1
;; Write out qwords
knotq k6, k0
vpermq code_lens2 {k5}{z}, zrot_perm, code_lens2
vpscatterqq [out_buf + code_lens2] {k6}, codes1
add out_buf, tmp2
cmp ptr, in_buf_end
jbe .main_loop
.main_loop_exit:
vmovq rcx, zbits_count %+ x
vmovq bits, zbits %+ x
jmp .finish
.small_codes:
;; Merge overlapping last bytes
vpermq codes4 {k6}{z}, zrot_perm, codes3
vporq codes3, codes3, codes4
kshiftlq k7, k6, 1
ktestd k6, k7
jz .small_codes_next
kandq k6, k6, k7
jmp .small_codes
.long_codes:
add end_ptr, VECTOR_SLOP
sub ptr, VECTOR_SIZE
vmovdqa32 codes3 {k1}{z}, codes1
vmovdqa32 code_lens3 {k1}{z}, code_lens1
vmovdqa32 codes4 {k1}{z}, codes2
vpsllvq codes4, codes4, code_lens3
vpxord codes3, codes3, codes4
vpaddd code_lens3, code_lens1, code_lens2
vpsrlq codes1, codes1, 32
vpsrlq code_lens1, code_lens1, 32
vpsrlq codes2, codes2, 32
vpsllvq codes2, codes2, code_lens1
vpxord codes1, codes1, codes2
vpsrlq code_lens1, code_lens3, 32
vmovdqa32 code_lens3 {k1}{z}, code_lens3
;; Merge bitbuf bits
vpsllvq codes3, codes3, zbits_count
vpxord codes3, codes3, zbits
vpaddq code_lens3, code_lens3, zbits_count
vpaddq code_lens1, code_lens1, code_lens3
xor bits, bits
xor rcx, rcx
vpsubq code_lens1, code_lens1, code_lens3
vmovdqu64 codes2, codes1
vmovdqu64 code_lens2, code_lens1
vmovdqu64 codes4, codes3
vmovdqu64 code_lens4, code_lens3
%assign i 0
%rep 4
%assign i (i + 1)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
cmp out_buf, end_ptr
ja .overflow
;; insert LL code
vmovq sym, codes3 %+ x
vmovq tmp2, code_lens3 %+ x
SHLX sym, sym, rcx
or bits, sym
add rcx, tmp2
; empty bits
mov [out_buf], bits
mov tmp, rcx
shr tmp, 3 ; byte count
add out_buf, tmp
mov tmp, rcx
and rcx, ~7
SHRX bits, bits, rcx
mov rcx, tmp
and rcx, 7
add ptr, 4
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
cmp out_buf, end_ptr
ja .overflow
;; insert LL code
vmovq sym, codes1 %+ x
vmovq tmp2, code_lens1 %+ x
SHLX sym, sym, rcx
or bits, sym
add rcx, tmp2
; empty bits
mov [out_buf], bits
mov tmp, rcx
shr tmp, 3 ; byte count
add out_buf, tmp
mov tmp, rcx
and rcx, ~7
SHRX bits, bits, rcx
mov rcx, tmp
and rcx, 7
add ptr, 4
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
cmp out_buf, end_ptr
ja .overflow
;; insert LL code
vpextrq sym, codes3 %+ x, 1
vpextrq tmp2, code_lens3 %+ x, 1
SHLX sym, sym, rcx
or bits, sym
add rcx, tmp2
; empty bits
mov [out_buf], bits
mov tmp, rcx
shr tmp, 3 ; byte count
add out_buf, tmp
mov tmp, rcx
and rcx, ~7
SHRX bits, bits, rcx
mov rcx, tmp
and rcx, 7
add ptr, 4
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
cmp out_buf, end_ptr
ja .overflow
;; insert LL code
vpextrq sym, codes1 %+ x, 1
vpextrq tmp2, code_lens1 %+ x, 1
SHLX sym, sym, rcx
or bits, sym
add rcx, tmp2
; empty bits
mov [out_buf], bits
mov tmp, rcx
shr tmp, 3 ; byte count
add out_buf, tmp
mov tmp, rcx
and rcx, ~7
SHRX bits, bits, rcx
mov rcx, tmp
and rcx, 7
add ptr, 4
vextracti32x4 codes3 %+ x, codes4, i
vextracti32x4 code_lens3 %+ x, code_lens4, i
vextracti32x4 codes1 %+ x, codes2, i
vextracti32x4 code_lens1 %+ x, code_lens2, i
%endrep
sub end_ptr, VECTOR_SLOP
vmovq zbits %+ x, bits
vmovq zbits_count %+ x, rcx
cmp ptr, in_buf_end
jbe .main_loop
.finish:
add in_buf_end, VECTOR_LOOP_PROCESSED
add end_ptr, VECTOR_SLOP
cmp ptr, in_buf_end
jge .overflow
.finish_loop:
mov DWORD(data), [ptr]
cmp out_buf, end_ptr
ja .overflow
mov sym, data
and sym, LIT_MASK ; sym has ll_code
mov DWORD(sym), [hufftables + _lit_len_table + sym * 4]
; look up dist sym
mov dsym, data
shr dsym, DIST_OFFSET
and dsym, DIST_MASK
mov DWORD(dsym), [hufftables + _dist_table + dsym * 4]
; insert LL code
; sym: 31:24 length; 23:0 code
mov tmp2, sym
and sym, 0xFFFFFF
SHLX sym, sym, rcx
shr tmp2, 24
or bits, sym
add rcx, tmp2
; insert dist code
movzx tmp, WORD(dsym)
SHLX tmp, tmp, rcx
or bits, tmp
mov tmp, dsym
shr tmp, 24
add rcx, tmp
; insert dist extra bits
shr data, EXTRA_BITS_OFFSET
add ptr, 4
SHLX data, data, rcx
or bits, data
shr dsym, 16
and dsym, 0xFF
add rcx, dsym
; empty bits
mov [out_buf], bits
mov tmp, rcx
shr tmp, 3 ; byte count
add out_buf, tmp
mov tmp, rcx
and rcx, ~7
SHRX bits, bits, rcx
mov rcx, tmp
and rcx, 7
cmp ptr, in_buf_end
jb .finish_loop
.overflow:
mov tmp, [rsp + bitbuf_mem_offset]
mov [tmp + _m_bits], bits
mov [tmp + _m_bit_count], ecx
mov [tmp + _m_out_buf], out_buf
mov rax, ptr
FUNC_RESTORE
ret
section .data
align 64
;; 64 byte data
rot_perm:
dq 0x00000007, 0x00000000, 0x00000001, 0x00000002
dq 0x00000003, 0x00000004, 0x00000005, 0x00000006
;; 16 byte data
q_64:
dq 0x0000000000000040, 0x0000000000000000
q_8 :
dq 0x0000000000000000, 0x0000000000000008
;; 8 byte data
offset_mask:
dq 0x0000000000000007
;; 4 byte data
max_write_d:
dd 0x1c
lit_mask:
dd LIT_MASK
dist_mask:
dd DIST_MASK
lit_icr_mask:
dd 0x00ffffff
eb_icr_mask:
dd 0x000000ff
;; k mask constants
k_mask_1: dq 0x55555555
k_mask_2: dq 0xfffffff0
k_mask_3: dq 0xfffffffc
k_mask_4: dw 0x0101, 0x0101, 0x0101, 0x0101
k_mask_5: dq 0xfffffffe
%endif
Reference in New Issue View Git Blame Copy Permalink