isa-l/igzip/igzip_set_long_icf_fg_06.asm
Roy Oursler 47e914f98f igzip: Fix Windows prologue for avx 512 gen_map and set_long
Change-Id: I8e326dc7fb67f30101d03dc364ffba25242e1f67
Signed-off-by: Roy Oursler <roy.j.oursler@intel.com>
2018-03-01 13:27:48 -07:00

322 lines
8.7 KiB
NASM

%include "reg_sizes.asm"
%include "lz0a_const.asm"
%include "data_struct2.asm"
%include "igzip_compare_types.asm"
%define NEQ 4
%ifdef HAVE_AS_KNOWS_AVX512
%ifidn __OUTPUT_FORMAT__, win64
%define arg1 rcx
%define arg2 rdx
%define arg3 r8
%define dist_code rsi
%define len rdi
%else
%define arg1 rdi
%define arg2 rsi
%define arg3 rdx
%define dist_code rcx
%define len r8
%endif
%define next_in arg1
%define end_in arg2
%define match_lookup arg3
%define match_in rax
%define dist r9
%define match_offset r10
%define tmp1 r11
%define zmatch_lookup zmm0
%define zmatch_lookup2 zmm1
%define zlens zmm2
%define zdist_codes zmm3
%define zdist_extras zmm4
%define zdists zmm5
%define zdists2 zmm6
%define zlens1 zmm7
%define zlens2 zmm8
%define zlookup zmm9
%define zlookup2 zmm10
%define datas zmm11
%define ztmp1 zmm12
%define ztmp2 zmm13
%define zvect_size zmm17
%define ztwofiftyfour zmm18
%define ztwofiftysix zmm19
%define ztwosixtytwo zmm20
%define znlen_mask zmm21
%define zbswap zmm22
%define zqword_shuf zmm23
%define zdatas_perm3 zmm24
%define zdatas_perm2 zmm25
%define zincrement zmm26
%define zdists_mask zmm27
%define zdists_start zmm28
%define zlong_lens2 zmm29
%define zlong_lens zmm30
%define zlens_mask zmm31
%ifidn __OUTPUT_FORMAT__, win64
%define stack_size 8*16 + 2 * 8 + 8
%define func(x) proc_frame x
%macro FUNC_SAVE 0
alloc_stack stack_size
vmovdqa [rsp + 0*16], xmm6
vmovdqa [rsp + 1*16], xmm7
vmovdqa [rsp + 2*16], xmm8
vmovdqa [rsp + 3*16], xmm9
vmovdqa [rsp + 4*16], xmm10
vmovdqa [rsp + 5*16], xmm11
vmovdqa [rsp + 6*16], xmm12
vmovdqa [rsp + 7*16], xmm13
save_reg rsi, 8*16 + 0*8
save_reg rdi, 8*16 + 1*8
end_prolog
%endm
%macro FUNC_RESTORE 0
vmovdqa xmm6, [rsp + 0*16]
vmovdqa xmm7, [rsp + 1*16]
vmovdqa xmm8, [rsp + 2*16]
vmovdqa xmm9, [rsp + 3*16]
vmovdqa xmm10, [rsp + 4*16]
vmovdqa xmm11, [rsp + 5*16]
vmovdqa xmm12, [rsp + 6*16]
vmovdqa xmm13, [rsp + 7*16]
mov rsi, [rsp + 8*16 + 0*8]
mov rdi, [rsp + 8*16 + 1*8]
add rsp, stack_size
%endm
%else
%define func(x) x:
%macro FUNC_SAVE 0
%endm
%macro FUNC_RESTORE 0
%endm
%endif
%define VECT_SIZE 16
global set_long_icf_fg_06
func(set_long_icf_fg_06)
FUNC_SAVE
sub end_in, LA + 15
vmovdqu32 zlong_lens, [long_len]
vmovdqu32 zlong_lens2, [long_len2]
vmovdqu32 zlens_mask, [len_mask]
vmovdqu16 zdists_start, [dist_start]
vmovdqu32 zdists_mask, [dists_mask]
vmovdqu32 zincrement, [increment]
vmovdqu64 zdatas_perm2, [datas_perm2]
vmovdqu64 zdatas_perm3, [datas_perm3]
vmovdqu64 zqword_shuf, [qword_shuf]
vmovdqu64 zbswap, [bswap_shuf]
vmovdqu64 znlen_mask, [nlen_mask]
vmovdqu64 zvect_size, [vect_size]
vmovdqu64 ztwofiftyfour, [twofiftyfour]
vmovdqu64 ztwofiftysix, [twofiftysix]
vmovdqu64 ztwosixtytwo, [twosixtytwo]
vmovdqu32 zmatch_lookup, [match_lookup]
fill_loop: ; Tahiti is a magical place
vmovdqu32 zmatch_lookup2, zmatch_lookup
vmovdqu32 zmatch_lookup, [match_lookup + ICF_CODE_BYTES * VECT_SIZE]
cmp next_in, end_in
jae end_fill
vpandd zlens, zmatch_lookup2, zlens_mask
vpcmpgtd k3, zlens, zlong_lens
;; Speculatively increment
add next_in, VECT_SIZE
add match_lookup, ICF_CODE_BYTES * VECT_SIZE
ktestw k3, k3
jz fill_loop
vpsrld zdist_codes, zmatch_lookup2, DIST_OFFSET
vpmovdw zdists %+ y, zdist_codes ; Relies on perm working mod 32
vpermw zdists, zdists, zdists_start
vpmovzxwd zdists, zdists %+ y
vpsrld zdist_extras, zmatch_lookup2, EXTRA_BITS_OFFSET
vpsubd zdist_extras, zincrement, zdist_extras
vpsubd zdists, zdist_extras, zdists
vextracti32x8 zdists2 %+ y, zdists, 1
kmovb k6, k3
kshiftrw k7, k3, 8
vpgatherdq zlens1 {k6}, [next_in + zdists %+ y - 8]
vpgatherdq zlens2 {k7}, [next_in + zdists2 %+ y - 8]
vmovdqu8 datas %+ y, [next_in - 8]
vpermq zlookup, zdatas_perm2, datas
vpshufb zlookup, zlookup, zqword_shuf
vpermq zlookup2, zdatas_perm3, datas
vpshufb zlookup2, zlookup2, zqword_shuf
vpxorq zlens1, zlens1, zlookup
vpxorq zlens2, zlens2, zlookup2
vpshufb zlens1, zlens1, zbswap
vpshufb zlens2, zlens2, zbswap
vplzcntq zlens1, zlens1
vplzcntq zlens2, zlens2
vpmovqd zlens1 %+ y, zlens1
vpmovqd zlens2 %+ y, zlens2
vinserti32x8 zlens1, zlens2 %+ y, 1
vpsrld zlens1 {k3}{z}, zlens1, 3
vpandd zmatch_lookup2 {k3}{z}, zmatch_lookup2, znlen_mask
vpaddd zmatch_lookup2 {k3}{z}, zmatch_lookup2, ztwosixtytwo
vpaddd zmatch_lookup2 {k3}{z}, zmatch_lookup2, zlens1
vmovdqu32 [match_lookup - ICF_CODE_BYTES * VECT_SIZE] {k3}, zmatch_lookup2
vpcmpgtd k3, zlens1, zlong_lens2
ktestw k3, k3
jz fill_loop
vpsubd zdists, zincrement, zdists
vpcompressd zdists2 {k3}, zdists
vpcompressd zmatch_lookup2 {k3}, zmatch_lookup2
kmovq match_offset, k3
tzcnt match_offset, match_offset
vmovd dist %+ d, zdists2 %+ x
lea next_in, [next_in + match_offset - VECT_SIZE]
lea match_lookup, [match_lookup + ICF_CODE_BYTES * (match_offset - VECT_SIZE)]
mov match_in, next_in
sub match_in, dist
mov len, 2
%rep 3
vmovdqu8 ztmp1, [next_in + len]
vmovdqu8 ztmp2, [match_in + len]
vpcmpb k3, ztmp1, [match_in + len], NEQ
ktestq k3, k3
jnz miscompare
add len, 64
%endrep
vmovdqu8 ztmp1, [next_in + len]
vmovdqu8 ztmp2, [match_in + len]
vpcmpb k3, ztmp1, ztmp2, 4
miscompare:
kmovq tmp1, k3
tzcnt tmp1, tmp1
add len, tmp1
add next_in, len
lea match_lookup, [match_lookup + ICF_CODE_BYTES * len]
vmovdqu32 zmatch_lookup, [match_lookup]
vpbroadcastd zmatch_lookup2, zmatch_lookup2 %+ x
vpandd zmatch_lookup2, zmatch_lookup2, znlen_mask
vpbroadcastd zlens1, len %+ d
vpsubd zlens1, zlens1, zincrement
vpaddd zlens1, zlens1, ztwofiftyfour
neg len
update_match_lookup:
vpandd zlens2, zlens_mask, [match_lookup + ICF_CODE_BYTES * len]
vpcmpgtd k3, zlens1, zlens2
vpcmpgtd k4, zlens1, ztwofiftysix
kandw k3, k3, k4
vpaddd zlens2 {k3}{z}, zlens1, zmatch_lookup2
vmovdqu32 [match_lookup + ICF_CODE_BYTES * len] {k3}, zlens2
knotw k3, k3
ktestw k3, k3
jnz fill_loop
add len, VECT_SIZE
vpsubd zlens1, zlens1, zvect_size
jmp update_match_lookup
end_fill:
FUNC_RESTORE
ret
endproc_frame
section .data
align 64
dist_start:
dw 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0007, 0x0009, 0x000d
dw 0x0011, 0x0019, 0x0021, 0x0031, 0x0041, 0x0061, 0x0081, 0x00c1
dw 0x0101, 0x0181, 0x0201, 0x0301, 0x0401, 0x0601, 0x0801, 0x0c01
dw 0x1001, 0x1801, 0x2001, 0x3001, 0x4001, 0x6001, 0x0000, 0x0000
len_mask:
dd LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK
dd LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK
dd LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK
dd LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK, LIT_LEN_MASK
dists_mask:
dd LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK
dd LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK
dd LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK
dd LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK, LIT_DIST_MASK
long_len:
dd 0x105, 0x105, 0x105, 0x105, 0x105, 0x105, 0x105, 0x105
dd 0x105, 0x105, 0x105, 0x105, 0x105, 0x105, 0x105, 0x105
long_len2:
dd 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7
dd 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7
increment:
dd 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
dd 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
datas_perm2:
dq 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1
datas_perm3:
dq 0x1, 0x2, 0x1, 0x2, 0x1, 0x2, 0x1, 0x2
bswap_shuf:
db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
qword_shuf:
db 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
db 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8
db 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9
db 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa
db 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb
db 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc
db 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd
db 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe
db 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
vect_size:
dd VECT_SIZE, VECT_SIZE, VECT_SIZE, VECT_SIZE
dd VECT_SIZE, VECT_SIZE, VECT_SIZE, VECT_SIZE
dd VECT_SIZE, VECT_SIZE, VECT_SIZE, VECT_SIZE
dd VECT_SIZE, VECT_SIZE, VECT_SIZE, VECT_SIZE
twofiftyfour:
dd 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe
dd 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe
twofiftysix:
dd 0x100, 0x100, 0x100, 0x100, 0x100, 0x100, 0x100, 0x100
dd 0x100, 0x100, 0x100, 0x100, 0x100, 0x100, 0x100, 0x100
twosixtytwo:
dd 0x106, 0x106, 0x106, 0x106, 0x106, 0x106, 0x106, 0x106
dd 0x106, 0x106, 0x106, 0x106, 0x106, 0x106, 0x106, 0x106
nlen_mask:
dd 0xfffffc00, 0xfffffc00, 0xfffffc00, 0xfffffc00
dd 0xfffffc00, 0xfffffc00, 0xfffffc00, 0xfffffc00
dd 0xfffffc00, 0xfffffc00, 0xfffffc00, 0xfffffc00
dd 0xfffffc00, 0xfffffc00, 0xfffffc00, 0xfffffc00
%endif