isa-l/igzip/igzip_set_long_icf_fg_06.asm

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;  Copyright(c) 2011-2018 Intel Corporation All rights reserved.
;
;  Redistribution and use in source and binary forms, with or without
;  modification, are permitted provided that the following conditions
;  are met:
;    * Redistributions of source code must retain the above copyright
;      notice, this list of conditions and the following disclaimer.
;    * Redistributions in binary form must reproduce the above copyright
;      notice, this list of conditions and the following disclaimer in
;      the documentation and/or other materials provided with the
;      distribution.
;    * Neither the name of Intel Corporation nor the names of its
;      contributors may be used to endorse or promote products derived
;      from this software without specific prior written permission.
;
;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

%include "reg_sizes.asm"
%include "lz0a_const.asm"
%include "data_struct2.asm"
%include "igzip_compare_types.asm"
%define NEQ 4

%ifdef HAVE_AS_KNOWS_AVX512
%ifidn __OUTPUT_FORMAT__, win64
%define arg1 rcx
%define arg2 rdx
%define arg3 r8
%define arg4 r9
%define len rdi
%define dist rsi
%else
%define arg1 rdi
%define arg2 rsi
%define arg3 rdx
%define arg4 rcx
%define len r8
%define dist r9
%endif

%define next_in arg1
%define end_processed arg2
%define end_in arg3
%define match_lookup arg4
%define match_in rax
%define match_offset r10
%define tmp1 r11
%define end_processed_orig r12
%define dist_code r13
%define tmp2 r13

%define zmatch_lookup zmm0
%define zmatch_lookup2 zmm1
%define zlens zmm2
%define zdist_codes zmm3
%define zdist_extras zmm4
%define zdists zmm5
%define zdists2 zmm6
%define zlens1 zmm7
%define zlens2 zmm8
%define zlookup zmm9
%define zlookup2 zmm10
%define datas zmm11
%define ztmp1 zmm12
%define ztmp2 zmm13
%define zvect_size zmm16
%define zmax_len zmm17
%define ztwofiftyfour zmm18
%define ztwofiftysix zmm19
%define ztwosixtytwo zmm20
%define znlen_mask zmm21
%define zbswap zmm22
%define zqword_shuf zmm23
%define zdatas_perm3 zmm24
%define zdatas_perm2 zmm25
%define zincrement zmm26
%define zdists_mask zmm27
%define zdists_start zmm28
%define zlong_lens2 zmm29
%define zlong_lens zmm30
%define zlens_mask zmm31

%ifidn __OUTPUT_FORMAT__, win64
%define stack_size  8*16 + 4 * 8 + 8
%define func(x) proc_frame x
%macro FUNC_SAVE 0
	alloc_stack	stack_size
	vmovdqa	[rsp + 0*16], xmm6
	vmovdqa	[rsp + 1*16], xmm7
	vmovdqa	[rsp + 2*16], xmm8
	vmovdqa	[rsp + 3*16], xmm9
	vmovdqa	[rsp + 4*16], xmm10
	vmovdqa	[rsp + 5*16], xmm11
	vmovdqa	[rsp + 6*16], xmm12
	vmovdqa	[rsp + 7*16], xmm13
	save_reg	rsi, 8*16 + 0*8
	save_reg	rdi, 8*16 + 1*8
	save_reg	r12, 8*16 + 2*8
	save_reg	r13, 8*16 + 3*8
	end_prolog
%endm

%macro FUNC_RESTORE 0
	vmovdqa	xmm6, [rsp + 0*16]
	vmovdqa	xmm7, [rsp + 1*16]
	vmovdqa	xmm8, [rsp + 2*16]
	vmovdqa	xmm9, [rsp + 3*16]
	vmovdqa	xmm10, [rsp + 4*16]
	vmovdqa	xmm11, [rsp + 5*16]
	vmovdqa	xmm12, [rsp + 6*16]
	vmovdqa	xmm13, [rsp + 7*16]

	mov	rsi, [rsp + 8*16 + 0*8]
	mov	rdi, [rsp + 8*16 + 1*8]
	mov	r12, [rsp + 8*16 + 2*8]
	mov	r13, [rsp + 8*16 + 3*8]
	add	rsp, stack_size
%endm
%else
%define func(x) x:
%macro FUNC_SAVE 0
	push	r12
	push	r13
%endm

%macro FUNC_RESTORE 0
	pop	r13
	pop	r12
%endm
%endif
%define VECT_SIZE 16

global set_long_icf_fg_06
func(set_long_icf_fg_06)
	FUNC_SAVE

	lea	end_in, [next_in + arg3]
	add	end_processed, next_in
	mov	end_processed_orig, end_processed
	lea	tmp1, [end_processed + LA_STATELESS]
	cmp	end_in, tmp1
	cmovg	end_in, tmp1
	sub	end_processed, 15
	vpbroadcastd zlong_lens, [long_len]
	vpbroadcastd zlong_lens2, [long_len2]
	vpbroadcastd zlens_mask, [len_mask]
	vmovdqu16 zdists_start, [dist_start]
	vpbroadcastd zdists_mask, [dists_mask]
	vmovdqu32 zincrement, [increment]
	vbroadcasti64x2 zdatas_perm2, [datas_perm2]
	vbroadcasti64x2 zdatas_perm3, [datas_perm3]
	vmovdqu64 zqword_shuf, [qword_shuf]
	vbroadcasti64x2 zbswap, [bswap_shuf]
	vpbroadcastd znlen_mask, [nlen_mask]
	vpbroadcastd zvect_size, [vect_size]
	vpbroadcastd zmax_len, [max_len]
	vpbroadcastd ztwofiftyfour, [twofiftyfour]
	vpbroadcastd ztwofiftysix, [twofiftysix]
	vpbroadcastd ztwosixtytwo, [twosixtytwo]
	vmovdqu32 zmatch_lookup, [match_lookup]

.fill_loop: ; Tahiti is a magical place
	vmovdqu32 zmatch_lookup2, zmatch_lookup
	vmovdqu32 zmatch_lookup, [match_lookup + ICF_CODE_BYTES * VECT_SIZE]

	cmp	next_in, end_processed
	jae	.end_fill

.finish_entry:
	vpandd	zlens, zmatch_lookup2, zlens_mask
	vpcmpgtd k3, zlens, zlong_lens

;; Speculatively increment
	add	next_in, VECT_SIZE
	add	match_lookup, ICF_CODE_BYTES * VECT_SIZE

	ktestw	k3, k3
	jz	.fill_loop

	vpsrld	zdist_codes, zmatch_lookup2, DIST_OFFSET
	vpmovdw	zdists %+ y, zdist_codes ; Relies on perm working mod 32
	vpermw	zdists, zdists, zdists_start
	vpmovzxwd zdists, zdists %+ y

	vpsrld	zdist_extras, zmatch_lookup2, EXTRA_BITS_OFFSET
	vpsubd	zdist_extras, zincrement, zdist_extras

	vpsubd	zdists, zdist_extras, zdists
	vextracti32x8 zdists2 %+ y, zdists, 1
	kmovb	k6, k3
	kshiftrw k7, k3, 8
	vpgatherdq zlens1 {k6}, [next_in + zdists %+ y - 8]
	vpgatherdq zlens2 {k7}, [next_in + zdists2 %+ y - 8]

	vmovdqu8 datas %+ y, [next_in - 8]
	vpermq	zlookup, zdatas_perm2, datas
	vpshufb	zlookup, zlookup, zqword_shuf
	vpermq	zlookup2, zdatas_perm3, datas
	vpshufb	zlookup2, zlookup2, zqword_shuf

	vpxorq	zlens1, zlens1, zlookup
	vpxorq	zlens2, zlens2, zlookup2

	vpshufb	zlens1, zlens1, zbswap
	vpshufb	zlens2, zlens2, zbswap
	vplzcntq zlens1, zlens1
	vplzcntq zlens2, zlens2
	vpmovqd	zlens1 %+ y, zlens1
	vpmovqd	zlens2 %+ y, zlens2
	vinserti32x8 zlens1, zlens2 %+ y, 1
	vpsrld	zlens1 {k3}{z}, zlens1, 3

	vpandd	zmatch_lookup2 {k3}{z}, zmatch_lookup2, znlen_mask
	vpaddd	zmatch_lookup2 {k3}{z}, zmatch_lookup2, ztwosixtytwo
	vpaddd	zmatch_lookup2 {k3}{z}, zmatch_lookup2, zlens1

	vmovdqu32 [match_lookup - ICF_CODE_BYTES * VECT_SIZE] {k3}, zmatch_lookup2

	vpcmpgtd k3, zlens1, zlong_lens2
	ktestw	k3, k3
	jz	.fill_loop

	vpsubd	zdists, zincrement, zdists

	vpcompressd zdists2 {k3}, zdists
	vpcompressd zmatch_lookup2 {k3}, zmatch_lookup2
	kmovq	match_offset, k3
	tzcnt	match_offset, match_offset

	vmovd	dist %+ d, zdists2 %+ x
	lea	next_in, [next_in + match_offset - VECT_SIZE]
	lea	match_lookup, [match_lookup + ICF_CODE_BYTES * (match_offset - VECT_SIZE)]
	mov	match_in, next_in
	sub	match_in, dist

	mov	len, 16
	mov	tmp2, end_in
	sub	tmp2, next_in

	compare_z next_in, match_in, len, tmp2, tmp1, k3, ztmp1, ztmp2

	vpbroadcastd zlens1, len %+ d
	vpsubd	zlens1, zlens1, zincrement
	vpaddd	zlens1, zlens1, ztwofiftyfour

	mov	tmp2, end_processed
	sub	tmp2, next_in
	cmp	len, tmp2
	cmovg	len, tmp2

	add	next_in, len
	lea	match_lookup, [match_lookup + ICF_CODE_BYTES * len]
	vmovdqu32 zmatch_lookup, [match_lookup]

	vpbroadcastd zmatch_lookup2, zmatch_lookup2 %+ x
	vpandd	zmatch_lookup2, zmatch_lookup2, znlen_mask

	neg	len

.update_match_lookup:
	vpandd	zlens2, zlens_mask, [match_lookup + ICF_CODE_BYTES * len]
	vpcmpgtd k3, zlens1, zlens2
	vpcmpgtd k4, zlens1, ztwofiftysix
	kandw	k3, k3, k4

	vpcmpgtd k4, zlens1, zmax_len
	vmovdqu32 zlens, zlens1
	vmovdqu32 zlens {k4}, zmax_len

	vpaddd	zlens2 {k3}{z}, zlens, zmatch_lookup2

	vmovdqu32 [match_lookup + ICF_CODE_BYTES * len] {k3}, zlens2

	knotw	k3, k3
	ktestw	k3, k3
	jnz	.fill_loop

	add	len, VECT_SIZE
	vpsubd	zlens1, zlens1, zvect_size

	jmp	.update_match_lookup

.end_fill:
	mov	end_processed, end_processed_orig
	cmp	next_in, end_processed
	jge	.finish

	mov	tmp1, end_processed
	sub	tmp1, next_in
	vpbroadcastd ztmp1, tmp1 %+ d
	vpcmpd k3, ztmp1, zincrement, 6
	vmovdqu32 zmatch_lookup2 {k3}{z}, zmatch_lookup2
	jmp	.finish_entry

.finish:

	FUNC_RESTORE
	ret

endproc_frame

section .data
align 64
;; 64 byte data
dist_start:
	dw 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0007, 0x0009, 0x000d
	dw 0x0011, 0x0019, 0x0021, 0x0031, 0x0041, 0x0061, 0x0081, 0x00c1
	dw 0x0101, 0x0181, 0x0201, 0x0301, 0x0401, 0x0601, 0x0801, 0x0c01
	dw 0x1001, 0x1801, 0x2001, 0x3001, 0x4001, 0x6001, 0x0000, 0x0000
qword_shuf:
	db 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
	db 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8
	db 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9
	db 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa
	db 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb
	db 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc
	db 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd
	db 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe
	db 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf

;; 16 byte data
increment:
	dd 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
	dd 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf

datas_perm2:
	dq 0x0, 0x1
datas_perm3:
	dq 0x1, 0x2
bswap_shuf:
	db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
	db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08

;; 4 byte data
len_mask:
	dd LIT_LEN_MASK
dists_mask:
	dd LIT_DIST_MASK
long_len:
	dd 0x105
long_len2:
	dd 0x7
max_len:
	dd 0xfe + 0x102
vect_size:
	dd VECT_SIZE
twofiftyfour:
	dd 0xfe
twofiftysix:
	dd 0x100
twosixtytwo:
	dd 0x106
nlen_mask:
	dd 0xfffffc00
%endif