isa-l/igzip/igzip_icf_body.asm

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;  Copyright(c) 2011-2016 Intel Corporation All rights reserved.
;
;  Redistribution and use in source and binary forms, with or without
;  modification, are permitted provided that the following conditions
;  are met:
;    * Redistributions of source code must retain the above copyright
;      notice, this list of conditions and the following disclaimer.
;    * Redistributions in binary form must reproduce the above copyright
;      notice, this list of conditions and the following disclaimer in
;      the documentation and/or other materials provided with the
;      distribution.
;    * Neither the name of Intel Corporation nor the names of its
;      contributors may be used to endorse or promote products derived
;      from this software without specific prior written permission.
;
;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%include "options.asm"

%include "lz0a_const.asm"
%include "data_struct2.asm"
%include "bitbuf2.asm"
%include "huffman.asm"
%include "igzip_compare_types.asm"
%include "reg_sizes.asm"

%include "stdmac.asm"

%ifdef DEBUG
%macro MARK 1
global %1
%1:
%endm
%else
%macro MARK 1
%endm
%endif

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

%define	tmp2		rcx
%define	hash2		rcx

%define	curr_data	rax
%define	code		rax
%define	tmp5		rax

%define	tmp4		rbx
%define	dist		rbx
%define	code2		rbx

%define	hash		rdx
%define	len		rdx
%define	code_len3	rdx
%define	tmp8		rdx

%define	tmp1		rsi
%define	code_len2	rsi
%define	code5		rsi

%define	file_start	rdi

%define	curr_data2	r8
%define	len2		r8
%define	tmp6		r8

%define	f_i		r10

%define	m_out_buf	r11

%define	f_end_i		r12
%define	dist2		r12
%define	tmp7		r12
%define	code4		r12

%define	tmp3		r13
%define	code3		r13

%define	stream		r14

%define	hufftables	r15

;; GPR r8 & r15 can be used

%define xtmp0		xmm0	; tmp
%define xtmp1		xmm1	; tmp
%define	xhash		xmm2
%define	xmask		xmm3
%define	xdata		xmm4

%define ytmp0		ymm0	; tmp
%define ytmp1		ymm1	; tmp


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;


m_out_end           equ  0	 ; local variable (8 bytes)
m_out_start         equ	 8
f_end_i_mem_offset  equ 16
gpr_save_mem_offset equ 24       ; gpr save area (8*8 bytes)
xmm_save_mem_offset equ 24 + 8*8 ; xmm save area (4*16 bytes) (16 byte aligned)
stack_size          equ 3*8 + 8*8 + 4*16
;;; 8 because stack address is odd multiple of 8 after a function call and
;;; we want it aligned to 16 bytes

; void isal_deflate_icf_body ( isal_zstream *stream )
; arg 1: rcx: addr of stream
global isal_deflate_icf_body_ %+ ARCH
isal_deflate_icf_body_ %+ ARCH %+ :
%ifidn __OUTPUT_FORMAT__, elf64
	mov	rcx, rdi
%endif

	;; do nothing if (avail_in == 0)
	cmp	dword [rcx + _avail_in], 0
	jne	skip1

	;; Set stream's next state
	mov	rdx, ZSTATE_FLUSH_READ_BUFFER
	mov	rax, ZSTATE_CREATE_HDR
	cmp	dword [rcx + _end_of_stream], 0
	cmovne	rax, rdx
	cmp	dword [rcx + _flush], _NO_FLUSH
	cmovne	rax, rdx
	mov	dword [rcx + _internal_state_state], eax
	ret
skip1:

%ifdef ALIGN_STACK
	push	rbp
	mov	rbp, rsp
	sub	rsp, stack_size
	and	rsp, ~15
%else
	sub	rsp, stack_size
%endif

	mov [rsp + gpr_save_mem_offset + 0*8], rbx
	mov [rsp + gpr_save_mem_offset + 1*8], rsi
	mov [rsp + gpr_save_mem_offset + 2*8], rdi
	mov [rsp + gpr_save_mem_offset + 3*8], rbp
	mov [rsp + gpr_save_mem_offset + 4*8], r12
	mov [rsp + gpr_save_mem_offset + 5*8], r13
	mov [rsp + gpr_save_mem_offset + 6*8], r14
	mov [rsp + gpr_save_mem_offset + 7*8], r15

	mov	stream, rcx
	mov	dword [stream + _internal_state_has_eob], 0

	MOVDQU	xmask, [mask]

	; state->bitbuf.set_buf(stream->next_out, stream->avail_out);
	mov	tmp1, [stream + _level_buf]
	mov	m_out_buf, [tmp1 + _icf_buf_next]

	mov     [rsp + m_out_start], m_out_buf
	mov	tmp1, [tmp1 + _icf_buf_avail_out]
	add	tmp1, m_out_buf
	sub	tmp1, SLOP

	mov	[rsp + m_out_end], tmp1

	mov	hufftables, [stream + _hufftables]

	mov	file_start, [stream + _next_in]

	mov	f_i %+ d, dword [stream + _total_in]
	sub	file_start, f_i

	mov	f_end_i %+ d, [stream + _avail_in]
	add	f_end_i, f_i

	; f_end_i -= LA;
	sub	f_end_i, LA
	mov	[rsp + f_end_i_mem_offset], f_end_i
	; if (f_end_i <= 0) continue;

	cmp	f_end_i, f_i
	jle	input_end

	; for (f_i = f_start_i; f_i < f_end_i; f_i++) {
MARK __body_compute_hash_ %+ ARCH
	MOVDQU	xdata, [file_start + f_i]
	mov	curr_data, [file_start + f_i]
	mov	tmp3, curr_data
	mov	tmp6, curr_data

	compute_hash	hash, curr_data

	shr	tmp3, 8
	compute_hash	hash2, tmp3

	and	hash, HASH_MASK
	and	hash2, HASH_MASK

	cmp	dword [stream + _internal_state_has_hist], 0
	je	write_first_byte

	jmp	loop2
	align	16

loop2:
	; if (state->bitbuf.is_full()) {
	cmp	m_out_buf, [rsp + m_out_end]
	ja	output_end

	xor	dist, dist
	xor	dist2, dist2
	xor	tmp3, tmp3

	lea	tmp1, [file_start + f_i]

	mov	dist %+ w, f_i %+ w
	dec	dist
	sub	dist %+ w, word [stream + _internal_state_head + 2 * hash]
	mov	[stream + _internal_state_head + 2 * hash], f_i %+ w

	inc	f_i

	MOVQ	tmp6, xdata
	shr	tmp5, 16
	mov	tmp8, tmp5
	compute_hash	tmp6, tmp5

	mov	dist2 %+ w, f_i %+ w
	dec	dist2
	sub	dist2 %+ w, word [stream + _internal_state_head + 2 * hash2]
	mov	[stream + _internal_state_head + 2 * hash2], f_i %+ w

	; if ((dist-1) < (D-1)) {
	and	dist %+ d, (D-1)
	neg	dist

	shr	tmp8, 8
	compute_hash	tmp2, tmp8

	and	dist2 %+ d, (D-1)
	neg	dist2

MARK __body_compare_ %+ ARCH
	;; Check for long len/dist match (>7) with first literal
	MOVQ	len, xdata
	mov	curr_data, len
	PSRLDQ	xdata, 1
	xor	len, [tmp1 + dist - 1]
	jz	compare_loop

	MOVD	xhash, tmp6 %+ d
	PINSRD	xhash, tmp2 %+ d, 1
	PAND	xhash, xhash, xmask

	;; Check for len/dist match (>7) with second literal
	MOVQ	len2, xdata
	xor	len2, [tmp1 + dist2]
	jz	compare_loop2

	;; Check for len/dist match for first literal
	test    len %+ d, 0xFFFFFFFF
	jz      len_dist_huffman_pre

	;; Check for len/dist match for second literal
	test    len2 %+ d, 0xFFFFFFFF
	jnz     write_lit_bits

MARK __body_len_dist_lit_huffman_ %+ ARCH
len_dist_lit_huffman_pre:
	bsf	len2, len2
	shr	len2, 3

len_dist_lit_huffman:
	neg	dist2

%ifndef LONGER_HUFFTABLE
	mov	tmp4, dist2
	get_dist_icf_code	tmp4, code4, tmp1 ;; clobbers dist, rcx
%else
	get_dist_icf_code	dist2, code4, tmp1
%endif

	movzx	code5, curr_data %+ b
	;; get_len_code
	add	f_i, len2
	neg	len2

	MOVQ	tmp5, xdata
	shr	tmp5, 24
	compute_hash	tmp4, tmp5
	and	tmp4, HASH_MASK

	;; Setup for updating hash
	lea	tmp3, [f_i + len2 + 1]	; tmp3 <= k

	MOVDQU	xdata, [file_start + f_i]
	mov	curr_data, [file_start + f_i]

	MOVD	hash %+ d, xhash
	PEXTRD	hash2 %+ d, xhash, 1
	mov	[stream + _internal_state_head + 2 * hash], tmp3 %+ w

	compute_hash	hash, curr_data

	add	tmp3,1
	mov	[stream + _internal_state_head + 2 * hash2], tmp3 %+ w

	add	tmp3, 1
	mov	[stream + _internal_state_head + 2 * tmp4], tmp3 %+ w

	neg	len2
	add	len2, 254
	or	code4, len2

	inc	word [stream + _internal_state_hist_lit_len + 2*code5]
	or	code5, LIT
	inc	word [stream + _internal_state_hist_lit_len + 2*len2]
	write_dword	code5, m_out_buf
	write_dword	code4, m_out_buf
	shr	code4, DIST_OFFSET
	and	code4, 0x1F
	inc	word [stream + _internal_state_hist_dist + 2*code4]


	mov	f_end_i, [rsp + f_end_i_mem_offset]

	mov	curr_data2, curr_data
	shr	curr_data2, 8
	compute_hash	hash2, curr_data2

	; hash = compute_hash(state->file_start + f_i) & HASH_MASK;
	and	hash %+ d, HASH_MASK
	and	hash2 %+ d, HASH_MASK

	; continue
	cmp	f_i, f_end_i
	jl	loop2
	jmp	input_end
	;; encode as dist/len

MARK __body_len_dist_huffman_ %+ ARCH
len_dist_huffman_pre:
	bsf	len, len
	shr	len, 3

len_dist_huffman:
	dec	f_i
	neg	dist

	; get_dist_code(dist, &code2, &code_len2);
%ifndef LONGER_HUFFTABLE
	mov tmp3, dist  ; since code2 and dist are rbx
	get_dist_icf_code   tmp3, code2, tmp1 ;; clobbers dist, rcx
%else
	get_dist_icf_code   dist, code2, tmp1
%endif

	; get_len_code(len, &code, &code_len);
	lea	code5, [len + 254]

	or	code2, code5

	;; Setup for updateing hash
	lea	tmp3, [f_i + 2]	; tmp3 <= k
	add	f_i, len

	MOVD	hash %+ d, xhash
	PEXTRD	hash2 %+ d, xhash, 1
	mov	[stream + _internal_state_head + 2 * hash], tmp3 %+ w
	add	tmp3,1
	mov	[stream + _internal_state_head + 2 * hash2], tmp3 %+ w

	MOVDQU	xdata, [file_start + f_i]
	mov	curr_data, [file_start + f_i]
	mov	curr_data2, curr_data
	compute_hash	hash, curr_data

	inc	word [stream + _internal_state_hist_lit_len + 2*code5]
	write_dword	code2, m_out_buf
	shr	code2, DIST_OFFSET
	and	code2, 0x1F
	inc	word [stream + _internal_state_hist_dist + 2*code2]

	mov	f_end_i, [rsp + f_end_i_mem_offset]

	shr	curr_data2, 8
	compute_hash	hash2, curr_data2

	; hash = compute_hash(state->file_start + f_i) & HASH_MASK;
	and	hash %+ d, HASH_MASK
	and	hash2 %+ d, HASH_MASK

	; continue
	cmp	f_i, f_end_i
	jl	loop2
	jmp	input_end

MARK __body_write_lit_bits_ %+ ARCH
write_lit_bits:
	movzx	code3, curr_data %+ b
	shr	curr_data, 8
	and	curr_data, 0xff
	mov	code2, curr_data

	MOVDQU	xdata, [file_start + f_i + 1]
	mov	f_end_i, [rsp + f_end_i_mem_offset]
	add	f_i, 1
	mov	curr_data, [file_start + f_i]

	MOVD	hash %+ d, xhash

	inc	word [stream + _internal_state_hist_lit_len + 2*code3]
	or	code3, LIT
	inc	word [stream + _internal_state_hist_lit_len + 2*code2]
	or	code2, LIT
	write_dword	code3, m_out_buf
	write_dword	code2, m_out_buf

	PEXTRD	hash2 %+ d, xhash, 1

	and	hash %+ d, HASH_MASK
	and	hash2 %+ d, HASH_MASK

	; continue
	cmp	f_i, f_end_i
	jl	loop2

input_end:
	mov	tmp1, ZSTATE_FLUSH_READ_BUFFER
	mov	tmp5, ZSTATE_BODY
	cmp	dword [stream + _end_of_stream], 0
	cmovne	tmp5, tmp1
	cmp	dword [stream + _flush], _NO_FLUSH
	cmovne	tmp5, tmp1
	mov	dword [stream + _internal_state_state], tmp5 %+ d
	jmp	end

output_end:
	mov	dword [stream + _internal_state_state], ZSTATE_CREATE_HDR

end:
	;; update input buffer
	add	f_end_i, LA
	mov	[stream + _total_in], f_i %+ d
	add	file_start, f_i
	mov     [stream + _next_in], file_start
	sub	f_end_i, f_i
	mov     [stream + _avail_in], f_end_i %+ d

	;; update output buffer
	mov	tmp1, [stream + _level_buf]
	mov	[tmp1 + _icf_buf_next], m_out_buf
	sub	m_out_buf, [rsp + m_out_start]
	sub	[tmp1 + _icf_buf_avail_out], m_out_buf %+ d

	mov rbx, [rsp + gpr_save_mem_offset + 0*8]
	mov rsi, [rsp + gpr_save_mem_offset + 1*8]
	mov rdi, [rsp + gpr_save_mem_offset + 2*8]
	mov rbp, [rsp + gpr_save_mem_offset + 3*8]
	mov r12, [rsp + gpr_save_mem_offset + 4*8]
	mov r13, [rsp + gpr_save_mem_offset + 5*8]
	mov r14, [rsp + gpr_save_mem_offset + 6*8]
	mov r15, [rsp + gpr_save_mem_offset + 7*8]

%ifndef ALIGN_STACK
	add	rsp, stack_size
%else
	mov	rsp, rbp
	pop	rbp
%endif
	ret

MARK __body_compare_loops_ %+ ARCH
compare_loop:
	MOVD	xhash, tmp6 %+ d
	PINSRD	xhash, tmp2 %+ d, 1
	PAND	xhash, xhash, xmask
	lea	tmp2, [tmp1 + dist - 1]
%if (COMPARE_TYPE == 1)
	compare250	tmp1, tmp2, len, tmp3
%elif (COMPARE_TYPE == 2)
	compare250_x	tmp1, tmp2, len, tmp3, xtmp0, xtmp1
%elif (COMPARE_TYPE == 3)
	compare250_y	tmp1, tmp2, len, tmp3, ytmp0, ytmp1
%else
	%error Unknown Compare type COMPARE_TYPE
	 % error
%endif
	jmp	len_dist_huffman

compare_loop2:
	lea	tmp2, [tmp1 + dist2]
	add	tmp1, 1
%if (COMPARE_TYPE == 1)
	compare250	tmp1, tmp2, len2, tmp3
%elif (COMPARE_TYPE == 2)
	compare250_x	tmp1, tmp2, len2, tmp3, xtmp0, xtmp1
%elif (COMPARE_TYPE == 3)
	compare250_y	tmp1, tmp2, len2, tmp3, ytmp0, ytmp1
%else
%error Unknown Compare type COMPARE_TYPE
 % error
%endif
	jmp	len_dist_lit_huffman

MARK __write_first_byte_ %+ ARCH
write_first_byte:
	cmp	m_out_buf, [rsp + m_out_end]
	ja	output_end

	mov	dword [stream + _internal_state_has_hist], 1

	mov	[stream + _internal_state_head + 2 * hash], f_i %+ w

	mov	hash, hash2
	shr	tmp6, 16
	compute_hash	hash2, tmp6

	and	curr_data, 0xff
	inc	word [stream + _internal_state_hist_lit_len + 2*curr_data]
	or	curr_data, LIT

	write_dword	curr_data, m_out_buf
	MOVDQU	xdata, [file_start + f_i + 1]
	add	f_i, 1
	mov	curr_data, [file_start + f_i]
	and	hash %+ d, HASH_MASK
	and	hash2 %+ d, HASH_MASK

	cmp	f_i, [rsp + f_end_i_mem_offset]
	jl	loop2
	jmp	input_end

section .data
	align 16
mask:	dd	HASH_MASK, HASH_MASK, HASH_MASK, HASH_MASK
const_D: dq	D