From 77e43ef6cf2da292fa2d5639e663de5e1543f4f5 Mon Sep 17 00:00:00 2001 From: Roy Oursler Date: Wed, 21 Sep 2016 12:33:51 -0700 Subject: [PATCH] igzip: Set isal_deflate_stateless to use same assembly functions as isal_deflate Change-Id: I813b620286c2f74fabff6937fbceddb65377f40c Signed-off-by: Roy Oursler --- igzip/Makefile.am | 5 - igzip/igzip.c | 70 ++-- igzip/igzip_multibinary.asm | 8 - igzip/igzip_stateless.asm | 636 ----------------------------------- igzip/igzip_stateless_01.asm | 7 - igzip/igzip_stateless_02.asm | 7 - igzip/igzip_stateless_04.asm | 8 - igzip/igzip_stateless_base.c | 151 --------- 8 files changed, 31 insertions(+), 861 deletions(-) delete mode 100644 igzip/igzip_stateless.asm delete mode 100644 igzip/igzip_stateless_01.asm delete mode 100644 igzip/igzip_stateless_02.asm delete mode 100644 igzip/igzip_stateless_04.asm delete mode 100644 igzip/igzip_stateless_base.c diff --git a/igzip/Makefile.am b/igzip/Makefile.am index 9dcc5a4..85e0e85 100644 --- a/igzip/Makefile.am +++ b/igzip/Makefile.am @@ -32,14 +32,10 @@ lsrc += igzip/igzip.c igzip/hufftables_c.c \ igzip/igzip_body_02.asm \ igzip/igzip_body_04.asm \ igzip/igzip_finish.asm \ - igzip/igzip_stateless_01.asm \ - igzip/igzip_stateless_02.asm \ - igzip/igzip_stateless_04.asm \ igzip/crc_data.asm \ igzip/rfc1951_lookup.asm \ igzip/crc32_gzip.asm igzip/detect_repeated_char.asm \ igzip/igzip_multibinary.asm \ - igzip/igzip_stateless_base.c \ igzip/igzip_base.c \ igzip/igzip_update_histogram_01.asm \ igzip/igzip_update_histogram_04.asm \ @@ -66,7 +62,6 @@ other_src += igzip/bitbuf2.asm igzip/data_struct2.asm \ igzip/lz0a_const.asm igzip/options.asm igzip/stdmac.asm igzip/igzip_compare_types.asm \ igzip/bitbuf2.h igzip/repeated_char_result.h \ igzip/igzip_body.asm \ - igzip/igzip_stateless.asm \ igzip/igzip_update_histogram.asm \ igzip/huffman.asm \ include/reg_sizes.asm \ diff --git a/igzip/igzip.c b/igzip/igzip.c index c3eaff3..831bb42 100644 --- a/igzip/igzip.c +++ b/igzip/igzip.c @@ -68,8 +68,6 @@ static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream) static int write_trailer_stateless(struct isal_zstream *stream, uint32_t avail_in, uint32_t crc32); -void isal_deflate_body_stateless(struct isal_zstream *stream); - unsigned int detect_repeated_char(uint8_t * buf, uint32_t size); #define STORED_BLK_HDR_BZ 5 @@ -314,7 +312,9 @@ static uint32_t write_constant_compressed_stateless(struct isal_zstream *stream, uint32_t rep_bytes = rep_bits / 8; uint32_t rep_extra = (repeated_length - 1) % 258; uint32_t bytes; - +#ifndef DEFLATE + uint8_t *start_in = stream->next_in; +#endif /* Guarantee there is enough space for the header even in the worst case */ if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8) return STATELESS_OVERFLOW; @@ -385,6 +385,9 @@ static uint32_t write_constant_compressed_stateless(struct isal_zstream *stream, stream->avail_out -= bytes; stream->total_out += bytes; +#ifndef DEFLATE + state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in); +#endif return COMP_OK; } @@ -411,7 +414,10 @@ static int isal_deflate_int_stateless(struct isal_zstream *stream, uint8_t * nex { uint32_t crc32 = 0; uint32_t repeated_char_length; + struct isal_zstate *state = &stream->internal_state; + state->state = ZSTATE_NEW_HDR; + state->crc = 0; #ifndef DEFLATE if (write_gzip_header_stateless(stream)) return STATELESS_OVERFLOW; @@ -432,18 +438,14 @@ static int isal_deflate_int_stateless(struct isal_zstream *stream, uint8_t * nex stream->end_of_stream) != COMP_OK) return STATELESS_OVERFLOW; -#ifndef DEFLATE - crc32 = crc32_gzip(0x0, next_in, avail_in); -#endif - - if (stream->internal_state.has_eob_hdr) { + if (state->has_eob_hdr) { if (write_trailer_stateless(stream, avail_in, crc32) != COMP_OK) return STATELESS_OVERFLOW; } else if (stream->avail_out >= 8) { sync_flush_stateless(stream); #ifndef USE_BITBUFB flush_write_buffer(stream); - if (stream->internal_state.bitbuf.m_bit_count != 0) + if (state->bitbuf.m_bit_count != 0) return STATELESS_OVERFLOW; #endif } else @@ -454,39 +456,21 @@ static int isal_deflate_int_stateless(struct isal_zstream *stream, uint8_t * nex if (write_constant_compressed_stateless (stream, stream->next_in[0], repeated_char_length, 0) != COMP_OK) return STATELESS_OVERFLOW; - stream->internal_state.has_eob = 0; + state->has_eob = 0; } if (write_deflate_header_unaligned_stateless(stream) != COMP_OK) return STATELESS_OVERFLOW; - if (stream->avail_out < 8) + + reset_match_history(stream); + state->file_start = stream->next_in - stream->total_in; + isal_deflate_pass(stream); + + if (state->state == ZSTATE_END + || (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH)) + return COMP_OK; + else return STATELESS_OVERFLOW; - - memset(stream->internal_state.head, 0, sizeof(stream->internal_state.head)); - stream->internal_state.file_start = stream->next_in; - isal_deflate_body_stateless(stream); - - if (!stream->internal_state.has_eob) - return STATELESS_OVERFLOW; - -#ifndef DEFLATE - crc32 = crc32_gzip(0x0, next_in, avail_in); -#endif - - if (stream->internal_state.has_eob_hdr) { - if (write_trailer_stateless(stream, avail_in, crc32) != COMP_OK) - return STATELESS_OVERFLOW; - } else if (stream->avail_out >= 8) { - sync_flush_stateless(stream); -#ifndef USE_BITBUFB - flush_write_buffer(stream); - if (stream->internal_state.bitbuf.m_bit_count != 0) - return STATELESS_OVERFLOW; -#endif - } else - return STATELESS_OVERFLOW; - - return COMP_OK; } static int write_stored_block_stateless(struct isal_zstream *stream, @@ -561,8 +545,12 @@ static inline void reset_match_history(struct isal_zstream *stream) uint16_t *head = stream->internal_state.head; int i = 0; - for (i = 0; i < sizeof(state->head) / 2; i++) { - head[i] = (uint16_t) (stream->total_in); + if (stream->total_in == 0) + memset(stream->internal_state.head, 0, sizeof(stream->internal_state.head)); + else { + for (i = 0; i < sizeof(state->head) / 2; i++) { + head[i] = (uint16_t) (stream->total_in); + } } } @@ -872,6 +860,8 @@ static int write_deflate_header_stateless(struct isal_zstream *stream) stream->avail_out -= count; stream->total_out += count; + state->state = ZSTATE_BODY; + return COMP_OK; } @@ -939,6 +929,8 @@ static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream) stream->avail_out -= count; stream->total_out += count; + state->state = ZSTATE_BODY; + return COMP_OK; } diff --git a/igzip/igzip_multibinary.asm b/igzip/igzip_multibinary.asm index 39186e2..c2a1252 100644 --- a/igzip/igzip_multibinary.asm +++ b/igzip/igzip_multibinary.asm @@ -38,11 +38,6 @@ default rel %include "reg_sizes.asm" -extern isal_deflate_body_stateless_base -extern isal_deflate_body_stateless_01 -extern isal_deflate_body_stateless_02 -extern isal_deflate_body_stateless_04 - extern isal_deflate_body_base extern isal_deflate_body_01 extern isal_deflate_body_02 @@ -61,9 +56,6 @@ section .text %include "multibinary.asm" -mbin_interface isal_deflate_body_stateless -mbin_dispatch_init5 isal_deflate_body_stateless, isal_deflate_body_stateless_base, isal_deflate_body_stateless_01, isal_deflate_body_stateless_02, isal_deflate_body_stateless_04 - mbin_interface isal_deflate_body mbin_dispatch_init5 isal_deflate_body, isal_deflate_body_base, isal_deflate_body_01, isal_deflate_body_02, isal_deflate_body_04 mbin_interface isal_deflate_finish diff --git a/igzip/igzip_stateless.asm b/igzip/igzip_stateless.asm deleted file mode 100644 index dc76eb0..0000000 --- a/igzip/igzip_stateless.asm +++ /dev/null @@ -1,636 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2016 Intel Corporation All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; * Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in -; the documentation and/or other materials provided with the -; distribution. -; * Neither the name of Intel Corporation nor the names of its -; contributors may be used to endorse or promote products derived -; from this software without specific prior written permission. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "options.asm" - -%include "lz0a_const.asm" -%include "data_struct2.asm" -%include "bitbuf2.asm" -%include "huffman.asm" -%include "igzip_compare_types.asm" -%include "reg_sizes.asm" - -%include "stdmac.asm" - -%define LA_STATELESS 280 ; Max number of bytes read in loop2 rounded up to 8 byte boundary - -%ifdef DEBUG -%macro MARK 1 -global %1 -%1: -%endm -%else -%macro MARK 1 -%endm -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%define tmp2 rcx -%define hash2 rcx - -%define curr_data rax -%define code rax -%define tmp5 rax - -%define tmp4 rbx -%define dist rbx -%define code2 rbx - -%define hash rdx -%define len rdx -%define code_len3 rdx -%define tmp8 rdx - -%define tmp1 rsi -%define code_len2 rsi - -%define file_start rdi - -%define m_bit_count rbp - -%define curr_data2 r8 -%define len2 r8 -%define tmp6 r8 - -%define m_bits r9 - -%define f_i r10 - -%define m_out_buf r11 - -%define f_end_i r12 -%define dist2 r12 -%define tmp7 r12 -%define code4 r12 - -%define tmp3 r13 -%define code3 r13 - -%define stream r14 - -%define hufftables r15 - -;; GPR r8 & r15 can be used - -%define xtmp0 xmm0 ; tmp -%define xtmp1 xmm1 ; tmp -%define xhash xmm2 -%define xmask xmm3 -%define xdata xmm4 - -%define ytmp0 ymm0 ; tmp -%define ytmp1 ymm1 ; tmp - - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - -blen_mem_offset equ 0 ; local variable (8 bytes) -f_end_i_mem_offset equ 8 -gpr_save_mem_offset equ 16 ; gpr save area (8*8 bytes) -xmm_save_mem_offset equ 16 + 8*8 ; xmm save area (4*16 bytes) (16 byte aligned) -stack_size equ 2*8 + 8*8 + 4*16 + 8 -;;; 8 because stack address is odd multiple of 8 after a function call and -;;; we want it aligned to 16 bytes - -; void isal_deflate_body_stateless ( isal_zstream *stream ) -; arg 1: rcx: addr of stream -global isal_deflate_body_stateless_ %+ ARCH -isal_deflate_body_stateless_ %+ ARCH %+ : -%ifidn __OUTPUT_FORMAT__, elf64 - mov rcx, rdi -%endif - - ;; do nothing if (avail_in == 0) - cmp dword [rcx + _avail_in], 0 - jne skip1 - ret -skip1: - -%ifdef ALIGN_STACK - push rbp - mov rbp, rsp - sub rsp, stack_size - and rsp, ~15 -%else - sub rsp, stack_size -%endif - - mov [rsp + gpr_save_mem_offset + 0*8], rbx - mov [rsp + gpr_save_mem_offset + 1*8], rsi - mov [rsp + gpr_save_mem_offset + 2*8], rdi - mov [rsp + gpr_save_mem_offset + 3*8], rbp - mov [rsp + gpr_save_mem_offset + 4*8], r12 - mov [rsp + gpr_save_mem_offset + 5*8], r13 - mov [rsp + gpr_save_mem_offset + 6*8], r14 - mov [rsp + gpr_save_mem_offset + 7*8], r15 - - mov stream, rcx - mov dword [stream + _internal_state_has_eob], 0 - - MOVDQU xmask, [mask] - - ; state->bitbuf.set_buf(stream->next_out, stream->avail_out); - mov m_out_buf, [stream + _next_out] - mov [stream + _internal_state_bitbuf_m_out_start], m_out_buf - mov tmp1 %+ d, [stream + _avail_out] - add tmp1, m_out_buf - sub tmp1, SLOP - -skip_SLOP: - mov [stream + _internal_state_bitbuf_m_out_end], tmp1 - - mov m_bits, [stream + _internal_state_bitbuf_m_bits] - mov m_bit_count %+ d, [stream + _internal_state_bitbuf_m_bit_count] - mov hufftables, [stream + _hufftables] - ; state->b_bytes_valid = stream->avail_in; - mov f_end_i %+ d, [stream + _avail_in] - mov [stream + _internal_state_b_bytes_valid], f_end_i %+ d - - mov f_i, 0 - mov file_start, [stream + _internal_state_file_start] - - ; f_end_i -= LA; - sub f_end_i, LA_STATELESS - mov [rsp + f_end_i_mem_offset], f_end_i - ; if (f_end_i <= 0) continue; - cmp f_end_i, 0 - jle end_loop_2 - - ; for (f_i = f_start_i; f_i < f_end_i; f_i++) { -MARK __stateless_compute_hash_ %+ ARCH - mov curr_data, [file_start + f_i] - - cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end] - ja end - - ;; Encode first byte in the stream as a literal - compute_hash hash, curr_data - and hash %+ d, HASH_MASK - mov [stream + _internal_state_head + 2 * hash], f_i %+ w - and curr_data, 0xff - get_lit_code curr_data, code2, code_len2, hufftables - - mov tmp3, [file_start + f_i + 1] - mov tmp6, tmp3 - compute_hash hash, tmp3 - - shr tmp6, 8 - compute_hash hash2, tmp6 - - MOVD xhash, hash %+ d - PINSRD xhash, hash2 %+ d, 1 - PAND xhash, xhash, xmask - - jmp write_lit_bits - - align 16 - -loop2: - ; if (state->bitbuf.is_full()) { - cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end] - ja end - - xor dist, dist - xor dist2, dist2 - xor tmp3, tmp3 - - lea tmp1, [file_start + f_i] - - mov dist %+ w, f_i %+ w - dec dist - sub dist %+ w, word [stream + _internal_state_head + 2 * hash] - mov [stream + _internal_state_head + 2 * hash], f_i %+ w - - inc f_i - - MOVQ tmp6, xdata - shr tmp5, 16 - mov tmp8, tmp5 - compute_hash tmp6, tmp5 - - mov dist2 %+ w, f_i %+ w - dec dist2 - sub dist2 %+ w, word [stream + _internal_state_head + 2 * hash2] - mov [stream + _internal_state_head + 2 * hash2], f_i %+ w - - ; if ((dist-1) < (D-1)) { - and dist %+ d, (D-1) - neg dist - - shr tmp8, 8 - compute_hash tmp2, tmp8 - - and dist2 %+ d, (D-1) - neg dist2 - -MARK __stateless_compare_ %+ ARCH - ;; Check for long len/dist match (>7) with first literal - MOVQ len, xdata - mov curr_data, len - PSRLDQ xdata, 1 - xor len, [tmp1 + dist - 1] - jz compare_loop - - MOVD xhash, tmp6 %+ d - PINSRD xhash, tmp2 %+ d, 1 - PAND xhash, xhash, xmask - - ;; Check for len/dist match (>7) with second literal - MOVQ len2, xdata - xor len2, [tmp1 + dist2] - jz compare_loop2 - - ;; Specutively load the code for the first literal - movzx tmp1, curr_data %+ b - get_lit_code tmp1, code3, rcx, hufftables - - ;; Check for len/dist match for first literal - test len %+ d, 0xFFFFFFFF - jz len_dist_huffman_pre - - ;; Specutively load the code for the second literal - shr curr_data, 8 - and curr_data, 0xff - get_lit_code curr_data, code2, code_len2, hufftables - - SHLX code2, code2, rcx - or code2, code3 - add code_len2, rcx - - ;; Check for len/dist match for second literal - test len2 %+ d, 0xFFFFFFFF - jnz write_lit_bits - -MARK __stateless_len_dist_lit_huffman_ %+ ARCH -len_dist_lit_huffman_pre: - mov code_len3, rcx - bsf len2, len2 - shr len2, 3 - -len_dist_lit_huffman: - neg dist2 - -%ifndef LONGER_HUFFTABLE - mov tmp4, dist2 - get_dist_code tmp4, code4, code_len2, hufftables ;; clobbers dist, rcx -%else - get_dist_code dist2, code4, code_len2, hufftables -%endif - get_len_code len2, code, rcx, hufftables ;; rcx is code_len - - SHLX code4, code4, rcx - or code4, code - add code_len2, rcx - - add f_i, len2 - neg len2 - - MOVQ tmp5, xdata - shr tmp5, 24 - compute_hash tmp4, tmp5 - and tmp4, HASH_MASK - - SHLX code4, code4, code_len3 - or code4, code3 - add code_len2, code_len3 - - ;; Setup for updating hash - lea tmp3, [f_i + len2 + 1] ; tmp3 <= k - - MOVDQU xdata, [file_start + f_i] - mov curr_data, [file_start + f_i] - mov curr_data2, curr_data - - MOVD hash %+ d, xhash - PEXTRD hash2 %+ d, xhash, 1 - mov [stream + _internal_state_head + 2 * hash], tmp3 %+ w - - compute_hash hash, curr_data - - add tmp3,1 - mov [stream + _internal_state_head + 2 * hash2], tmp3 %+ w - - add tmp3, 1 - mov [stream + _internal_state_head + 2 * tmp4], tmp3 %+ w - - write_bits m_bits, m_bit_count, code4, code_len2, m_out_buf, tmp4 - mov f_end_i, [rsp + f_end_i_mem_offset] - - shr curr_data2, 8 - compute_hash hash2, curr_data2 - -%ifdef NO_LIMIT_HASH_UPDATE -loop3: - add tmp3,1 - cmp tmp3, f_i - jae loop3_done - mov tmp6, [file_start + tmp3] - compute_hash tmp4, tmp6 - and tmp4 %+ d, HASH_MASK - ; state->head[hash] = k; - mov [stream + _internal_state_head + 2 * tmp4], tmp3 %+ w - jmp loop3 -loop3_done: -%endif - ; hash = compute_hash(state->file_start + f_i) & HASH_MASK; - and hash %+ d, HASH_MASK - and hash2 %+ d, HASH_MASK - - ; continue - cmp f_i, f_end_i - jl loop2 - jmp end_loop_2 - ;; encode as dist/len - -MARK __stateless_len_dist_huffman_ %+ ARCH -len_dist_huffman_pre: - bsf len, len - shr len, 3 - -len_dist_huffman: - dec f_i - neg dist - - ; get_dist_code(dist, &code2, &code_len2); -%ifndef LONGER_HUFFTABLE - mov tmp3, dist ; since code2 and dist are rbx - get_dist_code tmp3, code2, code_len2, hufftables ;; clobbers dist, rcx -%else - get_dist_code dist, code2, code_len2, hufftables -%endif - ; get_len_code(len, &code, &code_len); - get_len_code len, code, rcx, hufftables ;; rcx is code_len - - ; code2 <<= code_len - ; code2 |= code - ; code_len2 += code_len - SHLX code2, code2, rcx - or code2, code - add code_len2, rcx - - ;; Setup for updateing hash - lea tmp3, [f_i + 2] ; tmp3 <= k - add f_i, len - - MOVD hash %+ d, xhash - PEXTRD hash2 %+ d, xhash, 1 - mov [stream + _internal_state_head + 2 * hash], tmp3 %+ w - add tmp3,1 - mov [stream + _internal_state_head + 2 * hash2], tmp3 %+ w - - MOVDQU xdata, [file_start + f_i] - mov curr_data, [file_start + f_i] - mov curr_data2, curr_data - compute_hash hash, curr_data - - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp7 - mov f_end_i, [rsp + f_end_i_mem_offset] - - shr curr_data2, 8 - compute_hash hash2, curr_data2 - -%ifdef NO_LIMIT_HASH_UPDATE -loop4: - add tmp3,1 - cmp tmp3, f_i - jae loop4_done - mov tmp6, [file_start + tmp3] - compute_hash tmp4, tmp6 - and tmp4, HASH_MASK - mov [stream + _internal_state_head + 2 * tmp4], tmp3 %+ w - jmp loop4 -loop4_done: -%endif - - ; hash = compute_hash(state->file_start + f_i) & HASH_MASK; - and hash %+ d, HASH_MASK - and hash2 %+ d, HASH_MASK - - ; continue - cmp f_i, f_end_i - jl loop2 - jmp end_loop_2 - -MARK __stateless_write_lit_bits_ %+ ARCH -write_lit_bits: - MOVDQU xdata, [file_start + f_i + 1] - mov f_end_i, [rsp + f_end_i_mem_offset] - add f_i, 1 - mov curr_data, [file_start + f_i] - - MOVD hash %+ d, xhash - - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3 - - PEXTRD hash2 %+ d, xhash, 1 - - ; continue - cmp f_i, f_end_i - jl loop2 - -MARK __stateless_end_loops_ %+ ARCH -end_loop_2: - ;; Handle the last bytes (at most LA_statless bytes) - add f_end_i, LA_STATELESS - LAST_BYTES_COUNT - cmp f_i, f_end_i - jge end_loop_2_finish - -loop2_finish: - ;; Check for space in out buffer - cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end] - ja end - - mov curr_data %+ d, dword [file_start + f_i] - compute_hash hash, curr_data - and hash %+ d, HASH_MASK - - ;; Calculate possible distance for length/dist pair. - xor dist, dist - mov dist %+ w, f_i %+ w - sub dist %+ w, word [stream + _internal_state_head + 2 * hash] - mov [stream + _internal_state_head + 2 * hash], f_i %+ w - - ;; Check if look back distance is valid (the dec is to handle when dist = 0) - dec dist - cmp dist %+ d, (D-1) - jae encode_literal_finish - inc dist - - ;; Check if look back distance is a match - lea tmp6, [f_end_i + LAST_BYTES_COUNT] - sub tmp6, f_i - lea tmp1, [file_start + f_i] - mov tmp2, tmp1 - sub tmp2, dist - compare tmp6, tmp1, tmp2, len, tmp3 - - ;; Limit len to maximum value of 258 - mov tmp2, 258 - cmp len, 258 - cmova len, tmp2 - cmp len, SHORTEST_MATCH - jb encode_literal_finish - - dec dist - ;; Encode len/dist pair -%ifndef LONGER_HUFFTABLE - mov tmp3, dist - get_dist_code tmp3, code2, code_len2, hufftables ;; clobbers dist, rcx -%else - get_dist_code dist, code2, code_len2, hufftables ;; clobbers dist, rcx -%endif - get_len_code len, code, rcx, hufftables ;; rcx is code_len - - ;; Combine length and distance code for writing it out - SHLX code2, code2, rcx - or code2, code - add code_len2, rcx - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3 - - ;; Setup for next loop - add f_i, len - cmp f_i, f_end_i - jl loop2_finish - jmp end_loop_2_finish - -encode_literal_finish: - ;; Encode literal - and curr_data %+ d, 0xFF - get_lit_code curr_data, code2, code_len2, hufftables - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3 - - ;; Setup for next loop - add f_i, 1 - cmp f_i, f_end_i - jl loop2_finish -end_loop_2_finish: - cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end] - ja end - - ;; Check if any bytes left (at most LAST_BYTES_COUNT bytes) - add f_end_i, LAST_BYTES_COUNT - cmp f_i, f_end_i - jz write_eob - - ;; Handle encoding last few bytes by encoding them as literals - xor curr_data, curr_data -final_bytes: - movzx curr_data, byte [file_start + f_i] - get_lit_code curr_data, code2, code_len2, hufftables - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3 - - cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end] - ja end - - inc f_i - cmp f_i, f_end_i - jl final_bytes - -write_eob: - ;; Write out end of block - get_lit_code 256, code2, code_len2, hufftables - write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3 - mov dword [stream + _internal_state_has_eob], 1 - -end: - ;; update input buffer - add [stream + _total_in], f_i %+ d - add [stream + _next_in], f_i - sub [stream + _avail_in], f_i %+ d - - ;; update output buffer - mov [stream + _next_out], m_out_buf - sub m_out_buf, [stream + _internal_state_bitbuf_m_out_start] - sub [stream + _avail_out], m_out_buf %+ d - add [stream + _total_out], m_out_buf %+ d - - mov [stream + _internal_state_bitbuf_m_bits], m_bits - mov [stream + _internal_state_bitbuf_m_bit_count], m_bit_count %+ d - - mov rbx, [rsp + gpr_save_mem_offset + 0*8] - mov rsi, [rsp + gpr_save_mem_offset + 1*8] - mov rdi, [rsp + gpr_save_mem_offset + 2*8] - mov rbp, [rsp + gpr_save_mem_offset + 3*8] - mov r12, [rsp + gpr_save_mem_offset + 4*8] - mov r13, [rsp + gpr_save_mem_offset + 5*8] - mov r14, [rsp + gpr_save_mem_offset + 6*8] - mov r15, [rsp + gpr_save_mem_offset + 7*8] - -%ifndef ALIGN_STACK - add rsp, stack_size -%else - mov rsp, rbp - pop rbp -%endif - ret - -MARK __stateless_compare_loops_ %+ ARCH -compare_loop: - MOVD xhash, tmp6 %+ d - PINSRD xhash, tmp2 %+ d, 1 - PAND xhash, xhash, xmask - lea tmp2, [tmp1 + dist - 1] -%if (COMPARE_TYPE == 1) - compare250 tmp1, tmp2, len, tmp3 -%elif (COMPARE_TYPE == 2) - compare250_x tmp1, tmp2, len, tmp3, xtmp0, xtmp1 -%elif (COMPARE_TYPE == 3) - compare250_y tmp1, tmp2, len, tmp3, ytmp0, ytmp1 -%else - %error Unknown Compare type COMPARE_TYPE - % error -%endif - jmp len_dist_huffman - -compare_loop2: - lea tmp2, [tmp1 + dist2] - add tmp1, 1 -%if (COMPARE_TYPE == 1) - compare250 tmp1, tmp2, len2, tmp3 -%elif (COMPARE_TYPE == 2) - compare250_x tmp1, tmp2, len2, tmp3, xtmp0, xtmp1 -%elif (COMPARE_TYPE == 3) - compare250_y tmp1, tmp2, len2, tmp3, ytmp0, ytmp1 -%else -%error Unknown Compare type COMPARE_TYPE - % error -%endif - and curr_data, 0xff - get_lit_code curr_data, code3, code_len3, hufftables - jmp len_dist_lit_huffman - -section .data - align 16 -mask: dd HASH_MASK, HASH_MASK, HASH_MASK, HASH_MASK -const_D: dq D diff --git a/igzip/igzip_stateless_01.asm b/igzip/igzip_stateless_01.asm deleted file mode 100644 index 83ed1ba..0000000 --- a/igzip/igzip_stateless_01.asm +++ /dev/null @@ -1,7 +0,0 @@ -%define ARCH 01 - -%ifndef COMPARE_TYPE -%define COMPARE_TYPE 1 -%endif - -%include "igzip_stateless.asm" diff --git a/igzip/igzip_stateless_02.asm b/igzip/igzip_stateless_02.asm deleted file mode 100644 index cb2a3ae..0000000 --- a/igzip/igzip_stateless_02.asm +++ /dev/null @@ -1,7 +0,0 @@ -%define ARCH 02 - -%ifndef COMPARE_TYPE -%define COMPARE_TYPE 1 -%endif - -%include "igzip_stateless.asm" diff --git a/igzip/igzip_stateless_04.asm b/igzip/igzip_stateless_04.asm deleted file mode 100644 index 39d8ef5..0000000 --- a/igzip/igzip_stateless_04.asm +++ /dev/null @@ -1,8 +0,0 @@ -%define ARCH 04 -%define USE_HSWNI - -%ifndef COMPARE_TYPE -%define COMPARE_TYPE 3 -%endif - -%include "igzip_stateless.asm" diff --git a/igzip/igzip_stateless_base.c b/igzip/igzip_stateless_base.c deleted file mode 100644 index 9aa0f47..0000000 --- a/igzip/igzip_stateless_base.c +++ /dev/null @@ -1,151 +0,0 @@ -#include -#include "igzip_lib.h" -#include "huffman.h" -#include "huff_codes.h" -#include "bitbuf2.h" - -static inline void update_state(struct isal_zstream *stream, struct isal_zstate *state, - uint8_t * end_in, uint8_t * start_in) -{ - uint32_t count; - stream->avail_in = end_in - stream->next_in; - stream->total_in += stream->next_in - start_in; - count = buffer_used(&state->bitbuf); - stream->next_out = buffer_ptr(&state->bitbuf); - stream->avail_out -= count; - stream->total_out += count; - -} - -void isal_deflate_body_stateless_base(struct isal_zstream *stream) -{ - uint32_t literal = 0, hash; - uint8_t *start_in, *end_in, *end, *next_hash; - uint16_t match_length; - uint32_t dist; - uint64_t code, code_len, code2, code_len2, i; - struct isal_zstate *state = &stream->internal_state; - uint16_t *last_seen = state->head; - - if (stream->avail_in == 0) - return; - - set_buf(&state->bitbuf, stream->next_out, stream->avail_out); - start_in = stream->next_in; - end_in = stream->next_in + stream->avail_in; - - while (stream->next_in < end_in - 3) { - if (is_full(&state->bitbuf)) { - update_state(stream, state, end_in, start_in); - return; - } - - literal = *(uint32_t *) stream->next_in; - hash = compute_hash(literal) & HASH_MASK; - dist = (uint64_t) (stream->next_in - last_seen[hash]) & 0xFFFF; - last_seen[hash] = (uint64_t) stream->next_in; - - if (dist - 1 < IGZIP_HIST_SIZE - 1 && stream->next_in - dist >= start_in) { /* The -1 are to handle the case when dist = 0 */ - match_length = - compare258(stream->next_in - dist, stream->next_in, - end_in - stream->next_in); - - if (match_length >= SHORTEST_MATCH) { - next_hash = stream->next_in; -#ifdef ISAL_LIMIT_HASH_UPDATE - end = next_hash + 3; -#else - end = next_hash + match_length; -#endif - if (end > end_in - 3) - end = end_in - 3; - next_hash++; - for (; next_hash < end; next_hash++) { - literal = *(uint32_t *) next_hash; - hash = compute_hash(literal) & HASH_MASK; - last_seen[hash] = (uint64_t) next_hash; - } - - get_len_code(stream->hufftables, match_length, &code, - &code_len); - get_dist_code(stream->hufftables, dist, &code2, &code_len2); - - code |= code2 << code_len; - code_len += code_len2; - - write_bits(&state->bitbuf, code, code_len); - - stream->next_in += match_length; - - continue; - } - } - - get_lit_code(stream->hufftables, literal & 0xFF, &code, &code_len); - write_bits(&state->bitbuf, code, code_len); - stream->next_in++; - } - - if (is_full(&state->bitbuf)) { - update_state(stream, state, end_in, start_in); - return; - } - - literal = *(uint32_t *) (end_in - 4); - - for (i = 4; i > end_in - stream->next_in; i--) - literal = literal >> 8; - - hash = compute_hash(literal) & HASH_MASK; - dist = (uint64_t) (stream->next_in - last_seen[hash]) & 0xFFFF; - - if (dist - 1 < IGZIP_HIST_SIZE - 1 && stream->next_in - dist >= start_in) { - match_length = - compare258(stream->next_in - dist, stream->next_in, - end_in - stream->next_in); - if (match_length >= SHORTEST_MATCH) { - get_len_code(stream->hufftables, match_length, &code, &code_len); - get_dist_code(stream->hufftables, dist, &code2, &code_len2); - code |= code2 << code_len; - code_len += code_len2; - write_bits(&state->bitbuf, code, code_len); - stream->next_in += 3; - - if (is_full(&state->bitbuf)) { - update_state(stream, state, end_in, start_in); - return; - } - - get_lit_code(stream->hufftables, 256, &code, &code_len); - write_bits(&state->bitbuf, code, code_len); - - if (is_full(&state->bitbuf)) { - update_state(stream, state, end_in, start_in); - return; - } - - state->has_eob = 1; - update_state(stream, state, end_in, start_in); - return; - } - } - - while (stream->next_in < end_in) { - get_lit_code(stream->hufftables, literal & 0xFF, &code, &code_len); - write_bits(&state->bitbuf, code, code_len); - stream->next_in++; - - if (is_full(&state->bitbuf)) { - update_state(stream, state, end_in, start_in); - return; - } - literal >>= 8; - } - - get_lit_code(stream->hufftables, 256, &code, &code_len); - write_bits(&state->bitbuf, code, code_len); - - state->has_eob = 1; - update_state(stream, state, end_in, start_in); - return; -}