From ff1928e8eca3b3e18b94252d892f742669d9e611 Mon Sep 17 00:00:00 2001 From: Roy Oursler Date: Mon, 9 Jul 2018 10:42:03 -0700 Subject: [PATCH] igzip: Create functions to write gzip/zlib headers Change-Id: If5aaa277a01214bd36406ee11680df0904ad12f7 Signed-off-by: Roy Oursler --- igzip/igzip.c | 122 +++++++++++++++++++++++++++++ igzip/igzip_inflate.c | 107 ++++++++++++++------------ igzip/igzip_wrapper.h | 52 +++++++++++++ igzip/igzip_wrapper_hdr_test.c | 136 +++++++++++++++++++-------------- include/igzip_lib.h | 42 ++++++++++ isa-l.def | 2 + 6 files changed, 352 insertions(+), 109 deletions(-) create mode 100644 igzip/igzip_wrapper.h diff --git a/igzip/igzip.c b/igzip/igzip.c index 5682b10..fe940da 100644 --- a/igzip/igzip.c +++ b/igzip/igzip.c @@ -49,6 +49,7 @@ #include "encode_df.h" #include "igzip_level_buf_structs.h" #include "igzip_checksums.h" +#include "igzip_wrapper.h" #ifdef __FreeBSD__ #include @@ -1009,6 +1010,127 @@ void isal_deflate_reset(struct isal_zstream *stream) } +void isal_gzip_header_init(struct isal_gzip_header *gz_hdr) +{ + gz_hdr->text = 0; + gz_hdr->time = 0; + gz_hdr->xflags = 0; + gz_hdr->os = 0xff; + gz_hdr->extra = NULL; + gz_hdr->extra_buf_len = 0; + gz_hdr->extra_len = 0; + gz_hdr->name = NULL; + gz_hdr->name_buf_len = 0; + gz_hdr->comment = NULL; + gz_hdr->comment_buf_len = 0; + gz_hdr->hcrc = 0; +}; + +uint32_t isal_write_gzip_header(struct isal_zstream *stream, struct isal_gzip_header *gz_hdr) +{ + uint32_t flags = 0, hcrc, hdr_size = GZIP_HDR_BASE; + uint8_t *out_buf = stream->next_out, *out_buf_start = stream->next_out; + uint32_t name_len = 0, comment_len = 0; + + if (gz_hdr->text) + flags |= TEXT_FLAG; + if (gz_hdr->extra) { + flags |= EXTRA_FLAG; + hdr_size += GZIP_EXTRA_LEN + gz_hdr->extra_len; + } + if (gz_hdr->name) { + flags |= NAME_FLAG; + name_len = strnlen(gz_hdr->name, gz_hdr->name_buf_len); + if (name_len < gz_hdr->name_buf_len) + name_len++; + hdr_size += name_len; + } + if (gz_hdr->comment) { + flags |= COMMENT_FLAG; + comment_len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len); + if (comment_len < gz_hdr->comment_buf_len) + comment_len++; + hdr_size += comment_len; + } + if (gz_hdr->hcrc) { + flags |= HCRC_FLAG; + hdr_size += GZIP_HCRC_LEN; + } + + if (stream->avail_out < hdr_size) + return hdr_size; + + out_buf[0] = 0x1f; + out_buf[1] = 0x8b; + out_buf[2] = DEFLATE_METHOD; + out_buf[3] = flags; + *(uint32_t *) (out_buf + 4) = gz_hdr->time; + out_buf[8] = gz_hdr->xflags; + out_buf[9] = gz_hdr->os; + + out_buf += GZIP_HDR_BASE; + if (flags & EXTRA_FLAG) { + *(uint16_t *) out_buf = gz_hdr->extra_len; + out_buf += GZIP_EXTRA_LEN; + + memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len); + out_buf += gz_hdr->extra_len; + } + + if (flags & NAME_FLAG) { + memcpy(out_buf, gz_hdr->name, name_len); + out_buf += name_len; + } + + if (flags & COMMENT_FLAG) { + memcpy(out_buf, gz_hdr->comment, comment_len); + out_buf += comment_len; + } + + if (flags & HCRC_FLAG) { + hcrc = crc32_gzip(0, out_buf_start, out_buf - out_buf_start); + *(uint16_t *) out_buf = hcrc; + out_buf += GZIP_HCRC_LEN; + } + + stream->next_out += hdr_size; + stream->total_out += hdr_size; + stream->avail_out -= hdr_size; + + return ISAL_DECOMP_OK; +} + +uint32_t isal_write_zlib_header(struct isal_zstream * stream, struct isal_zlib_header * z_hdr) +{ + uint32_t cmf, flg, dict_flag = 0, hdr_size = ZLIB_HDR_BASE; + uint8_t *out_buf = stream->next_out; + + if (z_hdr->dict_flag) { + dict_flag = ZLIB_DICT_FLAG; + hdr_size = ZLIB_HDR_BASE + ZLIB_DICT_LEN; + } + + if (stream->avail_out < hdr_size) + return hdr_size; + + cmf = DEFLATE_METHOD | (z_hdr->info << 4); + flg = (z_hdr->level << 6) | dict_flag; + + flg += 31 - ((256 * cmf + flg) % 31); + + out_buf[0] = cmf; + out_buf[1] = flg; + + if (dict_flag) + *(uint32_t *) (out_buf + 2) = z_hdr->dict_id; + + stream->next_out += hdr_size; + stream->total_out += hdr_size; + stream->avail_out -= hdr_size; + + return ISAL_DECOMP_OK; +} + int isal_deflate_set_hufftables(struct isal_zstream *stream, struct isal_hufftables *hufftables, int type) { diff --git a/igzip/igzip_inflate.c b/igzip/igzip_inflate.c index 5085f88..6b98383 100644 --- a/igzip/igzip_inflate.c +++ b/igzip/igzip_inflate.c @@ -31,6 +31,7 @@ #include "igzip_lib.h" #include "huff_codes.h" #include "igzip_checksums.h" +#include "igzip_wrapper.h" #ifdef __FreeBSD__ #include @@ -94,25 +95,6 @@ extern int decode_huffman_code_block_stateless(struct inflate_state *, uint8_t * #define SINGLE_SYM_THRESH (2 * 1024) #define DOUBLE_SYM_THRESH (4 * 1024) -#define DEFLATE_METHOD 8 -#define ZLIB_DICT_FLAG (1 << 5) -#define TEXT_FLAG (1 << 0) -#define HCRC_FLAG (1 << 1) -#define EXTRA_FLAG (1 << 2) -#define NAME_FLAG (1 << 3) -#define COMMENT_FLAG (1 << 4) -#define UNDEFINED_FLAG (-1) - -#define GZIP_HDR_BASE 10 -#define GZIP_EXTRA_LEN 2 -#define GZIP_HCRC_LEN 2 -#define GZIP_TRAILER_LEN 8 - -#define ZLIB_HDR_BASE 2 -#define ZLIB_DICT_LEN 4 -#define ZLIB_INFO_OFFSET 4 -#define ZLIB_LEVEL_OFFSET 6 -#define ZLIB_TRAILER_LEN 4 /* structure contain lookup data based on RFC 1951 */ struct rfc1951_tables { uint8_t dist_extra_bit_count[32]; @@ -1824,7 +1806,7 @@ static inline uint32_t string_header_copy(struct inflate_state *state, { uint32_t len, max_len = str_len; - if (max_len > state->avail_in) + if (max_len > state->avail_in || str_buf == NULL) max_len = state->avail_in; len = strnlen((char *)state->next_in, max_len); @@ -1836,7 +1818,7 @@ static inline uint32_t string_header_copy(struct inflate_state *state, state->avail_in -= len; state->count += len; - if (len == str_len) + if (str_buf != NULL && len == str_len) return str_error; else if (state->avail_in <= 0) return ISAL_END_INPUT; @@ -1858,27 +1840,40 @@ static int check_gzip_checksum(struct inflate_state *state) uint32_t byte_count, offset, tmp_in_size = state->tmp_in_size; int ret; - if (state->read_in_length >= 8) { - byte_count = state->read_in_length / 8; - offset = state->read_in_length % 8; + if (state->read_in_length >= 8 * GZIP_TRAILER_LEN) { + /* The following is unecessary as state->read_in_length == 64 */ + /* bit_count = state->read_in_length % 8; */ + /* state->read_in >>= bit_count; */ + /* state->read_in_length -= bit_count; */ - *(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = state->read_in >> offset; - state->read_in = 0; + trailer = state->read_in; state->read_in_length = 0; + state->read_in = 0; + } else { + if (state->read_in_length >= 8) { + byte_count = state->read_in_length / 8; + offset = state->read_in_length % 8; - tmp_in_size += byte_count; - state->tmp_in_size = tmp_in_size; - } + *(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = + state->read_in >> offset; + state->read_in = 0; + state->read_in_length = 0; - ret = fixed_size_read(state, &next_in, GZIP_TRAILER_LEN); - if (ret) { - state->block_state = ISAL_CHECKSUM_CHECK; - return ret; + tmp_in_size += byte_count; + state->tmp_in_size = tmp_in_size; + } + + ret = fixed_size_read(state, &next_in, GZIP_TRAILER_LEN); + if (ret) { + state->block_state = ISAL_CHECKSUM_CHECK; + return ret; + } + + trailer = *(uint64_t *) next_in; } state->block_state = ISAL_BLOCK_FINISH; - trailer = *(uint64_t *) next_in; crc = state->crc; total_out = state->total_out; @@ -1894,30 +1889,42 @@ static int check_zlib_checksum(struct inflate_state *state) uint32_t trailer; uint8_t *next_in; uint32_t byte_count, offset, tmp_in_size = state->tmp_in_size; - int ret; + int ret, bit_count; - if (state->read_in_length >= 8) { - byte_count = state->read_in_length / 8; - offset = state->read_in_length % 8; + if (state->read_in_length >= 8 * ZLIB_TRAILER_LEN) { + bit_count = state->read_in_length % 8; + state->read_in >>= bit_count; + state->read_in_length -= bit_count; - *(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = state->read_in >> offset; - state->read_in = 0; - state->read_in_length = 0; + trailer = state->read_in; - tmp_in_size += byte_count; - state->tmp_in_size = tmp_in_size; - } + state->read_in_length -= 8 * ZLIB_TRAILER_LEN; + state->read_in >>= 8 * ZLIB_TRAILER_LEN; + } else { + if (state->read_in_length >= 8) { + byte_count = state->read_in_length / 8; + offset = state->read_in_length % 8; - ret = fixed_size_read(state, &next_in, ZLIB_TRAILER_LEN); - if (ret) { - state->block_state = ISAL_CHECKSUM_CHECK; - return ret; + *(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = + state->read_in >> offset; + state->read_in = 0; + state->read_in_length = 0; + + tmp_in_size += byte_count; + state->tmp_in_size = tmp_in_size; + } + + ret = fixed_size_read(state, &next_in, ZLIB_TRAILER_LEN); + if (ret) { + state->block_state = ISAL_CHECKSUM_CHECK; + return ret; + } + + trailer = *(uint32_t *) next_in; } state->block_state = ISAL_BLOCK_FINISH; - trailer = *(uint32_t *) next_in; - if (bswap_32(trailer) != state->crc) return ISAL_INCORRECT_CHECKSUM; else diff --git a/igzip/igzip_wrapper.h b/igzip/igzip_wrapper.h new file mode 100644 index 0000000..f1b4bce --- /dev/null +++ b/igzip/igzip_wrapper.h @@ -0,0 +1,52 @@ +/********************************************************************** + Copyright(c) 2011-2018 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#ifndef IGZIP_WRAPPER_H + +#define DEFLATE_METHOD 8 +#define ZLIB_DICT_FLAG (1 << 5) +#define TEXT_FLAG (1 << 0) +#define HCRC_FLAG (1 << 1) +#define EXTRA_FLAG (1 << 2) +#define NAME_FLAG (1 << 3) +#define COMMENT_FLAG (1 << 4) +#define UNDEFINED_FLAG (-1) + +#define GZIP_HDR_BASE 10 +#define GZIP_EXTRA_LEN 2 +#define GZIP_HCRC_LEN 2 +#define GZIP_TRAILER_LEN 8 + +#define ZLIB_HDR_BASE 2 +#define ZLIB_DICT_LEN 4 +#define ZLIB_INFO_OFFSET 4 +#define ZLIB_LEVEL_OFFSET 6 +#define ZLIB_TRAILER_LEN 4 + +#endif diff --git a/igzip/igzip_wrapper_hdr_test.c b/igzip/igzip_wrapper_hdr_test.c index 9f9ddfb..49c2df5 100644 --- a/igzip/igzip_wrapper_hdr_test.c +++ b/igzip/igzip_wrapper_hdr_test.c @@ -4,6 +4,7 @@ #include #include "checksum_test_ref.h" #include "igzip_lib.h" +#include "igzip_wrapper.h" #define TEST_SEED 0x1234 #define RANDOMS 0x4000 @@ -16,13 +17,6 @@ #define NAME_SIZE 25 #define COMMENT_SIZE 192 -#define TEXT_FLAG (1 << 0) -#define HCRC_FLAG (1 << 1) -#define EXTRA_FLAG (1 << 2) -#define NAME_FLAG (1 << 3) -#define COMMENT_FLAG (1 << 4) -#define DICT_FLAG (1 << 5) - enum { INVALID_WRAPPER = ISAL_INVALID_WRAPPER, UNSUPPORTED_METHOD = ISAL_UNSUPPORTED_METHOD, @@ -44,6 +38,9 @@ enum { INCORRECT_LEVEL, INCORRECT_DICT_FLAG, INCORRECT_DICT_ID, + INCORRECT_HDR_LEN, + INSUFFICIENT_BUFFER_SIZE, + INCORRECT_WRITE_RETURN, MALLOC_FAILED }; @@ -107,6 +104,15 @@ void print_error(int32_t error) case INCORRECT_DICT_ID: printf("Incorrect dictionary id found\n"); break; + case INCORRECT_HDR_LEN: + printf("Incorrect header length found\n"); + break; + case INSUFFICIENT_BUFFER_SIZE: + printf("Insufficient buffer size to write header\n"); + break; + case INCORRECT_WRITE_RETURN: + printf("Header write returned an incorrect value\n"); + break; case MALLOC_FAILED: printf("Failed to allocate buffers\n"); break; @@ -384,74 +390,78 @@ void gen_rand_zlib_header(struct isal_zlib_header *z_hdr) z_hdr->dict_id = rand(); } -void write_gzip_header(uint8_t * out_buf, struct isal_gzip_header *gz_hdr) +int write_gzip_header(uint8_t * hdr_buf, uint32_t hdr_buf_len, struct isal_gzip_header *gz_hdr) { - uint32_t flags = 0, len, hcrc; - uint8_t *out_buf_start = out_buf; - if (gz_hdr->text) - flags |= TEXT_FLAG; - if (gz_hdr->extra) - flags |= EXTRA_FLAG; - if (gz_hdr->name) - flags |= NAME_FLAG; - if (gz_hdr->comment) - flags |= COMMENT_FLAG; - if (gz_hdr->hcrc) - flags |= HCRC_FLAG; + struct isal_zstream stream; + uint32_t hdr_len = gzip_header_size(gz_hdr); + uint32_t len; - out_buf[0] = 0x1f; - out_buf[1] = 0x8b; - out_buf[2] = 8; - out_buf[3] = flags; - *(uint32_t *) (out_buf + 4) = gz_hdr->time; - out_buf[8] = gz_hdr->xflags; - out_buf[9] = gz_hdr->os; + isal_deflate_init(&stream); + stream.next_out = hdr_buf; + stream.avail_out = rand() % hdr_len; + len = isal_write_gzip_header(&stream, gz_hdr); - out_buf += 10; - if (flags & EXTRA_FLAG) { - *(uint16_t *) out_buf = gz_hdr->extra_len; - out_buf += 2; - - memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len); - out_buf += gz_hdr->extra_len; + if (len != hdr_len) { + printf("len = %d, hdr_buf_len = %d\n", len, hdr_len); + print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr, NULL); + print_error(INCORRECT_HDR_LEN); + return INCORRECT_HDR_LEN; } - if (flags & NAME_FLAG) { - len = strnlen(gz_hdr->name, gz_hdr->name_buf_len) + 1; - memcpy(out_buf, gz_hdr->name, len); - out_buf += len; + if (hdr_buf_len < hdr_len) { + print_gzip_final_verbose(NULL, 0, gz_hdr, NULL); + print_error(INSUFFICIENT_BUFFER_SIZE); + return INSUFFICIENT_BUFFER_SIZE; } - if (flags & COMMENT_FLAG) { - len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len) + 1; - memcpy(out_buf, gz_hdr->comment, len); - out_buf += len; + stream.avail_out = hdr_buf_len; + + len = isal_write_gzip_header(&stream, gz_hdr); + + if (len) { + print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr, NULL); + print_error(INCORRECT_WRITE_RETURN); + return INCORRECT_WRITE_RETURN;; } - if (flags & HCRC_FLAG) { - hcrc = crc32_gzip_refl_ref(0, out_buf_start, out_buf - out_buf_start); - *(uint16_t *) out_buf = hcrc; - out_buf += 2; - } + return 0; } -void write_zlib_header(uint8_t * out_buf, struct isal_zlib_header *z_hdr) +int write_zlib_header(uint8_t * hdr_buf, uint32_t hdr_buf_len, struct isal_zlib_header *z_hdr) { - uint32_t cmf, flg, dict_flag; + struct isal_zstream stream; + uint32_t hdr_len = zlib_header_size(z_hdr); + uint32_t len; - dict_flag = (z_hdr->dict_flag) ? DICT_FLAG : 0; + isal_deflate_init(&stream); + stream.next_out = hdr_buf; + stream.avail_out = rand() % hdr_len; + len = isal_write_zlib_header(&stream, z_hdr); - cmf = 8 | (z_hdr->info << 4); - flg = (z_hdr->level << 6) | dict_flag; + if (len != hdr_len) { + print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr, NULL); + print_error(INCORRECT_HDR_LEN); + return INCORRECT_HDR_LEN; + } - flg += 31 - ((256 * cmf + flg) % 31); + if (hdr_buf_len < hdr_len) { + print_zlib_final_verbose(NULL, 0, z_hdr, NULL); + print_error(INSUFFICIENT_BUFFER_SIZE); + return INSUFFICIENT_BUFFER_SIZE; + } - out_buf[0] = cmf; - out_buf[1] = flg; + stream.avail_out = hdr_buf_len; - if (dict_flag) - *(uint32_t *) (out_buf + 2) = z_hdr->dict_id; + len = isal_write_zlib_header(&stream, z_hdr); + + if (len) { + print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr, NULL); + print_error(INCORRECT_WRITE_RETURN); + return INCORRECT_WRITE_RETURN;; + } + + return 0; } int compare_gzip_headers(struct isal_gzip_header *gz_hdr1, struct isal_gzip_header *gz_hdr2) @@ -806,7 +816,11 @@ int main(int argc, char *argv[]) hdr_buf_len = gzip_header_size(&gz_hdr_orig); hdr_buf = malloc(hdr_buf_len); - write_gzip_header(hdr_buf, &gz_hdr_orig); + ret = write_gzip_header(hdr_buf, hdr_buf_len, &gz_hdr_orig); + + fin_ret |= ret; + if (ret) + return (ret == 0); ret = read_gzip_header_simple(hdr_buf, hdr_buf_len, &gz_hdr_orig); @@ -838,7 +852,11 @@ int main(int argc, char *argv[]) hdr_buf_len = zlib_header_size(&z_hdr_orig); hdr_buf = malloc(hdr_buf_len); - write_zlib_header(hdr_buf, &z_hdr_orig); + ret = write_zlib_header(hdr_buf, hdr_buf_len, &z_hdr_orig); + + fin_ret |= ret; + if (ret) + return (ret == 0); ret = read_zlib_header_simple(hdr_buf, hdr_buf_len, &z_hdr_orig); diff --git a/include/igzip_lib.h b/include/igzip_lib.h index bea0060..6dbbcb3 100644 --- a/include/igzip_lib.h +++ b/include/igzip_lib.h @@ -591,6 +591,48 @@ void isal_deflate_init(struct isal_zstream *stream); */ void isal_deflate_reset(struct isal_zstream *stream); + +/** + * @brief Set gzip header default values + * + * @param gz_hdr: Gzip header to initialize. + */ +void isal_gzip_header_init(struct isal_gzip_header *gz_hdr); + +/** + * @brief Write gzip header to output stream + * + * Writes the gzip header to the output stream. On entry this function assumes + * that the output buffer has been initialized, so stream->next_out, + * stream->avail_out and stream->total_out have been set. If the output buffer + * contains insufficient space, stream is not modified. + * + * @param stream: Structure holding state information on the compression stream. + * @param gz_hdr: Structure holding the gzip header information to encode. + * + * @returns Returns 0 if the header is sucessfully written, otherwise returns + * the minimum size required to sucessfully write the gzip header to the output + * buffer. + */ +uint32_t isal_write_gzip_header(struct isal_zstream * stream, struct isal_gzip_header *gz_hdr); + +/** + * @brief Write zlib header to output stream + * + * Writes the zlib header to the output stream. On entry this function assumes + * that the output buffer has been initialized, so stream->next_out, + * stream->avail_out and stream->total_out have been set. If the output buffer + * contains insufficient space, stream is not modified. + * + * @param stream: Structure holding state information on the compression stream. + * @param z_hdr: Structure holding the zlib header information to encode. + * + * @returns Returns 0 if the header is sucessfully written, otherwise returns + * the minimum size required to sucessfully write the zlib header to the output + * buffer. + */ +uint32_t isal_write_zlib_header(struct isal_zstream * stream, struct isal_zlib_header *z_hdr); + /** * @brief Set stream to use a new Huffman code * diff --git a/isa-l.def b/isa-l.def index 086d398..1a64232 100644 --- a/isa-l.def +++ b/isa-l.def @@ -107,3 +107,5 @@ isal_inflate_reset @103 crc16_t10dif_copy @104 isal_read_gzip_header @105 isal_read_zlib_header @106 +isal_write_gzip_header @107 +isal_write_zlib_header @108