generic-library/isa-l
1
0
Fork 0
mirror of https://github.com/intel/isa-l.git synced 2025-02-21 06:37:40 +01:00
Code Issues Projects Releases Wiki Activity

igzip: Implement gzip/zlib header/trailer parsing

Browse Source 
Change-Id: I3fe8653f2286212a9d6c6ecfa3b78752b2cac8ef
Signed-off-by: Roy Oursler <roy.j.oursler@intel.com>
This commit is contained in:
Roy Oursler 2018-06-06 09:23:14 -07:00
parent aa8b51930f
commit 43374f6776
9 changed files with 1519 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.nmake
View File

@ -205,6 +205,7 @@ checks = \
crc16_t10dif_copy_test.exe \
crc32_funcs_test.exe \
crc64_funcs_test.exe \
igzip_wrapper_hdr_test.exe \
igzip_rand_test.exe
checks: lib $(checks)

1
igzip/Makefile.am
View File

@ -66,6 +66,7 @@ src_include += -I $(srcdir)/igzip
extern_hdrs += include/igzip_lib.h
check_tests += igzip/igzip_rand_test
check_tests += igzip/igzip_wrapper_hdr_test
unit_tests += igzip/checksum32_funcs_test
perf_tests += igzip/igzip_perf

2
igzip/checksum_test_ref.h
View File

@ -5,6 +5,8 @@
#ifndef CHECKSUM_TEST_REF_H
#define CHECKSUM_TEST_REF_H
#include <stdint.h>
uint32_t inflate_crc_table[256] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,

464
igzip/igzip_inflate.c
View File

@ -32,6 +32,19 @@
#include "huff_codes.h"
#include "igzip_checksums.h"
#ifdef __FreeBSD__
#include <sys/types.h>
#include <sys/endian.h>
# define bswap_32(x) bswap32(x)
#elif defined (__APPLE__)
#include <libkern/OSByteOrder.h>
# define bswap_32(x) OSSwapInt32(x)
#elif defined (__GNUC__) && !defined (__MINGW32__)
# include <byteswap.h>
#elif defined _WIN64
# define bswap_32(x) _byteswap_ulong(x)
#endif
extern int decode_huffman_code_block_stateless(struct inflate_state *, uint8_t * start_out);
#define LARGE_SHORT_SYM_LEN 25
@ -80,6 +93,26 @@ extern int decode_huffman_code_block_stateless(struct inflate_state *, uint8_t *
#define SINGLE_SYM_THRESH (2 * 1024)
#define DOUBLE_SYM_THRESH (4 * 1024)
#define DEFLATE_METHOD 8
#define ZLIB_DICT_FLAG (1 << 5)
#define TEXT_FLAG (1 << 0)
#define HCRC_FLAG (1 << 1)
#define EXTRA_FLAG (1 << 2)
#define NAME_FLAG (1 << 3)
#define COMMENT_FLAG (1 << 4)
#define UNDEFINED_FLAG (-1)
#define GZIP_HDR_BASE 10
#define GZIP_EXTRA_LEN 2
#define GZIP_HCRC_LEN 2
#define GZIP_TRAILER_LEN 8
#define ZLIB_HDR_BASE 2
#define ZLIB_DICT_LEN 4
#define ZLIB_INFO_OFFSET 4
#define ZLIB_LEVEL_OFFSET 6
#define ZLIB_TRAILER_LEN 4
/* structure contain lookup data based on RFC 1951 */
struct rfc1951_tables {
uint8_t dist_extra_bit_count[32];
@ -155,10 +188,12 @@ static void update_checksum(struct inflate_state *state, uint8_t * start_in, uin
switch (state->crc_flag) {
case ISAL_GZIP:
case ISAL_GZIP_NO_HDR:
case ISAL_GZIP_NO_HDR_VER:
state->crc = crc32_gzip(state->crc, start_in, length);
break;
case ISAL_ZLIB:
case ISAL_ZLIB_NO_HDR:
case ISAL_ZLIB_NO_HDR_VER:
state->crc = isal_adler32_bam1(state->crc, start_in, length);
break;
}
@ -1701,6 +1736,7 @@ void isal_inflate_init(struct inflate_state *state)
state->write_overflow_len = 0;
state->copy_overflow_length = 0;
state->copy_overflow_distance = 0;
state->wrapper_flag = 0;
state->tmp_in_size = 0;
state->tmp_out_processed = 0;
state->tmp_out_valid = 0;
@ -1725,6 +1761,337 @@ void isal_inflate_reset(struct inflate_state *state)
state->tmp_out_valid = 0;
}
static inline uint32_t fixed_size_read(struct inflate_state *state,
uint8_t ** read_buf, int read_size)
{
uint32_t tmp_in_size = state->tmp_in_size;
if (state->avail_in + tmp_in_size < read_size) {
memcpy(state->tmp_in_buffer + tmp_in_size, state->next_in, state->avail_in);
tmp_in_size += state->avail_in;
state->tmp_in_size = tmp_in_size;
state->next_in += state->avail_in;
state->avail_in = 0;
return ISAL_END_INPUT;
}
*read_buf = state->next_in;
if (tmp_in_size) {
memcpy(state->tmp_in_buffer + tmp_in_size, state->next_in,
read_size - tmp_in_size);
*read_buf = state->tmp_in_buffer;
state->tmp_in_size = 0;
}
state->next_in += read_size - tmp_in_size;
state->avail_in -= read_size - tmp_in_size;
tmp_in_size = 0;
return 0;
}
static inline uint32_t buffer_header_copy(struct inflate_state *state, uint32_t in_len,
uint8_t * buf, uint32_t buf_len, uint32_t buf_error)
{
uint32_t len = in_len;
if (len > state->avail_in)
len = state->avail_in;
if (buf != NULL && buf_len < len) {
memcpy(buf, state->next_in, buf_len);
state->next_in += buf_len;
state->avail_in -= buf_len;
state->count = in_len - buf_len;
return buf_error;
} else {
if (buf != NULL)
memcpy(buf, state->next_in, len);
state->next_in += len;
state->avail_in -= len;
state->count = in_len - len;
if (len == in_len)
return 0;
else
return ISAL_END_INPUT;
}
}
static inline uint32_t string_header_copy(struct inflate_state *state,
char *str_buf, uint32_t str_len, uint32_t str_error)
{
uint32_t len, max_len = str_len;
if (max_len > state->avail_in)
max_len = state->avail_in;
len = strnlen((char *)state->next_in, max_len);
if (str_buf != NULL)
memcpy(str_buf, state->next_in, len);
state->next_in += len;
state->avail_in -= len;
state->count += len;
if (len == str_len)
return str_error;
else if (state->avail_in <= 0)
return ISAL_END_INPUT;
else {
state->next_in++;
state->avail_in--;
state->count = 0;
if (str_buf != NULL)
str_buf[len] = 0;
}
return 0;
}
static int check_gzip_checksum(struct inflate_state *state)
{
uint64_t trailer, crc, total_out;
uint8_t *next_in;
uint32_t byte_count, offset, tmp_in_size = state->tmp_in_size;
int ret;
if (state->read_in_length >= 8) {
byte_count = state->read_in_length / 8;
offset = state->read_in_length % 8;
*(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = state->read_in >> offset;
state->read_in = 0;
state->read_in_length = 0;
tmp_in_size += byte_count;
state->tmp_in_size = tmp_in_size;
}
ret = fixed_size_read(state, &next_in, GZIP_TRAILER_LEN);
if (ret) {
state->block_state = ISAL_CHECKSUM_CHECK;
return ret;
}
state->block_state = ISAL_BLOCK_FINISH;
trailer = *(uint64_t *) next_in;
crc = state->crc;
total_out = state->total_out;
if (trailer != (crc | (total_out << 32)))
return ISAL_INCORRECT_CHECKSUM;
else
return ISAL_DECOMP_OK;
}
static int check_zlib_checksum(struct inflate_state *state)
{
uint32_t trailer;
uint8_t *next_in;
uint32_t byte_count, offset, tmp_in_size = state->tmp_in_size;
int ret;
if (state->read_in_length >= 8) {
byte_count = state->read_in_length / 8;
offset = state->read_in_length % 8;
*(uint64_t *) (state->tmp_in_buffer + tmp_in_size) = state->read_in >> offset;
state->read_in = 0;
state->read_in_length = 0;
tmp_in_size += byte_count;
state->tmp_in_size = tmp_in_size;
}
ret = fixed_size_read(state, &next_in, ZLIB_TRAILER_LEN);
if (ret) {
state->block_state = ISAL_CHECKSUM_CHECK;
return ret;
}
state->block_state = ISAL_BLOCK_FINISH;
trailer = *(uint32_t *) next_in;
if (bswap_32(trailer) != state->crc)
return ISAL_INCORRECT_CHECKSUM;
else
return ISAL_DECOMP_OK;
}
int isal_read_gzip_header(struct inflate_state *state, struct isal_gzip_header *gz_hdr)
{
int cm, flags = gz_hdr->flags, id1, id2;
uint16_t xlen = gz_hdr->extra_len;
uint32_t block_state = state->block_state;
uint8_t *start_in = state->next_in, *next_in;
uint32_t tmp_in_size = state->tmp_in_size;
uint32_t count = state->count, offset;
uint32_t hcrc = gz_hdr->hcrc;
int ret = 0;
/* This switch is a jump table into the function so that decoding the
* header can continue where it stopped on the last call */
switch (block_state) {
case ISAL_BLOCK_NEW_HDR:
state->count = 0;
flags = UNDEFINED_FLAG;
if (tmp_in_size == 0)
hcrc = 0;
ret = fixed_size_read(state, &next_in, GZIP_HDR_BASE);
if (ret)
break;
id1 = next_in[0];
id2 = next_in[1];
cm = next_in[2];
flags = next_in[3];
gz_hdr->time = *(uint32_t *) (next_in + 4);
gz_hdr->xflags = *(next_in + 8);
gz_hdr->os = *(next_in + 9);
if (id1 != 0x1f || id2 != 0x8b)
return ISAL_INVALID_WRAPPER;
if (cm != DEFLATE_METHOD)
return ISAL_UNSUPPORTED_METHOD;
gz_hdr->text = 0;
if (flags & TEXT_FLAG)
gz_hdr->text = 1;
gz_hdr->flags = flags;
if (flags & EXTRA_FLAG) {
case ISAL_GZIP_EXTRA_LEN:
ret = fixed_size_read(state, &next_in, GZIP_EXTRA_LEN);
if (ret) {
state->block_state = ISAL_GZIP_EXTRA_LEN;
break;
}
xlen = *(uint16_t *) next_in;
count = xlen;
gz_hdr->extra_len = xlen;
case ISAL_GZIP_EXTRA:
offset = gz_hdr->extra_len - count;
ret =
buffer_header_copy(state, count, gz_hdr->extra + offset,
gz_hdr->extra_buf_len - offset,
ISAL_EXTRA_OVERFLOW);
if (ret) {
state->block_state = ISAL_GZIP_EXTRA;
break;
}
} else {
gz_hdr->extra_len = 0;
}
if (flags & NAME_FLAG) {
case ISAL_GZIP_NAME:
offset = state->count;
ret = string_header_copy(state, gz_hdr->name + offset,
gz_hdr->name_buf_len - offset,
ISAL_NAME_OVERFLOW);
if (ret) {
state->block_state = ISAL_GZIP_NAME;
break;
}
}
if (flags & COMMENT_FLAG) {
case ISAL_GZIP_COMMENT:
offset = state->count;
ret = string_header_copy(state, gz_hdr->comment + offset,
gz_hdr->comment_buf_len - offset,
ISAL_COMMENT_OVERFLOW);
if (ret) {
state->block_state = ISAL_GZIP_COMMENT;
break;
}
}
if (flags & HCRC_FLAG) {
hcrc = crc32_gzip(hcrc, start_in, state->next_in - start_in);
gz_hdr->hcrc = hcrc;
case ISAL_GZIP_HCRC:
ret = fixed_size_read(state, &next_in, GZIP_HCRC_LEN);
if (ret) {
state->block_state = ISAL_GZIP_HCRC;
return ret;
}
if ((hcrc & 0xffff) != *(uint16_t *) next_in)
return ISAL_INCORRECT_CHECKSUM;
}
state->wrapper_flag = 1;
state->block_state = ISAL_BLOCK_NEW_HDR;
return ISAL_DECOMP_OK;
}
if (flags & HCRC_FLAG)
gz_hdr->hcrc = crc32_gzip(hcrc, start_in, state->next_in - start_in);
return ret;
}
int isal_read_zlib_header(struct inflate_state *state, struct isal_zlib_header *zlib_hdr)
{
int cmf, method, flags;
uint32_t block_state = state->block_state;
uint8_t *next_in;
int ret = 0;
switch (block_state) {
case ISAL_BLOCK_NEW_HDR:
zlib_hdr->dict_flag = 0;
ret = fixed_size_read(state, &next_in, ZLIB_HDR_BASE);
if (ret)
break;
cmf = *next_in;
method = cmf & 0xf;
flags = *(next_in + 1);
zlib_hdr->info = cmf >> ZLIB_INFO_OFFSET;
zlib_hdr->dict_flag = (flags & ZLIB_DICT_FLAG) ? 1 : 0;
zlib_hdr->level = flags >> ZLIB_LEVEL_OFFSET;
if (method != DEFLATE_METHOD)
return ISAL_UNSUPPORTED_METHOD;
if ((256 * cmf + flags) % 31 != 0)
return ISAL_INCORRECT_CHECKSUM;
if (zlib_hdr->dict_flag) {
case ISAL_ZLIB_DICT:
ret = fixed_size_read(state, &next_in, ZLIB_DICT_LEN);
if (ret) {
state->block_state = ISAL_ZLIB_DICT;
break;
}
zlib_hdr->dict_id = *(int32_t *) next_in;
}
state->wrapper_flag = 1;
state->block_state = ISAL_BLOCK_NEW_HDR;
}
return ret;
}
int isal_inflate_set_dict(struct inflate_state *state, uint8_t * dict, uint32_t dict_len)
{
@ -1758,6 +2125,22 @@ int isal_inflate_stateless(struct inflate_state *state)
state->crc = 0;
state->total_out = 0;
state->hist_bits = 0;
state->tmp_in_size = 0;
if (state->crc_flag == IGZIP_GZIP) {
struct isal_gzip_header gz_hdr;
ret = isal_read_gzip_header(state, &gz_hdr);
if (ret)
return ret;
} else if (state->crc_flag == IGZIP_ZLIB) {
struct isal_zlib_header z_hdr;
ret = isal_read_zlib_header(state, &z_hdr);
if (ret)
return ret;
if (z_hdr.dict_flag)
return ISAL_NEED_DICT;
}
while (state->block_state != ISAL_BLOCK_FINISH) {
if (state->block_state == ISAL_BLOCK_NEW_HDR) {
@ -1778,14 +2161,31 @@ int isal_inflate_stateless(struct inflate_state *state)
state->block_state = ISAL_BLOCK_FINISH;
}
if (state->crc_flag) {
update_checksum(state, start_out, state->next_out - start_out);
if (state->crc_flag == ISAL_ZLIB || state->crc_flag == ISAL_ZLIB_NO_HDR)
finalize_adler32(state);
}
/* Undo count stuff of bytes read into the read buffer */
state->next_in -= state->read_in_length / 8;
state->avail_in += state->read_in_length / 8;
state->read_in_length = 0;
state->read_in = 0;
if (!ret && state->crc_flag) {
update_checksum(state, start_out, state->next_out - start_out);
switch (state->crc_flag) {
case ISAL_ZLIB:
case ISAL_ZLIB_NO_HDR_VER:
finalize_adler32(state);
ret = check_zlib_checksum(state);
break;
case ISAL_ZLIB_NO_HDR:
finalize_adler32(state);
break;
case ISAL_GZIP:
case ISAL_GZIP_NO_HDR_VER:
ret = check_gzip_checksum(state);
break;
}
}
return ret;
}
@ -1799,6 +2199,40 @@ int isal_inflate(struct inflate_state *state)
int32_t shift_size = 0;
int ret = 0;
if (!state->wrapper_flag && state->crc_flag == IGZIP_GZIP) {
struct isal_gzip_header gz_hdr;
ret = isal_read_gzip_header(state, &gz_hdr);
if (ret < 0)
return ret;
else if (ret > 0)
return ISAL_DECOMP_OK;
} else if (!state->wrapper_flag && state->crc_flag == IGZIP_ZLIB) {
struct isal_zlib_header z_hdr;
ret = isal_read_zlib_header(state, &z_hdr);
if (ret < 0)
return ret;
else if (ret > 0)
return ISAL_DECOMP_OK;
if (z_hdr.dict_flag) {
state->dict_id = z_hdr.dict_id;
return ISAL_NEED_DICT;
}
} else if (state->block_state == ISAL_CHECKSUM_CHECK) {
switch (state->crc_flag) {
case ISAL_ZLIB:
case ISAL_ZLIB_NO_HDR_VER:
ret = check_zlib_checksum(state);
break;
case ISAL_GZIP:
case ISAL_GZIP_NO_HDR_VER:
ret = check_gzip_checksum(state);
break;
}
return (ret > 0) ? ISAL_DECOMP_OK : ret;
}
if (state->block_state != ISAL_BLOCK_FINISH) {
state->total_out += state->tmp_out_valid - state->tmp_out_processed;
/* If space in tmp_out buffer, decompress into the tmp_out_buffer */
@ -1959,13 +2393,27 @@ int isal_inflate(struct inflate_state *state)
if (state->block_state == ISAL_BLOCK_INPUT_DONE
&& state->tmp_out_valid == state->tmp_out_processed) {
state->block_state = ISAL_BLOCK_FINISH;
if (state->crc_flag == ISAL_ZLIB
|| state->crc_flag == ISAL_ZLIB_NO_HDR)
switch (state->crc_flag) {
case ISAL_ZLIB:
case ISAL_ZLIB_NO_HDR_VER:
finalize_adler32(state);
ret = check_zlib_checksum(state);
break;
case ISAL_ZLIB_NO_HDR:
finalize_adler32(state);
break;
case ISAL_GZIP:
case ISAL_GZIP_NO_HDR_VER:
ret = check_gzip_checksum(state);
break;
}
}
state->total_out -= state->tmp_out_valid - state->tmp_out_processed;
}
return ISAL_DECOMP_OK;
return (ret > 0) ? ISAL_DECOMP_OK : ret;
}

162
igzip/igzip_rand_test.c
View File

@ -66,7 +66,7 @@
#define MAX_LOOPS 20
/* Defines for the possible error conditions */
enum IGZIP_TEST_ERROR_CODES {
IGZIP_COMP_OK,
IGZIP_COMP_OK = 0,
MALLOC_FAILED,
FILE_READ_FAILED,
@ -94,6 +94,8 @@ enum IGZIP_TEST_ERROR_CODES {
INVALID_ZLIB_HEADER,
INCORRECT_ZLIB_TRAILER,
UNSUPPORTED_METHOD,
INFLATE_GENERAL_ERROR,
INVALID_FLUSH_ERROR,
@ -104,17 +106,6 @@ enum IGZIP_TEST_ERROR_CODES {
static const int hdr_bytes = 300;
static const uint8_t gzip_hdr[10] = {
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xff
};
static const uint8_t zlib_hdr[2] = {
0x78, 0x01
};
static const uint32_t gzip_hdr_bytes = 10;
static const uint32_t zlib_hdr_bytes = 2;
static const uint32_t gzip_trl_bytes = 8;
static const uint32_t zlib_trl_bytes = 4;
static const int gzip_extra_bytes = 18; /* gzip_hdr_bytes + gzip_trl_bytes */
@ -347,6 +338,9 @@ void print_error(int error_code)
case INCORRECT_ZLIB_TRAILER:
printf("error: incorrect zlib trailer found when inflating data\n");
break;
case UNSUPPORTED_METHOD:
printf("error: invalid compression method in wrapper header\n");
break;
case INVALID_FLUSH_ERROR:
printf("error: invalid flush did not cause compression to error\n");
break;
@ -377,51 +371,6 @@ void print_uint8_t(uint8_t * array, uint64_t length)
printf("\n");
}
uint32_t check_gzip_header(uint8_t * z_buf)
{
/* These values are defined in RFC 1952 page 4 */
const uint8_t ID1 = 0x1f, ID2 = 0x8b, CM = 0x08, FLG = 0;
uint32_t ret = 0;
int i;
/* Verify that the gzip header is the one used in hufftables_c.c */
for (i = 0; i < gzip_hdr_bytes; i++)
if (z_buf[i] != gzip_hdr[i])
ret = INVALID_GZIP_HEADER;
/* Verify that the gzip header is a valid gzip header */
if (*z_buf++ != ID1)
ret = INVALID_GZIP_HEADER;
if (*z_buf++ != ID2)
ret = INVALID_GZIP_HEADER;
/* Verfiy compression method is Deflate */
if (*z_buf++ != CM)
ret = INVALID_GZIP_HEADER;
/* The following comparison is specific to how gzip headers are written in igzip */
/* Verify no extra flags are set */
if (*z_buf != FLG)
ret = INVALID_GZIP_HEADER;
/* The last 6 bytes in the gzip header do not contain any information
* important to decomrpessing the data */
return ret;
}
uint32_t check_zlib_header(uint8_t * z_buf)
{
/* These values are defined in RFC 1952 page 4 */
uint32_t ret = 0;
int i;
/* Verify that the gzip header is the one used in hufftables_c.c */
for (i = 0; i < zlib_hdr_bytes; i++)
if (z_buf[i] != zlib_hdr[i])
ret = INVALID_ZLIB_HEADER;
return ret;
}
uint32_t check_gzip_trl(uint64_t gzip_trl, uint32_t inflate_crc, uint8_t * uncompress_buf,
uint32_t uncompress_len)
{
@ -460,12 +409,33 @@ int inflate_stateless_pass(uint8_t * compress_buf, uint64_t compress_len,
uint32_t gzip_flag)
{
struct inflate_state state;
int ret = 0;
int ret = 0, offset = 0;
struct isal_gzip_header gz_hdr;
struct isal_zlib_header z_hdr;
state.next_in = compress_buf;
state.avail_in = compress_len;
state.next_out = uncompress_buf;
state.avail_out = *uncompress_len;
if (gzip_flag == IGZIP_GZIP) {
if (rand() % 2 == 0) {
memset(&gz_hdr, 0, sizeof(gz_hdr));
isal_inflate_reset(&state);
state.tmp_in_size = 0;
gzip_flag = ISAL_GZIP_NO_HDR_VER;
isal_read_gzip_header(&state, &gz_hdr);
}
} else if (gzip_flag == IGZIP_ZLIB) {
if (rand() % 2 == 0) {
memset(&z_hdr, 0, sizeof(z_hdr));
isal_inflate_reset(&state);
gzip_flag = ISAL_ZLIB_NO_HDR_VER;
isal_read_zlib_header(&state, &z_hdr);
}
}
state.crc_flag = gzip_flag;
ret = isal_inflate_stateless(&state);
@ -473,18 +443,30 @@ int inflate_stateless_pass(uint8_t * compress_buf, uint64_t compress_len,
*uncompress_len = state.total_out;
if (gzip_flag) {
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR) {
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR
|| gzip_flag == ISAL_GZIP_NO_HDR_VER) {
if (gzip_flag == IGZIP_GZIP || gzip_flag == ISAL_GZIP_NO_HDR_VER)
offset = gzip_trl_bytes;
if (!ret)
ret =
check_gzip_trl(*(uint64_t *) state.next_in, state.crc,
uncompress_buf, *uncompress_len);
state.avail_in -= gzip_trl_bytes;
} else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR) {
check_gzip_trl(*(uint64_t *) (state.next_in - offset),
state.crc, uncompress_buf, *uncompress_len);
else if (ret == ISAL_INCORRECT_CHECKSUM)
ret = INCORRECT_GZIP_TRAILER;
state.avail_in -= (gzip_trl_bytes - offset);
} else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR
|| gzip_flag == ISAL_ZLIB_NO_HDR_VER) {
if (gzip_flag == IGZIP_ZLIB || gzip_flag == ISAL_ZLIB_NO_HDR_VER)
offset = zlib_trl_bytes;
if (!ret)
ret =
check_zlib_trl(*(uint32_t *) state.next_in, state.crc,
uncompress_buf, *uncompress_len);
state.avail_in -= zlib_trl_bytes;
check_zlib_trl(*(uint32_t *) (state.next_in - offset),
state.crc, uncompress_buf, *uncompress_len);
else if (ret == ISAL_INCORRECT_CHECKSUM)
ret = INCORRECT_ZLIB_TRAILER;
state.avail_in -= (zlib_trl_bytes - offset);
}
@ -585,6 +567,18 @@ int inflate_multi_pass(uint8_t * compress_buf, uint64_t compress_len,
create_rand_repeat_data((uint8_t *) state, sizeof(state));
}
if (gzip_flag == IGZIP_GZIP_NO_HDR) {
if (rand() % 2 == 0)
compress_len -= gzip_trl_bytes;
else
gzip_flag = ISAL_GZIP_NO_HDR_VER;
} else if (gzip_flag == IGZIP_ZLIB_NO_HDR) {
if (rand() % 2 == 0)
compress_len -= zlib_trl_bytes;
else
gzip_flag = ISAL_ZLIB_NO_HDR_VER;
}
state->next_in = NULL;
state->next_out = NULL;
state->avail_in = 0;
@ -598,11 +592,6 @@ int inflate_multi_pass(uint8_t * compress_buf, uint64_t compress_len,
if (dict != NULL)
isal_inflate_set_dict(state, dict, dict_len);
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR)
compress_len -= gzip_trl_bytes;
else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR)
compress_len -= zlib_trl_bytes;
while (1) {
if (state->avail_in == 0) {
comp_tmp_size = rand() % (compress_len + 1);
@ -714,12 +703,19 @@ int inflate_multi_pass(uint8_t * compress_buf, uint64_t compress_len,
if (gzip_flag) {
if (!ret) {
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR) {
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR
|| gzip_flag == ISAL_GZIP_NO_HDR_VER) {
if (gzip_flag == ISAL_GZIP_NO_HDR_VER
|| gzip_flag == IGZIP_GZIP)
compress_len -= gzip_trl_bytes;
ret =
check_gzip_trl(*(uint64_t *) & compress_buf[compress_len],
state->crc, uncompress_buf,
*uncompress_len);
} else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR) {
} else if (gzip_flag == IGZIP_ZLIB_NO_HDR) {
if (gzip_flag == IGZIP_ZLIB
|| gzip_flag == ISAL_ZLIB_NO_HDR_VER)
compress_len -= zlib_trl_bytes;
ret =
check_zlib_trl(*(uint32_t *) & compress_buf[compress_len],
state->crc, uncompress_buf,
@ -786,16 +782,6 @@ int inflate_check(uint8_t * z_buf, uint32_t z_size, uint8_t * in_buf, uint32_t i
if (test_buf != NULL)
memset(test_buf, 0xff, test_size);
if (gzip_flag == IGZIP_GZIP) {
gzip_hdr_result = check_gzip_header(z_buf);
z_buf += gzip_hdr_bytes;
z_size -= gzip_hdr_bytes;
} else if (gzip_flag == IGZIP_ZLIB) {
gzip_hdr_result = check_zlib_header(z_buf);
z_buf += zlib_hdr_bytes;
z_size -= zlib_hdr_bytes;
}
if (inflate_type == 0 && dict == NULL) {
ret = inflate_stateless_pass(z_buf, z_size, test_buf, &test_size, gzip_flag);
inflate_type = 1;
@ -851,6 +837,16 @@ int inflate_check(uint8_t * z_buf, uint32_t z_size, uint8_t * in_buf, uint32_t i
case INCORRECT_ZLIB_TRAILER:
gzip_trl_result = INCORRECT_ZLIB_TRAILER;
break;
case ISAL_INCORRECT_CHECKSUM:
if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR
|| gzip_flag == ISAL_GZIP_NO_HDR_VER)
gzip_trl_result = INCORRECT_GZIP_TRAILER;
else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR
|| gzip_flag == ISAL_ZLIB_NO_HDR_VER)
gzip_trl_result = INCORRECT_GZIP_TRAILER;
break;
case ISAL_UNSUPPORTED_METHOD:
return UNSUPPORTED_METHOD;
case INFLATE_INPUT_STREAM_INTEGRITY_ERROR:
return INFLATE_INPUT_STREAM_INTEGRITY_ERROR;
break;

867
igzip/igzip_wrapper_hdr_test.c Normal file
View File

@ -0,0 +1,867 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "checksum_test_ref.h"
#include "igzip_lib.h"
#define TEST_SEED 0x1234
#define RANDOMS 0x4000
#define EXTRA_SIZE_MAX 256
#define NAME_SIZE_MAX 256
#define COMMENT_SIZE_MAX 1024
#define EXTRA_SIZE 10
#define NAME_SIZE 25
#define COMMENT_SIZE 192
#define TEXT_FLAG (1 << 0)
#define HCRC_FLAG (1 << 1)
#define EXTRA_FLAG (1 << 2)
#define NAME_FLAG (1 << 3)
#define COMMENT_FLAG (1 << 4)
#define DICT_FLAG (1 << 5)
enum {
INVALID_WRAPPER = ISAL_INVALID_WRAPPER,
UNSUPPORTED_METHOD = ISAL_UNSUPPORTED_METHOD,
INCORRECT_CHECKSUM = ISAL_INCORRECT_CHECKSUM,
NO_ERROR = ISAL_DECOMP_OK,
END_INPUT = ISAL_END_INPUT,
NAME_OVERFLOW = ISAL_NAME_OVERFLOW,
COMMENT_OVERFLOW = ISAL_COMMENT_OVERFLOW,
EXTRA_OVERFLOW = ISAL_EXTRA_OVERFLOW,
INCORRECT_TEXT_FLAG,
INCORRECT_TIME,
INCORRECT_XFLAGS,
INCORRECT_OS,
INCORRECT_EXTRA_LEN,
INCORRECT_EXTRA_BUF,
INCORRECT_NAME,
INCORRECT_COMMENT,
INCORRECT_INFO,
INCORRECT_LEVEL,
INCORRECT_DICT_FLAG,
INCORRECT_DICT_ID,
MALLOC_FAILED
};
void print_error(int32_t error)
{
printf("Error Code %d: ", error);
switch (error) {
case END_INPUT:
printf("End of input reached before header decompressed\n");
break;
case INVALID_WRAPPER:
printf("Invalid gzip wrapper found\n");
break;
case UNSUPPORTED_METHOD:
printf("Unsupported decompression method found\n");
break;
case INCORRECT_CHECKSUM:
printf("Incorrect header checksum found\n");
break;
case NAME_OVERFLOW:
printf("Name buffer overflow while decompression\n");
break;
case COMMENT_OVERFLOW:
printf("Comment buffer overflow while decompressing\n");
case EXTRA_OVERFLOW:
printf("Extra buffer overflow while decomrpessiong\n");
break;
case INCORRECT_TEXT_FLAG:
printf("Incorrect text field found\n");
break;
case INCORRECT_TIME:
printf("Incorrect time filed found\n");
break;
case INCORRECT_XFLAGS:
printf("Incorrect xflags field found\n");
break;
case INCORRECT_OS:
printf("Incorrect os field found\n");
break;
case INCORRECT_EXTRA_LEN:
printf("Incorect extra_len field found\n");
break;
case INCORRECT_EXTRA_BUF:
printf("Incorrect extra buffer found\n");
break;
case INCORRECT_NAME:
printf("Incorrect name found\n");
break;
case INCORRECT_COMMENT:
printf("Incorrect comment found\n");
break;
case INCORRECT_INFO:
printf("Incorrect info found\n");
break;
case INCORRECT_LEVEL:
printf("Incorrect level found\n");
break;
case INCORRECT_DICT_FLAG:
printf("Incorrect dictionary flag found\n");
break;
case INCORRECT_DICT_ID:
printf("Incorrect dictionary id found\n");
break;
case MALLOC_FAILED:
printf("Failed to allocate buffers\n");
break;
case NO_ERROR:
printf("No error found\n");
}
}
void print_uint8_t(uint8_t * array, uint64_t length, char *prepend)
{
const int line_size = 16;
int i;
if (array == NULL)
printf("%s(NULL)", prepend);
else if (length == 0)
printf("%s(Empty)", prepend);
for (i = 0; i < length; i++) {
if (i == 0)
printf("%s0x%04x\t", prepend, i);
else if ((i % line_size) == 0)
printf("\n%s0x%04x\t", prepend, i);
else
printf(" ");
printf("0x%02x,", array[i]);
}
printf("\n");
}
void print_string(char *str, uint32_t str_max_len, char *prepend)
{
const int line_size = 64;
uint32_t i = 0;
while (str[i] != 0 && i < str_max_len) {
if (i == 0)
printf("%s0x%04x\t", prepend, i);
else if ((i % line_size) == 0)
printf("\n%s0x%04x\t", prepend, i);
printf("%c", str[i]);
i++;
}
printf("\n");
}
void print_gzip_header(struct isal_gzip_header *gz_hdr, char *prepend1, char *prepend2)
{
printf("%sText: %d, Time: 0x%08x, Xflags: 0x%x, OS: 0x%x\n", prepend1,
gz_hdr->text, gz_hdr->time, gz_hdr->xflags, gz_hdr->os);
printf("%sExtra: Extra_len = 0x%x\n", prepend1, gz_hdr->extra_len);
if (gz_hdr->extra_len < EXTRA_SIZE_MAX)
print_uint8_t(gz_hdr->extra, gz_hdr->extra_len, prepend2);
else
printf("%sExtra field larger than EXTRA_SIZE_MAX\n", prepend2);
printf("%sName:\n", prepend1);
if (gz_hdr->name_buf_len < NAME_SIZE_MAX)
print_string(gz_hdr->name, gz_hdr->name_buf_len, prepend2);
else
printf("%sName field larger than NAME_SIZE_MAX\n", prepend2);
printf("%sComment:\n", prepend1);
if (gz_hdr->comment_buf_len < COMMENT_SIZE_MAX)
print_string(gz_hdr->comment, gz_hdr->comment_buf_len, prepend2);
else
printf("%sComment field larger than COMMENT_SIZE_MAX\n", prepend2);
}
void print_zlib_header(struct isal_zlib_header *z_hdr, char *prepend)
{
printf("%sInfo: 0x%x\n", prepend, z_hdr->info);
printf("%sLevel: 0x%x\n", prepend, z_hdr->level);
printf("%sDictionary: Flag = 0x%x, Id =0x%x\n", prepend, z_hdr->dict_flag,
z_hdr->dict_id);
}
void print_gzip_final_verbose(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_gzip_header *gz_hdr_orig,
struct isal_gzip_header *gz_hdr)
{
#ifdef VERBOSE
printf("\n");
if (gz_hdr_orig != NULL) {
printf("Original Gzip Header Struct:\n");
print_gzip_header(gz_hdr_orig, "\t", "\t\t");
printf("\n");
}
if (gz_hdr != NULL) {
printf("Parsed Gzip Header Struct:\n");
print_gzip_header(gz_hdr, "\t", "\t\t");
printf("\n");
}
if (hdr_buf != NULL) {
printf("Serialized Gzip Header:\n");
print_uint8_t(hdr_buf, hdr_buf_len, "\t");
printf("\n");
}
#endif
return;
}
void print_zlib_final_verbose(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_zlib_header *z_hdr_orig,
struct isal_zlib_header *z_hdr)
{
#ifdef VERBOSE
printf("\n");
if (z_hdr_orig != NULL) {
printf("Original Zlib Header Struct:\n");
print_zlib_header(z_hdr_orig, "\t");
printf("\n");
}
if (z_hdr != NULL) {
printf("Parsed Zlib Header Struct:\n");
print_zlib_header(z_hdr, "\t");
printf("\n");
}
if (hdr_buf != NULL) {
printf("Serialized Zlib Header:\n");
print_uint8_t(hdr_buf, hdr_buf_len, "\t");
printf("\n");
}
#endif
return;
}
int gzip_header_size(struct isal_gzip_header *gz_hdr)
{
int hdr_size = 10;
if (gz_hdr->extra != NULL) {
hdr_size += 2 + gz_hdr->extra_len;
}
if (gz_hdr->name != NULL) {
hdr_size += strnlen(gz_hdr->name, gz_hdr->name_buf_len) + 1;
}
if (gz_hdr->comment != NULL) {
hdr_size += strnlen(gz_hdr->comment, gz_hdr->comment_buf_len) + 1;
}
if (gz_hdr->hcrc) {
hdr_size += 2;
}
return hdr_size;
}
int zlib_header_size(struct isal_zlib_header *z_hdr)
{
if (z_hdr->dict_flag)
return 6;
else
return 2;
}
void rand_string(char *string, uint32_t str_len)
{
int i;
if (str_len == 0 || string == NULL)
return;
for (i = 0; i < str_len - 1; i++) {
string[i] = rand() % 26 + 65;
}
string[str_len - 1] = 0;
}
void rand_buf(uint8_t * buf, uint32_t buf_len)
{
int i;
if (buf_len == 0 || buf == NULL)
return;
for (i = 0; i < buf_len; i++) {
buf[i] = rand();
}
}
int malloc_gzip_header(struct isal_gzip_header *gz_hdr)
{
gz_hdr->extra = NULL;
if (gz_hdr->extra_buf_len) {
gz_hdr->extra = malloc(gz_hdr->extra_buf_len);
if (gz_hdr->extra == NULL)
return MALLOC_FAILED;
}
gz_hdr->name = NULL;
if (gz_hdr->name_buf_len) {
gz_hdr->name = malloc(gz_hdr->name_buf_len);
if (gz_hdr->name == NULL)
return MALLOC_FAILED;
}
gz_hdr->comment = NULL;
if (gz_hdr->comment_buf_len) {
gz_hdr->comment = malloc(gz_hdr->comment_buf_len);
if (gz_hdr->comment == NULL)
return MALLOC_FAILED;
}
return 0;
}
void free_gzip_header(struct isal_gzip_header *gz_hdr)
{
if (gz_hdr->extra != NULL) {
free(gz_hdr->extra);
gz_hdr->extra = NULL;
}
if (gz_hdr->name != NULL) {
free(gz_hdr->name);
gz_hdr->name = NULL;
}
if (gz_hdr->comment != NULL) {
free(gz_hdr->comment);
gz_hdr->comment = NULL;
}
}
int gen_rand_gzip_header(struct isal_gzip_header *gz_hdr)
{
int ret = 0;
gz_hdr->text = rand() % 2;
gz_hdr->time = rand();
gz_hdr->xflags = rand() % 256;
gz_hdr->os = rand() % 256;
if (rand() % 4 != 0) {
gz_hdr->extra_buf_len = rand() % EXTRA_SIZE_MAX;
gz_hdr->extra_len = gz_hdr->extra_buf_len;
} else {
gz_hdr->extra_buf_len = 0;
gz_hdr->extra_len = 0;
}
if (rand() % 4 != 0)
gz_hdr->name_buf_len = rand() % NAME_SIZE_MAX;
else
gz_hdr->name_buf_len = 0;
if (rand() % 4 != 0)
gz_hdr->comment_buf_len = rand() % COMMENT_SIZE_MAX;
else
gz_hdr->comment_buf_len = 0;
gz_hdr->hcrc = rand() % 2;
ret = malloc_gzip_header(gz_hdr);
if (ret)
return ret;
rand_buf(gz_hdr->extra, gz_hdr->extra_len);
rand_string(gz_hdr->name, gz_hdr->name_buf_len);
rand_string(gz_hdr->comment, gz_hdr->comment_buf_len);
return ret;
}
void gen_rand_zlib_header(struct isal_zlib_header *z_hdr)
{
z_hdr->info = rand() % 16;
z_hdr->level = rand() % 4;
z_hdr->dict_flag = rand() % 2;
z_hdr->dict_id = rand();
}
void write_gzip_header(uint8_t * out_buf, struct isal_gzip_header *gz_hdr)
{
uint32_t flags = 0, len, hcrc;
uint8_t *out_buf_start = out_buf;
if (gz_hdr->text)
flags |= TEXT_FLAG;
if (gz_hdr->extra)
flags |= EXTRA_FLAG;
if (gz_hdr->name)
flags |= NAME_FLAG;
if (gz_hdr->comment)
flags |= COMMENT_FLAG;
if (gz_hdr->hcrc)
flags |= HCRC_FLAG;
out_buf[0] = 0x1f;
out_buf[1] = 0x8b;
out_buf[2] = 8;
out_buf[3] = flags;
*(uint32_t *) (out_buf + 4) = gz_hdr->time;
out_buf[8] = gz_hdr->xflags;
out_buf[9] = gz_hdr->os;
out_buf += 10;
if (flags & EXTRA_FLAG) {
*(uint16_t *) out_buf = gz_hdr->extra_len;
out_buf += 2;
memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len);
out_buf += gz_hdr->extra_len;
}
if (flags & NAME_FLAG) {
len = strnlen(gz_hdr->name, gz_hdr->name_buf_len) + 1;
memcpy(out_buf, gz_hdr->name, len);
out_buf += len;
}
if (flags & COMMENT_FLAG) {
len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len) + 1;
memcpy(out_buf, gz_hdr->comment, len);
out_buf += len;
}
if (flags & HCRC_FLAG) {
hcrc = crc32_gzip_refl_ref(0, out_buf_start, out_buf - out_buf_start);
*(uint16_t *) out_buf = hcrc;
out_buf += 2;
}
}
void write_zlib_header(uint8_t * out_buf, struct isal_zlib_header *z_hdr)
{
uint32_t cmf, flg, dict_flag;
dict_flag = (z_hdr->dict_flag) ? DICT_FLAG : 0;
cmf = 8 | (z_hdr->info << 4);
flg = (z_hdr->level << 6) | dict_flag;
flg += 31 - ((256 * cmf + flg) % 31);
out_buf[0] = cmf;
out_buf[1] = flg;
if (dict_flag)
*(uint32_t *) (out_buf + 2) = z_hdr->dict_id;
}
int compare_gzip_headers(struct isal_gzip_header *gz_hdr1, struct isal_gzip_header *gz_hdr2)
{
int ret = 0;
uint32_t max_len;
if (gz_hdr1->text != gz_hdr2->text)
return INCORRECT_TEXT_FLAG;
if (gz_hdr1->time != gz_hdr2->time)
return INCORRECT_TIME;
if (gz_hdr1->xflags != gz_hdr2->xflags)
return INCORRECT_XFLAGS;
if (gz_hdr1->os != gz_hdr2->os)
return INCORRECT_OS;
if (gz_hdr1->extra_len != gz_hdr2->extra_len)
return INCORRECT_EXTRA_LEN;
if (gz_hdr1->extra != NULL && gz_hdr2->extra != NULL) {
ret = memcmp(gz_hdr1->extra, gz_hdr2->extra, gz_hdr1->extra_len);
if (ret)
return INCORRECT_EXTRA_BUF;
}
if (gz_hdr1->name != NULL && gz_hdr2->name != NULL) {
max_len = gz_hdr1->name_buf_len;
if (gz_hdr1->name_buf_len < gz_hdr2->name_buf_len)
max_len = gz_hdr2->name_buf_len;
ret = strncmp(gz_hdr1->name, gz_hdr2->name, max_len);
if (ret)
return INCORRECT_NAME;
}
if (gz_hdr1->comment != NULL && gz_hdr2->comment != NULL) {
max_len = gz_hdr1->comment_buf_len;
if (gz_hdr1->comment_buf_len < gz_hdr2->comment_buf_len)
max_len = gz_hdr2->comment_buf_len;
ret = strncmp(gz_hdr1->comment, gz_hdr2->comment, max_len);
if (ret)
return INCORRECT_COMMENT;
}
return ret;
}
int compare_zlib_headers(struct isal_zlib_header *z_hdr1, struct isal_zlib_header *z_hdr2)
{
if (z_hdr1->info != z_hdr2->info)
return INCORRECT_INFO;
if (z_hdr1->level != z_hdr2->level)
return INCORRECT_LEVEL;
if (z_hdr1->dict_flag != z_hdr2->dict_flag)
return INCORRECT_DICT_FLAG;
if (z_hdr1->dict_flag && z_hdr1->dict_id != z_hdr2->dict_id)
return INCORRECT_DICT_ID;
return 0;
}
int read_gzip_header_simple(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_gzip_header *gz_hdr_orig)
{
int ret = 0;
struct inflate_state state;
struct isal_gzip_header gz_hdr;
rand_buf((uint8_t *) & gz_hdr, sizeof(gz_hdr));
gz_hdr.extra_buf_len = gz_hdr_orig->extra_buf_len;
gz_hdr.name_buf_len = gz_hdr_orig->name_buf_len;
gz_hdr.comment_buf_len = gz_hdr_orig->comment_buf_len;
ret = malloc_gzip_header(&gz_hdr);
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, NULL);
print_error(ret);
free_gzip_header(&gz_hdr);
return ret;
}
isal_inflate_init(&state);
state.next_in = hdr_buf;
state.avail_in = hdr_buf_len;
ret = isal_read_gzip_header(&state, &gz_hdr);
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, &gz_hdr);
print_error(ret);
free_gzip_header(&gz_hdr);
return ret;
}
ret = compare_gzip_headers(gz_hdr_orig, &gz_hdr);
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, &gz_hdr);
print_error(ret);
}
free_gzip_header(&gz_hdr);
return ret;
}
int read_zlib_header_simple(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_zlib_header *z_hdr_orig)
{
int ret = 0;
struct inflate_state state;
struct isal_zlib_header z_hdr;
rand_buf((uint8_t *) & z_hdr, sizeof(z_hdr));
if (ret) {
print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr_orig, NULL);
print_error(ret);
return ret;
}
isal_inflate_init(&state);
state.next_in = hdr_buf;
state.avail_in = hdr_buf_len;
ret = isal_read_zlib_header(&state, &z_hdr);
if (ret) {
print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr_orig, &z_hdr);
print_error(ret);
return ret;
}
ret = compare_zlib_headers(z_hdr_orig, &z_hdr);
if (ret) {
print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr_orig, &z_hdr);
print_error(ret);
}
return ret;
}
int read_gzip_header_streaming(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_gzip_header *gz_hdr_orig)
{
int ret = 0;
uint32_t max_dec_size, dec_size, max_extra_len, extra_len;
uint32_t max_name_len, name_len, max_comment_len, comment_len;
struct inflate_state state;
struct isal_gzip_header gz_hdr;
void *tmp_ptr;
rand_buf((uint8_t *) & gz_hdr, sizeof(gz_hdr));
max_dec_size = (rand() % hdr_buf_len) + 2;
max_extra_len = 2;
max_name_len = 2;
max_comment_len = 2;
if (gz_hdr_orig->extra_buf_len)
max_extra_len = (rand() % gz_hdr_orig->extra_buf_len) + 2;
if (gz_hdr_orig->name_buf_len)
max_name_len = (rand() % gz_hdr_orig->name_buf_len) + 2;
if (gz_hdr_orig->comment_buf_len)
max_comment_len = (rand() % gz_hdr_orig->comment_buf_len) + 2;
extra_len = rand() % max_extra_len;
name_len = rand() % max_name_len;
comment_len = rand() % max_comment_len;
if (extra_len == 0)
extra_len = 1;
if (name_len == 0)
name_len = 1;
if (comment_len == 0)
comment_len = 1;
gz_hdr.extra_buf_len = extra_len;
gz_hdr.name_buf_len = name_len;
gz_hdr.comment_buf_len = comment_len;
ret = malloc_gzip_header(&gz_hdr);
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, NULL);
print_error(ret);
free_gzip_header(&gz_hdr);
return (ret == 0);
}
isal_inflate_init(&state);
state.next_in = hdr_buf;
dec_size = rand() % max_dec_size;
if (dec_size > hdr_buf_len)
dec_size = hdr_buf_len;
state.avail_in = dec_size;
hdr_buf_len -= dec_size;
while (1) {
ret = isal_read_gzip_header(&state, &gz_hdr);
switch (ret) {
case ISAL_NAME_OVERFLOW:
if (name_len >= NAME_SIZE_MAX)
break;
name_len += rand() % max_name_len;
tmp_ptr = realloc(gz_hdr.name, name_len);
if (tmp_ptr == NULL) {
ret = MALLOC_FAILED;
break;
}
gz_hdr.name = tmp_ptr;
gz_hdr.name_buf_len = name_len;
continue;
case ISAL_COMMENT_OVERFLOW:
if (comment_len >= COMMENT_SIZE_MAX)
break;
comment_len += rand() % max_comment_len;
tmp_ptr = realloc(gz_hdr.comment, comment_len);
if (tmp_ptr == NULL) {
ret = MALLOC_FAILED;
break;
}
gz_hdr.comment = tmp_ptr;
gz_hdr.comment_buf_len = comment_len;
continue;
case ISAL_EXTRA_OVERFLOW:
if (extra_len >= EXTRA_SIZE_MAX)
break;
extra_len += rand() % max_extra_len;
tmp_ptr = realloc(gz_hdr.extra, extra_len);
if (tmp_ptr == NULL) {
ret = MALLOC_FAILED;
break;
}
gz_hdr.extra = tmp_ptr;
gz_hdr.extra_buf_len = extra_len;
continue;
case ISAL_END_INPUT:
if (hdr_buf_len == 0)
break;
dec_size = rand() % max_dec_size;
if (dec_size > hdr_buf_len)
dec_size = hdr_buf_len;
state.avail_in = dec_size;
hdr_buf_len -= dec_size;
continue;
}
break;
}
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, &gz_hdr);
print_error(ret);
free_gzip_header(&gz_hdr);
return ret;
}
ret = compare_gzip_headers(gz_hdr_orig, &gz_hdr);
if (ret) {
print_gzip_final_verbose(hdr_buf, hdr_buf_len, gz_hdr_orig, &gz_hdr);
print_error(ret);
}
free_gzip_header(&gz_hdr);
return ret;
}
int read_zlib_header_streaming(uint8_t * hdr_buf, uint32_t hdr_buf_len,
struct isal_zlib_header *z_hdr_orig)
{
int ret = ISAL_END_INPUT;
uint32_t max_dec_size, dec_size;
struct inflate_state state;
struct isal_zlib_header z_hdr;
rand_buf((uint8_t *) & z_hdr, sizeof(z_hdr));
max_dec_size = (rand() % hdr_buf_len) + 2;
isal_inflate_init(&state);
state.next_in = hdr_buf;
while (ret == ISAL_END_INPUT && hdr_buf_len > 0) {
dec_size = rand() % max_dec_size;
if (dec_size > hdr_buf_len)
dec_size = hdr_buf_len;
state.avail_in = dec_size;
hdr_buf_len -= dec_size;
ret = isal_read_zlib_header(&state, &z_hdr);
}
if (ret) {
print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr_orig, &z_hdr);
print_error(ret);
return ret;
}
ret = compare_zlib_headers(z_hdr_orig, &z_hdr);
if (ret) {
print_zlib_final_verbose(hdr_buf, hdr_buf_len, z_hdr_orig, &z_hdr);
print_error(ret);
}
return ret;
}
int main(int argc, char *argv[])
{
uint8_t *hdr_buf;
uint32_t hdr_buf_len;
int ret = 0, fin_ret = 0;
struct isal_gzip_header gz_hdr_orig;
struct isal_zlib_header z_hdr_orig;
int i;
#ifndef VERBOSE
setbuf(stdout, NULL);
#endif
printf("Test Seed : %d\n", TEST_SEED);
printf("Randoms : %d\n", RANDOMS);
srand(TEST_SEED);
printf("gzip wrapper test: ");
for (i = 0; i < RANDOMS; i++) {
memset(&gz_hdr_orig, 0, sizeof(gz_hdr_orig));
ret = gen_rand_gzip_header(&gz_hdr_orig);
if (ret) {
print_error(ret);
return (ret == 0);
}
hdr_buf_len = gzip_header_size(&gz_hdr_orig);
hdr_buf = malloc(hdr_buf_len);
write_gzip_header(hdr_buf, &gz_hdr_orig);
ret = read_gzip_header_simple(hdr_buf, hdr_buf_len, &gz_hdr_orig);
fin_ret |= ret;
if (ret)
return (ret == 0);
ret = read_gzip_header_streaming(hdr_buf, hdr_buf_len, &gz_hdr_orig);
fin_ret |= ret;
if (ret)
return (ret == 0);
free_gzip_header(&gz_hdr_orig);
if (hdr_buf != NULL)
free(hdr_buf);
if (i % (RANDOMS / 16) == 0)
printf(".");
}
printf("Pass \n");
printf("zlib wrapper test: ");
for (i = 0; i < RANDOMS; i++) {
memset(&z_hdr_orig, 0, sizeof(z_hdr_orig));
gen_rand_zlib_header(&z_hdr_orig);
hdr_buf_len = zlib_header_size(&z_hdr_orig);
hdr_buf = malloc(hdr_buf_len);
write_zlib_header(hdr_buf, &z_hdr_orig);
ret = read_zlib_header_simple(hdr_buf, hdr_buf_len, &z_hdr_orig);
fin_ret |= ret;
if (ret)
return (ret == 0);
ret = read_zlib_header_streaming(hdr_buf, hdr_buf_len, &z_hdr_orig);
fin_ret |= ret;
if (ret)
return (ret == 0);
if (hdr_buf != NULL)
free(hdr_buf);
if (i % (RANDOMS / 16) == 0)
printf(".");
}
printf("Pass \n");
printf("igzip wrapper_hdr test finished:%s \n",
fin_ret ? " Some tests failed " : " All tests passed");
return 0;
}

123
include/igzip_lib.h
View File

@ -215,7 +215,14 @@ enum isal_block_state {
ISAL_BLOCK_TYPE0, /* Decoding a type 0 block */
ISAL_BLOCK_CODED, /* Decoding a huffman coded block */
ISAL_BLOCK_INPUT_DONE, /* Decompression of input is completed */
ISAL_BLOCK_FINISH /* Decompression of input is completed and all data has been flushed to output */
ISAL_BLOCK_FINISH, /* Decompression of input is completed and all data has been flushed to output */
ISAL_GZIP_EXTRA_LEN,
ISAL_GZIP_EXTRA,
ISAL_GZIP_NAME,
ISAL_GZIP_COMMENT,
ISAL_GZIP_HCRC,
ISAL_ZLIB_DICT,
ISAL_CHECKSUM_CHECK,
};
@ -225,14 +232,23 @@ enum isal_block_state {
#define ISAL_GZIP_NO_HDR 2
#define ISAL_ZLIB 3
#define ISAL_ZLIB_NO_HDR 4
#define ISAL_ZLIB_NO_HDR_VER 5
#define ISAL_GZIP_NO_HDR_VER 6
/* Inflate Return values */
#define ISAL_DECOMP_OK 0 /* No errors encountered while decompressing */
#define ISAL_END_INPUT 1 /* End of input reached */
#define ISAL_OUT_OVERFLOW 2 /* End of output reached */
#define ISAL_NAME_OVERFLOW 3 /* End of gzip name buffer reached */
#define ISAL_COMMENT_OVERFLOW 4 /* End of gzip name buffer reached */
#define ISAL_EXTRA_OVERFLOW 5 /* End of extra buffer reached */
#define ISAL_NEED_DICT 6 /* Stream needs a dictionary to continue */
#define ISAL_INVALID_BLOCK -1 /* Invalid deflate block found */
#define ISAL_INVALID_SYMBOL -2 /* Invalid deflate symbol found */
#define ISAL_INVALID_LOOKBACK -3 /* Invalid lookback distance found */
#define ISAL_INVALID_WRAPPER -4 /* Invalid gzip/zlib wrapper found */
#define ISAL_UNSUPPORTED_METHOD -5 /* Gzip/zlib wrapper specifies unsupported compress method */
#define ISAL_INCORRECT_CHECKSUM -6 /* Incorrect checksum found */
/******************************************************************************/
/* Compression structures */
@ -304,6 +320,29 @@ struct BitBuf2 {
uint8_t *m_out_start; //!< start of buffer to write to
};
struct isal_zlib_header {
uint32_t info; //!< base-2 logarithm of the LZ77 window size minus 8
uint32_t level; //!< Compression level (fastest, fast, default, maximum)
uint32_t dict_id; //!< Dictionary id
uint32_t dict_flag; //!< Whether to use a dictionary
};
struct isal_gzip_header {
uint32_t text; //!< Optional Text hint
uint32_t time; //!< Unix modification time in gzip header
uint32_t xflags; //!< xflags in gzip header
uint32_t os; //!< OS in gzip header
uint8_t *extra; //!< Extra field in gzip header
uint32_t extra_buf_len; //!< Length of extra buffer
uint32_t extra_len; //!< Actual length of gzip header extra field
char *name; //!< Name in gzip header
uint32_t name_buf_len; //!< Length of name buffer
char *comment; //!< Comments in gzip header
uint32_t comment_buf_len; //!< Length of comment buffer
uint32_t hcrc; //!< Header crc or header crc flag
uint32_t flags; //!< Internal data
};
/* Variable prefixes:
* b_ : Measured wrt the start of the buffer
* f_ : Measured wrt the start of the file (aka file_start)
@ -473,12 +512,17 @@ struct inflate_state {
uint32_t crc_flag; //!< Flag identifying whether to track of crc
uint32_t crc; //!< Contains crc of output if crc_flag is set
uint32_t hist_bits; //!< Log base 2 of maximum lookback distance
int32_t type0_block_len; //!< Length left to read of type 0 block when outbuffer overflow occured
union {
int32_t type0_block_len; //!< Length left to read of type 0 block when outbuffer overflow occured
int32_t count; //!< Count of bytes remaining to be parsed
uint32_t dict_id;
};
int32_t write_overflow_lits;
int32_t write_overflow_len;
int32_t copy_overflow_length; //!< Length left to copy when outbuffer overflow occured
int32_t copy_overflow_distance; //!< Lookback distance when outbuffer overlow occured
int32_t tmp_in_size; //!< Number of bytes in tmp_in_buffer
int16_t wrapper_flag;
int16_t tmp_in_size; //!< Number of bytes in tmp_in_buffer
int32_t tmp_out_valid; //!< Number of bytes in tmp_out_buffer
int32_t tmp_out_processed; //!< Number of bytes processed in tmp_out_buffer
uint8_t tmp_in_buffer[ISAL_DEF_MAX_HDR_SIZE]; //!< Temporary buffer containing data from the input stream
@ -715,6 +759,44 @@ void isal_inflate_reset(struct inflate_state *state);
*/
int isal_inflate_set_dict(struct inflate_state *state, uint8_t *dict, uint32_t dict_len);
/**
* @brief Read and return gzip header information
*
* On entry state must be initialized and next_in pointing to a gzip compressed
* buffer. The buffers gz_hdr->extra, gz_hdr->name, gz_hdr->comments and the
* buffer lengths must be set to record the corresponding field, or set to NULL
* to disregard that gzip header information. If one of these buffers overflows,
* the user can reallocate a larger buffer and call this function again to
* continue reading the header information.
*
* @param state: Structure holding state information on the decompression stream.
* @param gz_hdr: Structure to return data encoded in the gzip header
* @returns ISAL_DECOMP_OK (header was successfully parsed)
* ISAL_END_INPUT (all input was parsed),
* ISAL_NAME_OVERFLOW (gz_hdr->name overflowed while parsing),
* ISAL_COMMENT_OVERFLOW (gz_hdr->comment overflowed while parsing),
* ISAL_EXTRA_OVERFLOW (gz_hdr->extra overflowed while parsing),
* ISAL_INVALID_WRAPPER (invalid gzip header found),
* ISAL_UNSUPPORTED_METHOD (deflate is not the compression method),
* ISAL_INCORRECT_CHECKSUM (gzip header checksum was incorrect)
*/
int isal_read_gzip_header (struct inflate_state *state, struct isal_gzip_header *gz_hdr);
/**
* @brief Read and return zlib header information
*
* On entry state must be initialized and next_in pointing to a zlib compressed
* buffer.
*
* @param state: Structure holding state information on the decompression stream.
* @param zlib_hdr: Structure to return data encoded in the zlib header
* @returns ISAL_DECOMP_OK (header was successfully parsed),
* ISAL_END_INPUT (all input was parsed),
* ISAL_UNSUPPORTED_METHOD (deflate is not the compression method),
* ISAL_INCORRECT_CHECKSUM (zlib header checksum was incorrect)
*/
int isal_read_zlib_header (struct inflate_state *state, struct isal_zlib_header *zlib_hdr);
/**
* @brief Fast data (deflate) decompression for storage applications.
*
@ -728,12 +810,18 @@ int isal_inflate_set_dict(struct inflate_state *state, uint8_t *dict, uint32_t d
* The call to isal_inflate() will take data from the input buffer (updating
* next_in, avail_in and write a decompressed stream to the output buffer
* (updating next_out and avail_out). The function returns when the input buffer
* is empty, the output buffer is full or invalid data is found. The current
* state of the decompression on exit can be read from state->block-state. If
* the crc_flag is set to ISAL_GZIP_NO_HDR the gzip crc of the output is stored
* in state->crc. Alternatively, if the crc_flag is set to ISAL_ZLIB_NO_HDR the
* adler32 of the output is stored in state->crc. The element state->hist_bits
* has values from 0 to 15, where values of 1 to 15 are the log base 2 size of the
* is empty, the output buffer is full, invalid data is found, or in the case of
* zlib formatted data if a dictionary is specified. The current state of the
* decompression on exit can be read from state->block-state. If the crc_flag is
* set to ISAL_GZIP_NO_HDR the gzip crc of the output is stored in
* state->crc. Alternatively, if the crc_flag is set to ISAL_ZLIB_NO_HDR the
* adler32 of the output is stored in state->crc. When the crc_flag is set to
* ISAL_GZIP_NO_HDR_VER or ISAL_ZLIB_NO_HDR_VER, the behaviour is the same,
* except the checksum is verified with the checksum after immediately followin
* the deflate data. Finally, if the crc_flag is set to ISAL_GZIP or ISAL_ZLIB,
* the gzip/zlib header is parsed, state->crc is set to the appropriate
* checksum, and the checksum is verfied. The element state->hist_bits has
* values from 0 to 15, where values of 1 to 15 are the log base 2 size of the
* matching window and 0 is the default with maximum history size.
*
* If a dictionary is required, a call to isal_inflate_set_dict will set the
@ -741,11 +829,13 @@ int isal_inflate_set_dict(struct inflate_state *state, uint8_t *dict, uint32_t d
*
* @param state Structure holding state information on the compression streams.
* @return ISAL_DECOMP_OK (if everything is ok),
* ISAL_END_INPUT (if all input was decompressed),
* ISAL_OUT_OVERFLOW (if output buffer ran out of space),
* ISAL_INVALID_BLOCK,
* ISAL_NEED_DICT,
* ISAL_INVALID_SYMBOL,
* ISAL_INVALID_LOOKBACK.
* ISAL_INVALID_LOOKBACK,
* ISAL_INVALID_WRAPPER,
* ISAL_UNSUPPORTED_METHOD,
* ISAL_INCORRECT_CHECKSUM.
*/
int isal_inflate(struct inflate_state *state);
@ -755,15 +845,20 @@ int isal_inflate(struct inflate_state *state);
*
* Stateless (one shot) decompression routine with a similar interface to
* isal_inflate() but operates on entire input buffer at one time. Parameter
* avail_out must be large enough to fit the entire decompressed output.
* avail_out must be large enough to fit the entire decompressed
* output. Dictionaries are not supported.
*
* @param state Structure holding state information on the compression streams.
* @return ISAL_DECOMP_OK (if everything is ok),
* ISAL_END_INPUT (if all input was decompressed),
* ISAL_NEED_DICT,
* ISAL_OUT_OVERFLOW (if output buffer ran out of space),
* ISAL_INVALID_BLOCK,
* ISAL_INVALID_SYMBOL,
* ISAL_INVALID_LOOKBACK.
* ISAL_INVALID_LOOKBACK,
* ISAL_INVALID_WRAPPER,
* ISAL_UNSUPPORTED_METHOD,
* ISAL_INCORRECT_CHECKSUM.
*/
int isal_inflate_stateless(struct inflate_state *state);

2
isa-l.def
View File

@ -105,3 +105,5 @@ isal_deflate_reset @101
isal_inflate_set_dict @102
isal_inflate_reset @103
crc16_t10dif_copy @104
isal_read_gzip_header @105
isal_read_zlib_header @106

4
tests/fuzz/igzip_simple_round_trip_fuzz_test.c
View File

@ -104,8 +104,8 @@ int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
ret = isal_deflate_stateless(&cstate);
isal_inflate_init(&istate);
istate.next_in = isal_cmp_buf + header_size[wrapper_type];
istate.avail_in = cstate.total_out - header_size[wrapper_type];;
istate.next_in = isal_cmp_buf;
istate.avail_in = cstate.total_out;
istate.next_out = isal_out_buf;
istate.avail_out = size;
istate.crc_flag = wrapper_type;