generic-library/isa-l
1
0
Fork 0
mirror of https://github.com/intel/isa-l.git synced 2024-12-12 17:33:50 +01:00
Code Issues Projects Releases Wiki Activity

igzip: Add a hufftable for the static Huffman code in Deflate

Browse Source 
Change-Id: If8aadce05bcb705e608fe3ed6028be9a7448fa59
Signed-off-by: Roy Oursler <roy.j.oursler@intel.com>
This commit is contained in:
Roy Oursler 2016-10-31 20:06:55 -07:00 committed by Greg Tucker
parent e1bcebc93a
commit 7fefb53938
5 changed files with 2345 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
igzip/generate_custom_hufftables.c
View File

@ -204,12 +204,13 @@ void fprint_uint64_table(FILE * outfile, uint64_t * table, uint64_t length, char
}
void fprint_hufftables(FILE * output_file, uint8_t * header, uint32_t bit_count,
uint16_t * lit_code_table, uint8_t * lit_code_size_table,
uint16_t * dcodes_code_table, uint8_t * dcodes_code_size_table,
uint32_t * packed_len_table, uint32_t * packed_dist_table)
void fprint_hufftables(FILE * output_file, char *hufftables_name, uint8_t * header,
uint32_t bit_count, uint16_t * lit_code_table,
uint8_t * lit_code_size_table, uint16_t * dcodes_code_table,
uint8_t * dcodes_code_size_table, uint32_t * packed_len_table,
uint32_t * packed_dist_table)
{
fprintf(output_file, "struct isal_hufftables hufftables_default = {\n\n");
fprintf(output_file, "struct isal_hufftables %s = {\n\n", hufftables_name);
fprint_uint8_table(output_file, header, (bit_count + 7) / 8,
"\t.deflate_hdr = {", "\t},\n\n", "\t\t");
@ -247,10 +248,7 @@ void fprint_hufftables(FILE * output_file, uint8_t * header, uint32_t bit_count,
fprintf(output_file, "};\n");
}
void fprint_header(FILE * output_file, uint8_t * header, uint32_t bit_count,
uint16_t * lit_code_table, uint8_t * lit_code_size_table,
uint16_t * dcodes_code_table, uint8_t * dcodes_code_size_table,
uint32_t * packed_len_table, uint32_t * packed_dist_table)
void fprint_header(FILE * output_file)
{
fprintf(output_file, "#include <stdint.h>\n");
fprintf(output_file, "#include <igzip_lib.h>\n\n");
@ -259,15 +257,12 @@ void fprint_header(FILE * output_file, uint8_t * header, uint32_t bit_count,
"\t0x1f, 0x8b, 0x08, 0x00, 0x00,\n" "\t0x00, 0x00, 0x00, 0x00, 0xff\t};\n\n");
fprintf(output_file, "const uint32_t gzip_hdr_bytes = %d;\n", GZIP_HEADER_SIZE);
fprintf(output_file, "const uint32_t gzip_trl_bytes = %d;\n\n", GZIP_TRAILER_SIZE);
fprint_hufftables(output_file, header, bit_count, lit_code_table, lit_code_size_table,
dcodes_code_table, dcodes_code_size_table, packed_len_table,
packed_dist_table);
fprintf(output_file, "const uint32_t gzip_trl_bytes = %d;\n", GZIP_TRAILER_SIZE);
}
int main(int argc, char *argv[])
{
int i;
long int file_length;
uint8_t *stream = NULL;
struct isal_huff_histogram histogram;
@ -278,6 +273,8 @@ int main(int argc, char *argv[])
struct huff_tree lit_tree, dist_tree;
struct huff_tree lit_tree_array[2 * LIT_LEN - 1], dist_tree_array[2 * DIST_LEN - 1];
struct huff_code lit_huff_table[LIT_LEN], dist_huff_table[DIST_LEN];
uint16_t count_static_lit[16] = { 0, 0, 0, 0, 0, 0, 0, 24, 152, 112, 0, 0, 0, 0, 0 };
uint16_t count_static_dist[16] = { 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
uint64_t bit_count;
uint32_t packed_len_table[LEN_TABLE_SIZE];
uint32_t packed_dist_table[LONG_DIST_TABLE_SIZE];
@ -378,7 +375,6 @@ int main(int argc, char *argv[])
}
}
#ifdef PRINT_CODES
int i;
printf("Lit/Len codes\n");
for (i = 0; i < LIT_TABLE_SIZE - 1; i++)
printf("Lit %3d: Code 0x%04x, Code_Len %d\n", i, lit_huff_table[i].code,
@ -415,9 +411,46 @@ int main(int argc, char *argv[])
return 1;
}
fprint_header(file, header, bit_count, lit_code_table, lit_code_size_table,
dcodes_code_table, dcodes_code_size_table, packed_len_table,
packed_dist_table);
fprint_header(file);
fprintf(file, "\n");
fprint_hufftables(file, "hufftables_default", header, bit_count, lit_code_table,
lit_code_size_table, dcodes_code_table, dcodes_code_size_table,
packed_len_table, packed_dist_table);
fprintf(file, "\n");
/* Generate the static Huffman code defined in RFC 1951 */
memset(lit_huff_table, 0, sizeof(lit_huff_table));
memset(dist_huff_table, 0, sizeof(dist_huff_table));
for (i = 0; i < 144; i++)
lit_huff_table[i].length = 8;
for (i = 144; i < 256; i++)
lit_huff_table[i].length = 9;
for (i = 256; i < 280; i++)
lit_huff_table[i].length = 7;
for (i = 280; i < 286; i++)
lit_huff_table[i].length = 8;
for (i = 0; i < 30; i++)
dist_huff_table[i].length = 5;
set_huff_codes(lit_huff_table, LIT_LEN, count_static_lit);
set_huff_codes(dist_huff_table, DIST_LEN, count_static_dist);
create_code_tables(lit_code_table, lit_code_size_table, LIT_TABLE_SIZE,
lit_huff_table);
create_code_tables(dcodes_code_table, dcodes_code_size_table, DIST_LEN,
dist_huff_table);
create_packed_len_table(packed_len_table, lit_huff_table);
create_packed_dist_table(packed_dist_table, LONG_DIST_TABLE_SIZE, dist_huff_table);
bit_count = 3;
header[0] = 0x03;
fprint_hufftables(file, "hufftables_static", header, bit_count, lit_code_table,
lit_code_size_table, dcodes_code_table, dcodes_code_size_table,
packed_len_table, packed_dist_table);
fclose(file);

2217
igzip/hufftables_c.c
View File

File diff suppressed because it is too large Load Diff

30
igzip/igzip.c
View File

@ -50,6 +50,7 @@ extern const uint8_t gzip_hdr[];
extern const uint32_t gzip_hdr_bytes;
extern const uint32_t gzip_trl_bytes;
extern const struct isal_hufftables hufftables_default;
extern const struct isal_hufftables hufftables_static;
extern uint32_t CrcTable[256];
extern uint32_t crc32_gzip(uint32_t init_crc, const unsigned char *buf, uint64_t len);
@ -96,6 +97,9 @@ struct slver isal_deflate_slver = { 0x0082, 0x03, 0x01 };
struct slver isal_deflate_stateless_slver_01010083;
struct slver isal_deflate_stateless_slver = { 0x0083, 0x01, 0x01 };
struct slver isal_deflate_set_hufftables_slver_00_01_008b;
struct slver isal_deflate_set_hufftables_slver = { 0x008b, 0x01, 0x00 };
/*****************************************************************/
static
void sync_flush(struct isal_zstream *stream)
@ -508,6 +512,31 @@ void isal_deflate_init(struct isal_zstream *stream)
return;
}
int isal_deflate_set_hufftables(struct isal_zstream *stream,
struct isal_hufftables *hufftables, int type)
{
if (stream->internal_state.state != ZSTATE_NEW_HDR)
return ISAL_INVALID_OPERATION;
switch (type) {
case IGZIP_HUFFTABLE_DEFAULT:
stream->hufftables = (struct isal_hufftables *)&hufftables_default;
break;
case IGZIP_HUFFTABLE_STATIC:
stream->hufftables = (struct isal_hufftables *)&hufftables_static;
break;
case IGZIP_HUFFTABLE_CUSTOM:
if (hufftables != NULL) {
stream->hufftables = hufftables;
break;
}
default:
return ISAL_INVALID_OPERATION;
}
return COMP_OK;
}
void isal_deflate_stateless_init(struct isal_zstream *stream)
{
stream->total_in = 0;
@ -516,6 +545,7 @@ void isal_deflate_stateless_init(struct isal_zstream *stream)
stream->flush = NO_FLUSH;
stream->end_of_stream = 0;
stream->gzip_flag = 0;
stream->internal_state.state = ZSTATE_NEW_HDR;
return;
}

35
igzip/igzip_rand_test.c
View File

@ -666,6 +666,11 @@ int isal_deflate_with_checks(struct isal_zstream *stream, uint32_t data_size,
}
void set_random_hufftable(struct isal_zstream *stream)
{
isal_deflate_set_hufftables(stream, hufftables, rand() % 4);
}
/* Compress the input data into the output buffer where the input buffer and
* output buffer are randomly segmented to test state information for the
* compression*/
@ -688,9 +693,6 @@ int compress_multi_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed
isal_deflate_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
@ -770,6 +772,9 @@ int compress_multi_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed
}
}
if (state->state == ZSTATE_NEW_HDR)
set_random_hufftable(&stream);
ret =
isal_deflate_with_checks(&stream, data_size, *compressed_size, in_buf,
in_size, in_processed, out_buf, out_size,
@ -820,8 +825,7 @@ int compress_single_pass(uint8_t * data, uint32_t data_size, uint8_t * compresse
isal_deflate_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
set_random_hufftable(&stream);
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
@ -859,8 +863,7 @@ int compress_stateless(uint8_t * data, uint32_t data_size, uint8_t * compressed_
isal_deflate_stateless_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
set_random_hufftable(&stream);
stream.avail_in = data_size;
stream.next_in = data;
@ -924,9 +927,6 @@ int compress_stateless_full_flush(uint8_t * data, uint32_t data_size, uint8_t *
isal_deflate_stateless_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
stream.flush = FULL_FLUSH;
stream.end_of_stream = 0;
stream.avail_out = *compressed_size;
@ -965,6 +965,9 @@ int compress_stateless_full_flush(uint8_t * data, uint32_t data_size, uint8_t *
out_buf = stream.next_out;
if (stream.internal_state.state == ZSTATE_NEW_HDR)
set_random_hufftable(&stream);
ret = isal_deflate_stateless(&stream);
assert(stream.internal_state.bitbuf.m_bit_count == 0);
@ -1020,9 +1023,6 @@ int compress_full_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed
isal_deflate_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
@ -1069,6 +1069,9 @@ int compress_full_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed
out_buf = stream.next_out;
}
if (state->state == ZSTATE_NEW_HDR)
set_random_hufftable(&stream);
ret = isal_deflate(&stream);
if (ret)
@ -1120,8 +1123,7 @@ int compress_swap_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed
isal_deflate_init(&stream);
if (hufftables != NULL)
stream.hufftables = hufftables;
set_random_hufftable(&stream);
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
@ -1143,6 +1145,9 @@ int compress_swap_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed
if (ret)
return ret;
if (state->state == ZSTATE_NEW_HDR)
set_random_hufftable(&stream);
flush_type = rand() % 3;
stream.flush = flush_type;

27
include/igzip_lib.h
View File

@ -129,6 +129,10 @@ enum { IGZIP_DECODE_OFFSET = 0 };
enum {IGZIP_LEN_TABLE_SIZE = 256};
enum {IGZIP_LIT_TABLE_SIZE = ISAL_DEF_LIT_SYMBOLS};
#define IGZIP_HUFFTABLE_CUSTOM 0
#define IGZIP_HUFFTABLE_DEFAULT 1
#define IGZIP_HUFFTABLE_STATIC 2
/* Flush Flags */
#define NO_FLUSH 0 /* Default */
#define SYNC_FLUSH 1
@ -145,6 +149,7 @@ enum {IGZIP_LIT_TABLE_SIZE = ISAL_DEF_LIT_SYMBOLS};
#define INVALID_FLUSH -7
#define INVALID_PARAM -8
#define STATELESS_OVERFLOW -1
#define ISAL_INVALID_OPERATION -9
/**
* @enum isal_zstate
@ -419,6 +424,28 @@ int isal_create_hufftables_subset(struct isal_hufftables * hufftables,
*/
void isal_deflate_init(struct isal_zstream *stream);
/**
* @brief Set stream to use a new Huffman code
*
* Sets the Huffman code to be used in compression before compression start or
* after the sucessful completion of a SYNC_FLUSH or FULL_FLUSH. If type has
* value IGZIP_HUFFTABLE_DEFAULT, the stream is set to use the default Huffman
* code. If type has value IGZIP_HUFFTABLE_STATIC, the stream is set to use the
* deflate standard static Huffman code, or if type has value
* IGZIP_HUFFTABLE_CUSTOM, the stream is set to sue the isal_hufftables
* structure input to isal_deflate_set_hufftables.
*
* @param stream: Structure holding state information on the compression stream.
* @param hufftables: new huffman code to use if type is set to
* IGZIP_HUFFTABLE_CUSTOM.
* @param type: Flag specifying what hufftable to use.
*
* @returns Returns INVALID_OPERATION if the stream was unmodified. This may be
* due to the stream being in a state where changing the huffman code is not
* allowed or an invalid input is provided.
*/
int isal_deflate_set_hufftables(struct isal_zstream *stream,
struct isal_hufftables *hufftables, int type);
/**
* @brief Initialize compression stream data structure