Problem: fuzzer corpus file can be quite large

Solution: move all corpora to the zeromq/libzmq-fuzz-corpora repository
for easier handling, and to avoid inflating the size of zeromq/libzmq.
Clone it for the CI on the fly.
This commit is contained in:
Luca Boccassi 2020-07-04 14:53:03 +01:00
parent 61c7ae0610
commit e95e57d385
19 changed files with 94 additions and 138 deletions

View File

@ -1187,47 +1187,6 @@ endif
FUZZINGdir = ${prefix}/${FUZZING_INSTALLDIR}
FUZZING_PROGRAMS = ${fuzzer_apps}
FUZZING_DATA = tests/fuzzer_corpora/endpoint.dict tests/fuzzer_corpora/zmtp.dict \
tests/fuzzer_corpora/test_bind_curve_fuzzer.txt tests/fuzzer_corpora/test_bind_null_fuzzer.txt \
tests/fuzzer_corpora/test_connect_curve_fuzzer.txt tests/fuzzer_corpora/test_connect_null_fuzzer.txt
install-data-hook:
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/endpoint.dict $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.dict
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/endpoint.dict $(DESTDIR)/$(FUZZINGdir)/test_connect_fuzzer.dict
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/zmtp.dict $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer.dict
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/zmtp.dict $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer.dict
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/zmtp.dict $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer.dict
$(LN_S) -r -f $(DESTDIR)/$(FUZZINGdir)/zmtp.dict $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer.dict
$(shell while read -r test; do \
echo -n $$test | perl -e 'print pack "H*", <STDIN>' > $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer.seed; \
export fn=$$(cat $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer.seed | sha1sum | awk '{print $$1}'); \
mv $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer.seed $(DESTDIR)/$(FUZZINGdir)/$$fn; \
zip -j -m -g --quiet $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer_seed_corpus.zip $(DESTDIR)/$(FUZZINGdir)/$$fn; \
done < $(DESTDIR)/$(FUZZINGdir)/test_bind_curve_fuzzer.txt)
$(shell while read -r test; do \
echo -n $$test | perl -e 'print pack "H*", <STDIN>' > $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer.seed; \
export fn=$$(cat $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer.seed | sha1sum | awk '{print $$1}'); \
mv $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer.seed $(DESTDIR)/$(FUZZINGdir)/$$fn; \
zip -j -m -g --quiet $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer_seed_corpus.zip $(DESTDIR)/$(FUZZINGdir)/$$fn; \
done < $(DESTDIR)/$(FUZZINGdir)/test_bind_null_fuzzer.txt)
$(shell while read -r test; do \
echo -n $$test | perl -e 'print pack "H*", <STDIN>' > $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer.seed; \
export fn=$$(cat $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer.seed | sha1sum | awk '{print $$1}'); \
mv $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer.seed $(DESTDIR)/$(FUZZINGdir)/$$fn; \
zip -j -m -g --quiet $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer_seed_corpus.zip $(DESTDIR)/$(FUZZINGdir)/$$fn; \
done < $(DESTDIR)/$(FUZZINGdir)/test_connect_curve_fuzzer.txt)
$(shell while read -r test; do \
echo -n $$test | perl -e 'print pack "H*", <STDIN>' > $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer.seed; \
export fn=$$(cat $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer.seed | sha1sum | awk '{print $$1}'); \
mv $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer.seed $(DESTDIR)/$(FUZZINGdir)/$$fn; \
zip -j -m -g --quiet $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer_seed_corpus.zip $(DESTDIR)/$(FUZZINGdir)/$$fn; \
done < $(DESTDIR)/$(FUZZINGdir)/test_connect_null_fuzzer.txt)
$(shell while read -r test; do \
echo -n $$test | perl -e 'print pack "H*", <STDIN>' > $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.seed; \
export fn=$$(cat $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.seed | sha1sum | awk '{print $$1}'); \
mv $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.seed $(DESTDIR)/$(FUZZINGdir)/$$fn; \
zip -j -m -g --quiet $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.zip $(DESTDIR)/$(FUZZINGdir)/$$fn; \
done < $(DESTDIR)/$(FUZZINGdir)/test_bind_fuzzer.txt)
rm -f $(DESTDIR)/$(FUZZINGdir)/*.txt
else
test_apps += tests/test_bind_null_fuzzer \
tests/test_connect_null_fuzzer \
@ -1378,9 +1337,6 @@ EXTRA_DIST = \
src/version.rc.in \
tests/CMakeLists.txt \
tests/test_pair_tcp_cap_net_admin.cpp \
tests/fuzzer_corpora/endpoint.dict tests/fuzzer_corpora/zmtp.dict \
tests/fuzzer_corpora/test_bind_curve_fuzzer.txt tests/fuzzer_corpora/test_bind_null_fuzzer.txt \
tests/fuzzer_corpora/test_connect_curve_fuzzer.txt tests/fuzzer_corpora/test_connect_null_fuzzer.txt \
unittests/CMakeLists.txt \
tools/curve_keygen.cpp

View File

@ -16,4 +16,10 @@ export CXXFLAGS+=" $(PKG_CONFIG_PATH=/tmp/zmq_install_dir/install_prefix/lib/pkg
./configure --disable-shared --prefix=/install_prefix --disable-perf --disable-curve-keygen PKG_CONFIG_PATH=/tmp/zmq_install_dir/install_prefix/lib/pkgconfig --with-libsodium=yes --with-fuzzing-installdir=fuzzers --with-fuzzing-engine=$LIB_FUZZING_ENGINE
make -j$(nproc) V=1 install DESTDIR=/tmp/zmq_install_dir
cd "${SRC}/libzmq-fuzz-corpora"
cp dictionaries/* /tmp/zmq_install_dir/install_prefix/fuzzers/
for t in test_*_seed_corpus; do
zip -j --quiet /tmp/zmq_install_dir/install_prefix/fuzzers/${t}.zip ${t}/*
done
cp /tmp/zmq_install_dir/install_prefix/fuzzers/* "${OUT}"

View File

@ -20,6 +20,7 @@ if [ $BUILD_TYPE = "default" ]; then
CONFIG_OPTS+=("--enable-address-sanitizer=yes")
# distcheck does an out-of-tree build, and the fuzzer tests use a hard-coded relative path for simplicity
CHECK="check"
git clone --depth 1 https://github.com/zeromq/libzmq-fuzz-corpora.git tests/libzmq-fuzz-corpora
fi
if [ "$USE_NSS" = "yes" ]; then

View File

@ -1,4 +0,0 @@
ipc="ipc://"
inproc="inproc://"
tcp="tcp://"
udp="udp://"

View File

@ -1 +0,0 @@
ff00000000000000017f03014355525645000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004c80548454c4c4f010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000029ce356a87e7861b3ac340f8a76785982e7684627cc279564b05548b3425a66c00000000000000012ae574af7907bb06a481251c90df3d15f995a53ce358f904c834217a0d7cb021fbbd630f847cd261e1dcf9786f995fde297792bd0b521c827e41e841ae8d643e7757f98a6d8959ca7258cf1a09b6a10306000000000000012408494e49544941544573a40c9d42d89d69e7bee1124a13b8ad85749e2300e1e10f5c833529360f964a82761a463c1a85332e5fadd161d2ba391952c1d875a159d326786330b462fda30a4a6f4d28817a5726950a185a3cd9abf122c99b95fbb3b77b77702e7b57e7b600000000000000029d559274e0c937e8ecf25725b81caae149205ec686718e97f48b3788588280de18405bbd139f1a530b87f1ab4f139ef99ba711d45c55aae6001204fa986f342bac01f90b28fd69a3d9f3fdf5aaff5b7934a0a1f1a70c9fce70720e7f6c25773882d4f1a85460a79cf8da6972efb1571b48fe0d03d09c522523f32796eb635c44210a6bcedd8c3ca66c7bf62016ee08d80ccd4d64a7f2fdec8eb06fe3dfc91a9fc31c4a8c559b4a245de2ff344ab27da7725bd90041074d4553534147450000000000000003d5ce709afb2455b7402a18f01eb61cb7a65d227bb6de4c9714c2a027ba9f2481e43ed60ecbb615fa00b948fa34469f553f0041074d4553534147450000000000000004f08579612e848d6ad69131eba369cdc2634967eb5d223481cc6838e6c355a33edc948e4b6d9ca2b2796c458327a2872de2

View File

@ -1 +0,0 @@
77733a2f2f253a39

View File

@ -1,2 +0,0 @@
ff00000000000000017f03014e554c4c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004190552454144590b536f636b65742d5479706500000003535542040a09535542534352494245
ff000000200000000000008585858585000000000000004700280000000000000000002100006d000028000000000000000000000000000004000019

View File

@ -1 +0,0 @@
ff00000000000000067f03014355525645000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004a80757454c434f4d4574535c9301c7695f2db73109ce0c78ab4ea9012648001fb41f846af73271f30f92d967c988286958f93f795ba211178500a0ea3b8396f5a580868d1fb26f8d4cfac3473c37da8cb825864c61f96c805140e9885e349652bf4cf9b0623e4c6a6de41e2f3db58bed2d152dac0ba6a8c83a222aac67bea733e97815e9c4ebc3ff5968869a42116a9530ee6eba37820c87f382e7ec057c38dc2c337a8dd9b0bc9bcd044605524541445900000000000000012bf9ffd6aadcc15e02e02a9b6e63490b004cb446038b8d1d6e224f507c25c4be0dfd9f30c41a24bff8d277dcad0b83daf292c32bd7792e610041074d45535341474500000000000000028c302f949ffbd09aaf559bd228aeeda6c69c677910edc29642976632bec514afa1a2df7eb5abcfd91f94d9b9e4a61097910041074d4553534147450000000000000003300b7856f1d04647531b04fd46322d34a6ad1f6cf05cc5dfd032cc3ec1c4870a3fdfcbf0e8d6d8a7bbc64c1304c37b5854

View File

@ -1 +0,0 @@
ff00000000000000017f03014e554c4c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000041a0552454144590b536f636b65742d547970650000000458505542000568656c6c6f0005776f726c64

View File

@ -1 +0,0 @@
46555a5a2d54414746555a5ad6514147ec

View File

@ -1,45 +0,0 @@
# ZMTP magic binary streams
# 1.0 https://rfc.zeromq.org/spec/13/
# 2.0 https://rfc.zeromq.org/spec/15/
# 3.1 https://rfc.zeromq.org/spec/37/
# curve https://rfc.zeromq.org/spec/26/
signature="\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x7F"
signature_v1="\x01\x00"
version_v2="\x01"
version_v3="\x03\x00"
version_v3_1="\x03\x01"
mechanism_null="\x4E\x55\x4C\x4C\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
mechanism_curve="\x43\x55\x52\x56\x45\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
short_command="\x04"
long_command="\x06"
ready="\x05\x52\x45\x41\x44\x59"
error="\x05\x45\x52\x52\x4F\x52"
identity="\x08\x49\x64\x65\x6E\x74\x69\x74\x79"
socket_type="\x0B\x53\x6F\x63\x6B\x65\x74\x2D\x54\x79\x70\x65"
dealer="\x06\x44\x45\x41\x4C\x45\x52"
router="\x06\x52\x4F\x55\x54\x45\x52"
pub="\x03\x50\x55\x42"
sub="\x03\x53\x55\x42"
xpub="\x04\x58\x50\x55\x42"
xsub="\x04\x58\x53\x55\x42"
req="\x03\x52\x45\x51"
rep="\x03\x52\x45\x50"
push="\x04\x50\x55\x53\x48"
pull="\x04\x50\x55\x4C\x4C"
pair="\x04\x50\x41\x49\x52"
client="\x05\x43\x4C\x49\x45\x4E\x54"
server="\x05\x53\x45\x52\x56\x45\x52"
radio="\x05\x52\x41\x44\x49\x4F"
dish="\x04\x44\x49\x53\x48"
scatter="\x06\x53\x43\x41\x54\x54\x45\x52"
gather="\x06\x47\x41\x54\x48\x45\x52"
subscribe="\x09\x53\x55\x42\x53\x43\x52\x49\x42\x45"
cancel="\x06\x43\x41\x4E\x43\x45\x4C"
join="\x04\x4A\x4F\x49\x4E"
leave="\x05\x43\x41\x4E\x43\x45\x4C"
ping="\x04\x50\x49\x4E\x47"
pong="\x04\x50\x4F\x4E\x47"
hello="\x05\x48\x45\x4C\x4C\x4F\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
welcome="\x07\x57\x45\x4C\x43\x4F\x4D\x45"
initiate="\x08\x49\x4E\x49\x54\x49\x41\x54\x45"
message="\x07\x4D\x45\x53\x53\x41\x47\x45"

View File

@ -122,8 +122,9 @@ void test_bind_curve_fuzzer ()
{
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode ("tests/fuzzer_corpora/test_bind_curve_fuzzer.txt",
&data, &len, &num_cases)
if (fuzzer_corpus_encode (
"tests/libzmq-fuzz-corpora/test_bind_curve_fuzzer_seed_corpus", &data,
&len, &num_cases)
!= 0)
exit (77);

View File

@ -68,8 +68,9 @@ void test_bind_fuzzer ()
{
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode ("tests/fuzzer_corpora/test_bind_fuzzer.txt",
&data, &len, &num_cases)
if (fuzzer_corpus_encode (
"tests/libzmq-fuzz-corpora/test_bind_fuzzer_seed_corpus", &data, &len,
&num_cases)
!= 0)
exit (77);

View File

@ -85,8 +85,9 @@ void test_bind_null_fuzzer ()
{
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode ("tests/fuzzer_corpora/test_bind_null_fuzzer.txt",
&data, &len, &num_cases)
if (fuzzer_corpus_encode (
"tests/libzmq-fuzz-corpora/test_bind_null_fuzzer_seed_corpus", &data,
&len, &num_cases)
!= 0)
exit (77);

View File

@ -112,8 +112,8 @@ void test_connect_curve_fuzzer ()
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode (
"tests/fuzzer_corpora/test_connect_curve_fuzzer.txt", &data, &len,
&num_cases)
"tests/libzmq-fuzz-corpora/test_connect_curve_fuzzer_seed_corpus",
&data, &len, &num_cases)
!= 0)
exit (77);

View File

@ -53,10 +53,22 @@ extern "C" int LLVMFuzzerTestOneInput (const uint8_t *data, size_t size)
#ifndef ZMQ_USE_FUZZING_ENGINE
void test_connect_fuzzer ()
{
uint8_t buffer[32] = {0};
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode (
"tests/libzmq-fuzz-corpora/test_connect_fuzzer_seed_corpus", &data,
&len, &num_cases)
!= 0)
exit (77);
TEST_ASSERT_SUCCESS_ERRNO (
LLVMFuzzerTestOneInput (buffer, sizeof (buffer)));
while (num_cases-- > 0) {
TEST_ASSERT_SUCCESS_ERRNO (
LLVMFuzzerTestOneInput (data[num_cases], len[num_cases]));
free (data[num_cases]);
}
free (data);
free (len);
}
int main (int argc, char **argv)

View File

@ -91,8 +91,8 @@ void test_connect_null_fuzzer ()
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode (
"tests/fuzzer_corpora/test_connect_null_fuzzer.txt", &data, &len,
&num_cases)
"tests/libzmq-fuzz-corpora/test_connect_null_fuzzer_seed_corpus",
&data, &len, &num_cases)
!= 0)
exit (77);

View File

@ -62,8 +62,9 @@ void test_z85_decode_fuzzer ()
{
uint8_t **data;
size_t *len, num_cases = 0;
if (fuzzer_corpus_encode ("tests/fuzzer_corpora/test_z85_decode_fuzzer.txt",
&data, &len, &num_cases)
if (fuzzer_corpus_encode (
"tests/libzmq-fuzz-corpora/test_z85_decode_fuzzer_seed_corpus", &data,
&len, &num_cases)
!= 0)
exit (77);

View File

@ -60,6 +60,7 @@
#include <net/if.h>
#include <netdb.h>
#include <sys/un.h>
#include <dirent.h>
#if defined(ZMQ_HAVE_AIX)
#include <sys/types.h>
#include <sys/socketvar.h>
@ -519,50 +520,82 @@ bool strneq (const char *lhs_, const char *rhs_)
return strcmp (lhs_, rhs_) != 0;
}
int fuzzer_corpus_encode (const char *filename,
#if defined _WIN32
int fuzzer_corpus_encode (const char *dirname,
uint8_t ***data,
size_t **len,
size_t *num_cases)
{
TEST_ASSERT_NOT_NULL (filename);
(void) dirname;
(void) data;
(void) len;
(void) num_cases;
return -1;
}
#else
int fuzzer_corpus_encode (const char *dirname,
uint8_t ***data,
size_t **len,
size_t *num_cases)
{
TEST_ASSERT_NOT_NULL (dirname);
TEST_ASSERT_NOT_NULL (data);
TEST_ASSERT_NOT_NULL (len);
FILE *f = fopen (filename, "r");
if (!f)
struct dirent *ent;
DIR *dir = opendir (dirname);
if (!dir)
return -1;
fseek (f, 0, SEEK_END);
size_t text_len = ftell (f) + 1;
fseek (f, 0, SEEK_SET);
char *buf = (char *) malloc (text_len);
TEST_ASSERT_NOT_NULL (buf);
*len = NULL;
*data = NULL;
*num_cases = 0;
// Convert to binary format, corpus is stored in ascii (hex)
while (fgets (buf, (int) text_len, f)) {
while ((ent = readdir (dir)) != NULL) {
if (!strcmp (ent->d_name, ".") || !strcmp (ent->d_name, ".."))
continue;
char *filename =
(char *) malloc (strlen (dirname) + strlen (ent->d_name) + 2);
TEST_ASSERT_NOT_NULL (filename);
strcpy (filename, dirname);
strcat (filename, "/");
strcat (filename, ent->d_name);
FILE *f = fopen (filename, "r");
free (filename);
if (!f)
continue;
fseek (f, 0, SEEK_END);
size_t file_len = ftell (f);
fseek (f, 0, SEEK_SET);
if (file_len == 0) {
fclose (f);
continue;
}
*len = (size_t *) realloc (*len, (*num_cases + 1) * sizeof (size_t));
TEST_ASSERT_NOT_NULL (*len);
*(*len + *num_cases) = strlen (buf) / 2;
*(*len + *num_cases) = file_len;
*data =
(uint8_t **) realloc (*data, (*num_cases + 1) * sizeof (uint8_t *));
TEST_ASSERT_NOT_NULL (*data);
*(*data + *num_cases) =
(uint8_t *) malloc (*(*len + *num_cases) * sizeof (uint8_t));
(uint8_t *) malloc (file_len * sizeof (uint8_t));
TEST_ASSERT_NOT_NULL (*(*data + *num_cases));
const char *pos = buf;
for (size_t count = 0; count < *(*len + *num_cases);
++count, pos += 2) {
char tmp[3] = {pos[0], pos[1], 0};
*(*(*data + *num_cases) + count) = (uint8_t) strtol (tmp, NULL, 16);
}
size_t read_bytes = 0;
read_bytes = fread (*(*data + *num_cases), 1, file_len, f);
TEST_ASSERT_EQUAL (file_len, read_bytes);
(*num_cases)++;
fclose (f);
}
free (buf);
fclose (f);
closedir (dir);
return 0;
}
#endif