Merge branch 'master' of ssh://github.com/msgpack/msgpack

cpp/Makefile.am

@@ -11,3 +11,9 @@ DOC_FILES = \
 EXTRA_DIST = \
 		$(DOC_FILES)
 
+doxygen:
+	./preprocess
+	./preprocess clean
+	cd src && $(MAKE) doxygen
+	./preprocess
+

cpp/preprocess

@@ -6,6 +6,7 @@ preprocess() {
 		echo ""
 		echo "** preprocess failed **"
 		echo ""
+		exit 1
 	else
 		mv $1.tmp $1
 	fi
@@ -25,4 +26,6 @@ cp -f ../msgpack/pack_define.h     src/msgpack/
 cp -f ../msgpack/pack_template.h   src/msgpack/
 cp -f ../msgpack/unpack_define.h   src/msgpack/
 cp -f ../msgpack/unpack_template.h src/msgpack/
+cp -f ../test/cases.mpac           test/
+cp -f ../test/cases_compact.mpac   test/
 

cpp/src/Makefile.am

@@ -73,3 +73,20 @@ EXTRA_DIST = \
 		msgpack/type/define.hpp.erb \
 		msgpack/type/tuple.hpp.erb
 
+
+doxygen_c:
+	cat ../Doxyfile > Doxyfile_c
+	echo "FILE_PATTERNS      = *.h" >> Doxyfile_c
+	echo "OUTPUT_DIRECTORY   = doc_c" >> Doxyfile_c
+	echo "PROJECT_NAME       = \"MessagePack for C\"" >> Doxyfile_c
+	doxygen Doxyfile_c
+
+doxygen_cpp:
+	cat ../Doxyfile > Doxyfile_cpp
+	echo "FILE_PATTERNS      = *.hpp" >> Doxyfile_cpp
+	echo "OUTPUT_DIRECTORY   = doc_cpp" >> Doxyfile_cpp
+	echo "PROJECT_NAME       = \"MessagePack for C++\"" >> Doxyfile_cpp
+	doxygen Doxyfile_cpp
+
+doxygen: doxygen_c doxygen_cpp
+

cpp/src/msgpack.h

@@ -15,6 +15,11 @@
  *    See the License for the specific language governing permissions and
  *    limitations under the License.
  */
+/**
+ * @defgroup msgpack MessagePack C
+ * @{
+ * @}
+ */
 #include "msgpack/object.h"
 #include "msgpack/zone.h"
 #include "msgpack/pack.h"

cpp/src/msgpack/object.h

@@ -26,6 +26,12 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_object Dynamically typed object
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef enum {
 	MSGPACK_OBJECT_NIL					= 0x00,
 	MSGPACK_OBJECT_BOOLEAN				= 0x01,
@@ -81,6 +87,8 @@ void msgpack_object_print(FILE* out, msgpack_object o);
 
 bool msgpack_object_equal(const msgpack_object x, const msgpack_object y);
 
+/** @} */
+
 
 #ifdef __cplusplus
 }

cpp/src/msgpack/pack.h

@@ -27,6 +27,19 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_buffer Buffers
+ * @ingroup msgpack
+ * @{
+ * @}
+ */
+
+/**
+ * @defgroup msgpack_pack Serializer
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len);
 
 typedef struct msgpack_packer {
@@ -74,6 +87,8 @@ static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
 
 
+/** @} */
+
 
 #define msgpack_pack_inline_func(name) \
 	inline int msgpack_pack ## name

cpp/src/msgpack/sbuffer.h

@@ -21,15 +21,17 @@
 #include <stdlib.h>
 #include <string.h>
 
-#ifndef MSGPACK_SBUFFER_INIT_SIZE
-#define MSGPACK_SBUFFER_INIT_SIZE 8192
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_sbuffer Simple buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 typedef struct msgpack_sbuffer {
 	size_t size;
 	char* data;
@@ -46,6 +48,22 @@ static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf)
 	free(sbuf->data);
 }
 
+static inline msgpack_sbuffer* msgpack_sbuffer_new(void)
+{
+	return (msgpack_sbuffer*)calloc(1, sizeof(msgpack_sbuffer));
+}
+
+static inline void msgpack_sbuffer_free(msgpack_sbuffer* sbuf)
+{
+	if(sbuf == NULL) { return; }
+	msgpack_sbuffer_destroy(sbuf);
+	free(sbuf);
+}
+
+#ifndef MSGPACK_SBUFFER_INIT_SIZE
+#define MSGPACK_SBUFFER_INIT_SIZE 8192
+#endif
+
 static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len)
 {
 	msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data;
@@ -82,6 +100,9 @@ static inline void msgpack_sbuffer_clear(msgpack_sbuffer* sbuf)
 	sbuf->size = 0;
 }
 
+/** @} */
+
+
 #ifdef __cplusplus
 }
 #endif

cpp/src/msgpack/unpack.h

@@ -20,12 +20,36 @@
 
 #include "msgpack/zone.h"
 #include "msgpack/object.h"
+#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_unpack Deserializer
+ * @ingroup msgpack
+ * @{
+ */
+
+typedef struct msgpack_unpacked {
+	msgpack_zone* zone;
+	msgpack_object data;
+} msgpack_unpacked;
+
+bool msgpack_unpack_next(msgpack_unpacked* result,
+		const char* data, size_t len, size_t* off);
+
+/** @} */
+
+
+/**
+ * @defgroup msgpack_unpacker Streaming deserializer
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef struct msgpack_unpacker {
 	char* buffer;
 	size_t used;
@@ -38,18 +62,102 @@ typedef struct msgpack_unpacker {
 } msgpack_unpacker;
 
 
+#ifndef MSGPACK_UNPACKER_INIT_BUFFER_SIZE
+#define MSGPACK_UNPACKER_INIT_BUFFER_SIZE (64*1024)
+#endif
+
+/**
+ * Initializes a streaming deserializer.
+ * The initialized deserializer must be destroyed by msgpack_unpacker_destroy(msgpack_unpacker*).
+ */
 bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size);
+
+/**
+ * Destroys a streaming deserializer initialized by msgpack_unpacker_init(msgpack_unpacker*, size_t).
+ */
 void msgpack_unpacker_destroy(msgpack_unpacker* mpac);
 
+
+/**
+ * Creates a streaming deserializer.
+ * The created deserializer must be destroyed by msgpack_unpacker_free(msgpack_unpacker*).
+ */
 msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size);
+
+/**
+ * Frees a streaming deserializer created by msgpack_unpacker_new(size_t).
+ */
 void msgpack_unpacker_free(msgpack_unpacker* mpac);
 
+
+#ifndef MSGPACK_UNPACKER_RESERVE_SIZE
+#define MSGPACK_UNPACKER_RESERVE_SIZE (32*1024)
+#endif
+
+/**
+ * Reserves free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_buffer(msgpack_unpacker*),
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline bool   msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size);
+
+/**
+ * Gets pointer to the free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline char*  msgpack_unpacker_buffer(msgpack_unpacker* mpac);
+
+/**
+ * Gets size of the free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac);
+
+/**
+ * Notifies the deserializer that the internal buffer filled.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer(msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*).
+ */
 static inline void   msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size);
 
 
+/**
+ * Deserializes one object.
+ * Returns true if it successes. Otherwise false is returned.
+ * @param pac  pointer to an initialized msgpack_unpacked object.
+ */
+bool msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* pac);
+
+/**
+ * Initializes a msgpack_unpacked object.
+ * The initialized object must be destroyed by msgpack_unpacked_destroy(msgpack_unpacker*).
+ * Use the object with msgpack_unpacker_next(msgpack_unpacker*, msgpack_unpacked*) or
+ * msgpack_unpack_next(msgpack_unpacked*, const char*, size_t, size_t*).
+ */
+static inline void msgpack_unpacked_init(msgpack_unpacked* result);
+
+/**
+ * Destroys a streaming deserializer initialized by msgpack_unpacked().
+ */
+static inline void msgpack_unpacked_destroy(msgpack_unpacked* result);
+
+/**
+ * Releases the memory zone from msgpack_unpacked object.
+ * The released zone must be freed by msgpack_zone_free(msgpack_zone*).
+ */
+static inline msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* result);
+
+
 int msgpack_unpacker_execute(msgpack_unpacker* mpac);
 
 msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac);
@@ -63,7 +171,10 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac);
 static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac);
 
 
+/** @} */
 
+
+// obsolete
 typedef enum {
 	MSGPACK_UNPACK_SUCCESS				=  2,
 	MSGPACK_UNPACK_EXTRA_BYTES			=  1,
@@ -71,9 +182,10 @@ typedef enum {
 	MSGPACK_UNPACK_PARSE_ERROR			= -1,
 } msgpack_unpack_return;
 
+// obsolete
 msgpack_unpack_return
 msgpack_unpack(const char* data, size_t len, size_t* off,
-		msgpack_zone* z, msgpack_object* result);
+		msgpack_zone* result_zone, msgpack_object* result);
 
 
 static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac);
@@ -115,6 +227,31 @@ size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac)
 }
 
 
+void msgpack_unpacked_init(msgpack_unpacked* result)
+{
+	memset(result, 0, sizeof(msgpack_unpacked));
+}
+
+void msgpack_unpacked_destroy(msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone_free(result->zone);
+		result->zone = NULL;
+		memset(&result->data, 0, sizeof(msgpack_object));
+	}
+}
+
+msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone* z = result->zone;
+		result->zone = NULL;
+		return z;
+	}
+	return NULL;
+}
+
+
 #ifdef __cplusplus
 }
 #endif

cpp/src/msgpack/unpack.hpp

@@ -24,8 +24,9 @@
 #include <memory>
 #include <stdexcept>
 
+// backward compatibility
 #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE
-#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE (32*1024)
+#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE MSGPACK_UNPACKER_INIT_BUFFER_SIZE
 #endif
 
 namespace msgpack {
@@ -64,12 +65,12 @@ private:
 
 class unpacker : public msgpack_unpacker {
 public:
-	unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE);
+	unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_INIT_BUFFER_SIZE);
 	~unpacker();
 
 public:
 	/*! 1. reserve buffer. at least `size' bytes of capacity will be ready */
-	void reserve_buffer(size_t size);
+	void reserve_buffer(size_t size = MSGPACK_UNPACKER_RESERVE_SIZE);
 
 	/*! 2. read data to the buffer() up to buffer_capacity() bytes */
 	char* buffer();
@@ -160,7 +161,7 @@ private:
 };
 
 
-static bool unpack(unpacked* result,
+static void unpack(unpacked* result,
 		const char* data, size_t len, size_t* offset = NULL);
 
 
@@ -312,7 +313,7 @@ inline void unpacker::remove_nonparsed_buffer()
 }
 
 
-inline bool unpack(unpacked* result,
+inline void unpack(unpacked* result,
 		const char* data, size_t len, size_t* offset)
 {
 	msgpack::object obj;
@@ -326,12 +327,12 @@ inline bool unpack(unpacked* result,
 	case UNPACK_SUCCESS:
 		result->get() = obj;
 		result->zone() = z;
-		return false;
+		return;
 
 	case UNPACK_EXTRA_BYTES:
 		result->get() = obj;
 		result->zone() = z;
-		return true;
+		return;
 
 	case UNPACK_CONTINUE:
 		throw unpack_error("insufficient bytes");

cpp/src/msgpack/vrefbuffer.h

@@ -19,6 +19,7 @@
 #define MSGPACK_VREFBUFFER_H__
 
 #include "msgpack/zone.h"
+#include <stdlib.h>
 
 #ifndef _WIN32
 #include <sys/uio.h>
@@ -29,19 +30,17 @@ struct iovec {
 };
 #endif
 
-#ifndef MSGPACK_VREFBUFFER_REF_SIZE
-#define MSGPACK_VREFBUFFER_REF_SIZE 32
-#endif
-
-#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
-#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_vrefbuffer Vectored Referencing buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 struct msgpack_vrefbuffer_chunk;
 typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk;
 
@@ -63,10 +62,21 @@ typedef struct msgpack_vrefbuffer {
 } msgpack_vrefbuffer;
 
 
+#ifndef MSGPACK_VREFBUFFER_REF_SIZE
+#define MSGPACK_VREFBUFFER_REF_SIZE 32
+#endif
+
+#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
+#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192
+#endif
+
 bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
 		size_t ref_size, size_t chunk_size);
 void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
 
+static inline msgpack_vrefbuffer* msgpack_vrefbuffer_new(size_t ref_size, size_t chunk_size);
+static inline void msgpack_vrefbuffer_free(msgpack_vrefbuffer* vbuf);
+
 static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len);
 
 static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref);
@@ -82,6 +92,25 @@ int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to)
 
 void msgpack_vrefbuffer_clear(msgpack_vrefbuffer* vref);
 
+/** @} */
+
+
+msgpack_vrefbuffer* msgpack_vrefbuffer_new(size_t ref_size, size_t chunk_size)
+{
+	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)malloc(sizeof(msgpack_vrefbuffer));
+	if(!msgpack_vrefbuffer_init(vbuf, ref_size, chunk_size)) {
+		free(vbuf);
+		return NULL;
+	}
+	return vbuf;
+}
+
+void msgpack_vrefbuffer_free(msgpack_vrefbuffer* vbuf)
+{
+	if(vbuf == NULL) { return; }
+	msgpack_vrefbuffer_destroy(vbuf);
+	free(vbuf);
+}
 
 int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len)
 {

cpp/src/msgpack/zbuffer.h

@@ -23,30 +23,34 @@
 #include <string.h>
 #include <zlib.h>
 
-#ifndef MSGPACK_ZBUFFER_INIT_SIZE
-#define MSGPACK_ZBUFFER_INIT_SIZE 8192
-#endif
-
-#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
-#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_zbuffer Compressed buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 typedef struct msgpack_zbuffer {
 	z_stream stream;
 	char* data;
 	size_t init_size;
 } msgpack_zbuffer;
 
+#ifndef MSGPACK_ZBUFFER_INIT_SIZE
+#define MSGPACK_ZBUFFER_INIT_SIZE 8192
+#endif
 
 static inline bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf,
 		int level, size_t init_size);
 static inline void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf);
 
+static inline msgpack_zbuffer* msgpack_zbuffer_new(int level, size_t init_size);
+static inline void msgpack_zbuffer_free(msgpack_zbuffer* zbuf);
+
 static inline char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf);
 
 static inline const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf);
@@ -57,6 +61,10 @@ static inline void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf);
 static inline char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf);
 
 
+#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
+#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
+#endif
+
 static inline int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len);
 
 static inline bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf);
@@ -80,6 +88,23 @@ void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf)
 	free(zbuf->data);
 }
 
+msgpack_zbuffer* msgpack_zbuffer_new(int level, size_t init_size)
+{
+	msgpack_zbuffer* zbuf = (msgpack_zbuffer*)malloc(sizeof(msgpack_zbuffer));
+	if(!msgpack_zbuffer_init(zbuf, level, init_size)) {
+		free(zbuf);
+		return NULL;
+	}
+	return zbuf;
+}
+
+void msgpack_zbuffer_free(msgpack_zbuffer* zbuf)
+{
+	if(zbuf == NULL) { return; }
+	msgpack_zbuffer_destroy(zbuf);
+	free(zbuf);
+}
+
 bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf)
 {
 	size_t used = (char*)zbuf->stream.next_out - zbuf->data;
@@ -171,6 +196,8 @@ char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf)
 	return tmp;
 }
 
+/** @} */
+
 
 #ifdef __cplusplus
 }

cpp/src/msgpack/zone.h

@@ -25,6 +25,12 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_zone Memory zone
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef struct msgpack_zone_finalizer {
 	void (*func)(void* data);
 	void* data;
@@ -71,6 +77,7 @@ bool msgpack_zone_is_empty(msgpack_zone* zone);
 
 void msgpack_zone_clear(msgpack_zone* zone);
 
+/** @} */
 
 
 #ifndef MSGPACK_ZONE_ALIGN

cpp/src/unpack.c

@@ -363,20 +363,46 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac)
 	mpac->parsed = 0;
 }
 
+bool msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone_free(result->zone);
+	}
+
+	int ret = msgpack_unpacker_execute(mpac);
+
+	if(ret <= 0) {
+		result->zone = NULL;
+		memset(&result->data, 0, sizeof(msgpack_object));
+		return false;
+	}
+
+	result->zone = msgpack_unpacker_release_zone(mpac);
+	result->data = msgpack_unpacker_data(mpac);
+	msgpack_unpacker_reset(mpac);
+
+	return true;
+}
+
 
 msgpack_unpack_return
 msgpack_unpack(const char* data, size_t len, size_t* off,
-		msgpack_zone* z, msgpack_object* result)
+		msgpack_zone* result_zone, msgpack_object* result)
 {
+	size_t noff = 0;
+	if(off != NULL) { noff = *off; }
+
+	if(len <= noff) {
+		// FIXME
+		return MSGPACK_UNPACK_CONTINUE;
+	}
+
 	template_context ctx;
 	template_init(&ctx);
 
-	ctx.user.z = z;
+	ctx.user.z = result_zone;
 	ctx.user.referenced = false;
 
-	size_t noff = 0;
-	if(off != NULL) { noff = *off; }
-
 	int e = template_execute(&ctx, data, len, &noff);
 	if(e < 0) {
 		return MSGPACK_UNPACK_PARSE_ERROR;
@@ -397,3 +423,37 @@ msgpack_unpack(const char* data, size_t len, size_t* off,
 	return MSGPACK_UNPACK_SUCCESS;
 }
 
+bool msgpack_unpack_next(msgpack_unpacked* result,
+		const char* data, size_t len, size_t* off)
+{
+	msgpack_unpacked_destroy(result);
+
+	size_t noff = 0;
+	if(off != NULL) { noff = *off; }
+
+	if(len <= noff) {
+		return false;
+	}
+
+	msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
+
+	template_context ctx;
+	template_init(&ctx);
+
+	ctx.user.z = z;
+	ctx.user.referenced = false;
+
+	int e = template_execute(&ctx, data, len, &noff);
+	if(e <= 0) {
+		msgpack_zone_free(z);
+		return false;
+	}
+
+	if(off != NULL) { *off = noff; }
+
+	result->zone = z;
+	result->data = template_data(&ctx);
+
+	return true;
+}
+

cpp/src/zone.c

@@ -214,6 +214,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size)
 
 void msgpack_zone_free(msgpack_zone* zone)
 {
+	if(zone == NULL) { return; }
 	msgpack_zone_destroy(zone);
 	free(zone);
 }

cpp/test/Makefile.am

@@ -6,10 +6,13 @@ AM_LDFLAGS = ../src/libmsgpack.la -lgtest_main
 check_PROGRAMS = \
 		zone \
 		pack_unpack \
+		pack_unpack_c \
 		streaming \
+		streaming_c \
 		object \
 		convert \
 		buffer \
+		cases \
 		msgpackc_test \
 		msgpack_test
 
@@ -19,8 +22,12 @@ zone_SOURCES = zone.cc
 
 pack_unpack_SOURCES = pack_unpack.cc
 
+pack_unpack_c_SOURCES = pack_unpack_c.cc
+
 streaming_SOURCES = streaming.cc
 
+streaming_c_SOURCES = streaming_c.cc
+
 object_SOURCES = object.cc
 
 convert_SOURCES = convert.cc
@@ -28,7 +35,11 @@ convert_SOURCES = convert.cc
 buffer_SOURCES = buffer.cc
 buffer_LDADD = -lz
 
+cases_SOURCES = cases.cc
+
 msgpackc_test_SOURCES = msgpackc_test.cpp
 
 msgpack_test_SOURCES = msgpack_test.cpp
 
+EXTRA_DIST = cases.mpac cases_compact.mpac
+

cpp/test/cases.cc

@@ -0,0 +1,36 @@
+#include <msgpack.hpp>
+#include <fstream>
+#include <gtest/gtest.h>
+
+static void feed_file(msgpack::unpacker& pac, const char* path)
+{
+	std::ifstream fin(path);
+	while(true) {
+		pac.reserve_buffer(32*1024);
+		fin.read(pac.buffer(), pac.buffer_capacity());
+		if(fin.bad()) {
+			throw std::runtime_error("read failed");
+		}
+		pac.buffer_consumed(fin.gcount());
+		if(fin.fail()) {
+			break;
+		}
+	}
+}
+
+TEST(cases, format)
+{
+	msgpack::unpacker pac;
+	msgpack::unpacker pac_compact;
+
+	feed_file(pac, "cases.mpac");
+	feed_file(pac_compact, "cases_compact.mpac");
+
+	msgpack::unpacked result;
+	while(pac.next(&result)) {
+		msgpack::unpacked result_compact;
+		EXPECT_TRUE( pac_compact.next(&result_compact) );
+		EXPECT_EQ(result_compact.get(), result.get());
+	}
+}
+

cpp/test/pack_unpack.cc

@@ -77,21 +77,17 @@ TEST(unpack, sequence)
 	msgpack::pack(sbuf, 2);
 	msgpack::pack(sbuf, 3);
 
-	bool cont;
 	size_t offset = 0;
 
 	msgpack::unpacked msg;
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_TRUE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(1, msg.get().as<int>());
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_TRUE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(2, msg.get().as<int>());
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_FALSE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(3, msg.get().as<int>());
 }
 

cpp/test/pack_unpack_c.cc

@@ -0,0 +1,70 @@
+#include <msgpack.h>
+#include <gtest/gtest.h>
+#include <stdio.h>
+
+TEST(pack, num)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_packer_free(pk);
+}
+
+
+TEST(pack, array)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_array(pk, 3));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_packer_free(pk);
+}
+
+
+TEST(unpack, sequence)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+
+	msgpack_packer_free(pk);
+
+	bool success;
+	size_t offset = 0;
+
+	msgpack_unpacked msg;
+	msgpack_unpacked_init(&msg);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(1, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(2, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(3, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_FALSE(success);
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_unpacked_destroy(&msg);
+}
+

cpp/test/streaming.cc

@@ -2,28 +2,33 @@
 #include <gtest/gtest.h>
 #include <sstream>
 
-
 TEST(streaming, basic)
 {
-	std::ostringstream stream;
-	msgpack::packer<std::ostream> pk(&stream);
+	msgpack::sbuffer buffer;
 
+	msgpack::packer<msgpack::sbuffer> pk(&buffer);
 	pk.pack(1);
 	pk.pack(2);
 	pk.pack(3);
 
-	std::istringstream input(stream.str());
+	const char* input = buffer.data();
+	const char* const eof = input + buffer.size();
 
 	msgpack::unpacker pac;
+	msgpack::unpacked result;
 
 	int count = 0;
 	while(count < 3) {
 		pac.reserve_buffer(32*1024);
 
-		size_t len = input.readsome(pac.buffer(), pac.buffer_capacity());
+		// read buffer into pac.buffer() upto
+		// pac.buffer_capacity() bytes.
+		size_t len = 1;
+		memcpy(pac.buffer(), input, len);
+		input += len;
+
 		pac.buffer_consumed(len);
 
-		msgpack::unpacked result;
 		while(pac.next(&result)) {
 			msgpack::object obj = result.get();
 			switch(count++) {
@@ -38,6 +43,8 @@ TEST(streaming, basic)
 				return;
 			}
 		}
+
+		EXPECT_TRUE(input < eof);
 	}
 }
 

cpp/test/streaming_c.cc

@@ -0,0 +1,57 @@
+#include <msgpack.h>
+#include <gtest/gtest.h>
+#include <stdio.h>
+
+TEST(streaming, basic)
+{
+	msgpack_sbuffer* buffer = msgpack_sbuffer_new();
+
+	msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+	msgpack_packer_free(pk);
+
+	const char* input = buffer->data;
+	const char* const eof = input + buffer->size;
+
+	msgpack_unpacker pac;
+	msgpack_unpacker_init(&pac, MSGPACK_UNPACKER_INIT_BUFFER_SIZE);
+
+	msgpack_unpacked result;
+	msgpack_unpacked_init(&result);
+
+	int count = 0;
+	while(count < 3) {
+		msgpack_unpacker_reserve_buffer(&pac, 32*1024);
+
+		/* read buffer into msgpack_unapcker_buffer(&pac) upto
+		 * msgpack_unpacker_buffer_capacity(&pac) bytes. */
+		size_t len = 1;
+		memcpy(msgpack_unpacker_buffer(&pac), input, len);
+		input += len;
+
+		msgpack_unpacker_buffer_consumed(&pac, len);
+
+		while(msgpack_unpacker_next(&pac, &result)) {
+			msgpack_object obj = result.data;
+			switch(count++) {
+			case 0:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(1, result.data.via.u64);
+				break;
+			case 1:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(2, result.data.via.u64);
+				break;
+			case 2:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(3, result.data.via.u64);
+				return;
+			}
+		}
+
+		EXPECT_TRUE(input < eof);
+	}
+}
+

ruby/makegem.sh

@@ -13,7 +13,9 @@ cp ../msgpack/pack_template.h   msgpack/
 cp ../msgpack/unpack_define.h   msgpack/
 cp ../msgpack/unpack_template.h msgpack/
 cp ../msgpack/sysdep.h          msgpack/
-cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > test/msgpack_test.rb
+cp ../test/cases.mpac           test/
+cp ../test/cases_compact.mpac   test/
+cp ../test/cases.json           test/
 
 gem build msgpack.gemspec
 

ruby/test/test_cases.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+here = File.dirname(__FILE__)
+require "#{here}/test_helper"
+
+begin
+require 'json'
+rescue LoadError
+require 'rubygems'
+require 'json'
+end
+
+CASES_PATH         = "#{here}/cases.mpac"
+CASES_COMPACT_PATH = "#{here}/cases_compact.mpac"
+CASES_JSON_PATH    = "#{here}/cases.json"
+
+class MessagePackTestCases < Test::Unit::TestCase
+	def feed_file(path)
+		pac = MessagePack::Unpacker.new
+		pac.feed File.read(path)
+		pac
+	end
+
+	def test_compare_compact
+		pac  = feed_file(CASES_PATH)
+		cpac = feed_file(CASES_COMPACT_PATH)
+
+		 objs = [];  pac.each {| obj|  objs <<  obj }
+		cobjs = []; cpac.each {|cobj| cobjs << cobj }
+
+		objs.zip(cobjs).each {|obj, cobj|
+			assert_equal(obj, cobj)
+		}
+	end
+
+	def test_compare_json
+		pac  = feed_file(CASES_PATH)
+
+		objs = []; pac.each {|obj| objs <<  obj }
+		jobjs = JSON.load File.read(CASES_JSON_PATH)
+
+		objs.zip(jobjs) {|obj, jobj|
+			assert_equal(obj, jobj)
+		}
+	end
+end
+

ruby/test/test_helper.rb

@@ -1,3 +1,7 @@
 require 'test/unit'
+begin
+require File.dirname(__FILE__) + '/../msgpack'
+rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
+end
 

ruby/test/test_pack_unpack.rb

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
-require 'msgpack'
-require 'test/unit'
+require File.dirname(__FILE__)+'/test_helper'
 
-class MessagePackTestFormat < Test::Unit::TestCase
+class MessagePackTestPackUnpack < Test::Unit::TestCase
 	def self.it(name, &block)
 		define_method("test_#{name}", &block)
 	end
@@ -177,16 +176,18 @@ class MessagePackTestFormat < Test::Unit::TestCase
 		match ({}), "\x80"
 	end
 
-	it "{0=>0, 1=>1, ..., 14=>14}" do
-		a = (0..14).to_a;
-		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
-	end
-
-	it "{0=>0, 1=>1, ..., 15=>15}" do
-		a = (0..15).to_a;
-		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
-	end
+## FIXME
+#	it "{0=>0, 1=>1, ..., 14=>14}" do
+#		a = (0..14).to_a;
+#		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
+#	end
+#
+#	it "{0=>0, 1=>1, ..., 15=>15}" do
+#		a = (0..15).to_a;
+#		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
+#	end
 
+## FIXME
 #	it "fixmap" do
 #		check_map 1, 0
 #		check_map 1, (1<<4)-1

test/README.md

@@ -0,0 +1,38 @@
+MessagePack cross-language test cases
+=====================================
+
+## cases
+
+Valid serialized data are stored in "cases.mpac" and "cases_compact.mpac".
+These files describe same objects. And "cases.json" describes an array of the described objects.
+
+Thus you can verify your implementations as comparing the objects.
+
+
+## crosslang
+
+The *crosslang* tool reads serialized data from stdin and writes re-serialize data to stdout.
+
+There are C++ and Ruby implementation of crosslang tool. You can verify your implementation
+as comparing that implementations.
+
+### C++ version
+
+    $ cd ../cpp && ./configure && make && make install
+    or
+    $ port install msgpack  # MacPorts
+    
+    $ g++ -Wall -lmsgpack crosslang.cc -o crosslang
+
+    $ ./crosslang
+    Usage: ./crosslang [in-file] [out-file]
+
+### Ruby version
+
+    $ gem install msgpack
+    or
+    $ port install rb_msgpack   # MacPorts
+
+    $ ruby crosslang.rb
+    Usage: crosslang.rb [in-file] [out-file]
+

test/cases.json

@@ -0,0 +1 @@
+[false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,65535,4294967295,-32,-32,-128,-32768,-2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",[0],[0],[0],[],[],[],{},{},{},{"a":97},{"a":97},{"a":97},[[]],[["a"]]]

test/cases_gen.rb

@@ -0,0 +1,99 @@
+#
+# MessagePack format test case
+#
+begin
+require 'rubygems'
+rescue LoadError
+end
+require 'msgpack'
+require 'json'
+
+source = <<EOF
+c2                          # false
+c3                          # true
+c0                          # nil
+00                          # 0 Positive FixNum
+cc 00                       # 0 uint8
+cd 00 00                    # 0 uint16
+ce 00 00 00 00              # 0 uint32
+cf 00 00 00 00 00 00 00 00  # 0 uint64
+d0 00                       # 0 int8
+d1 00 00                    # 0 int16
+d2 00 00 00 00              # 0 int32
+d3 00 00 00 00 00 00 00 00  # 0 int64
+ff                          # -1 Negative FixNum
+d0 ff                       # -1 int8
+d1 ff ff                    # -1 int16
+d2 ff ff ff ff              # -1 int32
+d3 ff ff ff ff ff ff ff ff  # -1 int64
+7f                          # 127 Positive FixNum
+cc 7f                       # 127 uint8
+cd 00 ff                    # 255 uint16
+ce 00 00 ff ff              # 65535 uint32
+cf 00 00 00 00 ff ff ff ff  # 4294967295 uint64
+e0                          # -32 Negative FixNum
+d0 e0                       # -32 int8
+d1 ff 80                    # -128 int16
+d2 ff ff 80 00              # -32768 int32
+d3 ff ff ff ff 80 00 00 00  # -2147483648 int64
+#ca 00 00 00 00              # 0.0 float
+cb 00 00 00 00 00 00 00 00  # 0.0 double
+#ca 80 00 00 00              # -0.0 float
+cb 80 00 00 00 00 00 00 00  # -0.0 double
+cb 3f f0 00 00 00 00 00 00  # 1.0 double
+cb bf f0 00 00 00 00 00 00  # -1.0 double
+a1 61                       # "a" FixRaw
+da 00 01 61                 # "a" raw 16
+db 00 00 00 01 61           # "a" raw 32
+a0                          # "" FixRaw
+da 00 00                    # "" raw 16
+db 00 00 00 00              # "" raw 32
+91 00                       # [0] FixArray
+dc 00 01 00                 # [0] array 16
+dd 00 00 00 01 00           # [0] array 32
+90                          # [] FixArray
+dc 00 00                    # [] array 16
+dd 00 00 00 00              # [] array 32
+80                          # {} FixMap
+de 00 00                    # {} map 16
+df 00 00 00 00              # {} map 32
+81 a1 61 61                 # {"a"=>97} FixMap
+de 00 01 a1 61 61           # {"a"=>97} map 16
+df 00 00 00 01 a1 61 61     # {"a"=>97} map 32
+91 90                       # [[]]
+91 91 a1 61                 # [["a"]]
+EOF
+
+source.gsub!(/\#.+$/,'')
+bytes = source.strip.split(/\s+/).map {|x| x.to_i(16) }.pack('C*')
+
+objs = []
+compact_bytes = ""
+
+pac = MessagePack::Unpacker.new
+pac.feed(bytes)
+pac.each {|obj|
+	p obj
+	objs << obj
+	compact_bytes << obj.to_msgpack
+}
+
+json = objs.to_json
+
+# self check
+cpac = MessagePack::Unpacker.new
+cpac.feed(compact_bytes)
+cpac.each {|cobj|
+	obj = objs.shift
+	if obj != cobj
+		puts "** SELF CHECK FAILED **"
+		puts "expected: #{obj.inspect}"
+		puts "actual: #{cobj.inspect}"
+		exit 1
+	end
+}
+
+File.open("cases.mpac","w") {|f| f.write(bytes) }
+File.open("cases_compact.mpac","w") {|f| f.write(compact_bytes) }
+File.open("cases.json","w") {|f| f.write(json) }
+

test/crosslang.cc

@@ -1,5 +1,10 @@
 //
-// How to compile:
+// MessagePack cross-language test tool
+//
+// $ cd ../cpp && ./configure && make && make install
+// or
+// $ port install msgpack  # MacPorts
+//
 // $ g++ -Wall -lmsgpack crosslang.cc
 //
 #include <msgpack.hpp>

test/crosslang.rb

@@ -1,3 +1,10 @@
+#
+# MessagePack cross-language test tool
+#
+# $ gem install msgpack
+# or
+# $ port install rb_msgpack   # MacPorts
+#
 begin
 require 'rubygems'
 rescue LoadError