version 0.3.3

Makefile.am

@@ -26,5 +26,23 @@ EXTRA_DIST = \
 	ruby/test_format.rb \
 	ruby/test_pack.rb \
 	ruby/unpack.c \
-	ruby/unpack.h
+	ruby/unpack.h \
+	perl/Makefile.PL \
+	perl/MessagePack.c \
+	perl/benchmark/deserialize.pl \
+	perl/benchmark/serialize.pl \
+	perl/lib/Data/MessagePack.pm \
+	perl/lib/Data/MessagePack \
+	perl/lib/Data/MessagePack/Unpacker.pod \
+	perl/pack.c \
+	perl/ppport.h \
+	perl/t/00_compile.t \
+	perl/t/01_pack.t \
+	perl/t/02_unpack.t \
+	perl/t/03_stream_unpack.t \
+	perl/t/04_invert.t \
+	perl/t/Util.pm \
+	perl/t/data.pl \
+	perl/unpack.c \
+	perl/xt/99_pod.t
 

README

@@ -8,7 +8,7 @@ Binary-based efficient data interchange format.
   MessagePack is only tested on Linux and Mac OS X, but it may run on other
   UNIX-like platforms.
 
-  Following programs is required to build:
+  Following programs are required to build:
     - gcc >= 4.1 with C++ support
     - ruby >= 1.8   (ruby is used as a preprocessor)
 

c/Makefile.am

@@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la
 libmsgpackc_la_SOURCES = \
 		unpack.c \
 		object.c \
+		vrefbuffer.c \
 		zone.c
 
 nobase_include_HEADERS = \
 		msgpack.h \
 		msgpack/sbuffer.h \
+		msgpack/vrefbuffer.h \
 		msgpack/pack.h \
 		msgpack/unpack.h \
 		msgpack/object.h \

c/msgpack.h

@@ -20,3 +20,4 @@
 #include "msgpack/pack.h"
 #include "msgpack/unpack.h"
 #include "msgpack/sbuffer.h"
+#include "msgpack/vrefbuffer.h"

c/vrefbuffer.c

@@ -0,0 +1,135 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#include "msgpack/vrefbuffer.h"
+#include <stdlib.h>
+#include <string.h>
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size)
+{
+	if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) {
+		chunk_size = 72;
+	} else {
+		chunk_size -= sizeof(msgpack_vrefbuffer_chunk);
+	}
+
+	vbuf->chunk_size = chunk_size;
+	vbuf->ref_size = ref_size;
+
+	// glibcは72バイト以下のmallocが高速
+	size_t nfirst = (sizeof(struct iovec) < 72/2) ?
+			72 / sizeof(struct iovec) : 8;
+
+	struct iovec* array = (struct iovec*)malloc(
+			sizeof(struct iovec) * nfirst);
+	if(array == NULL) {
+		return false;
+	}
+
+	vbuf->tail  = array;
+	vbuf->end   = array + nfirst;
+	vbuf->array = array;
+
+	vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc(
+			chunk_size + sizeof(msgpack_vrefbuffer_chunk));
+	if(vbuf->chunk == NULL) {
+		free(array);
+		return false;
+	}
+
+	vbuf->chunk->next = NULL;
+	vbuf->chunk->free = chunk_size;
+
+	return true;
+}
+
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf)
+{
+	msgpack_vrefbuffer_chunk* c = vbuf->chunk;
+	while(true) {
+		msgpack_vrefbuffer_chunk* n = c->next;
+		free(c);
+		if(n) {
+			c = n;
+		} else {
+			break;
+		}
+	}
+	free(vbuf->array);
+}
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	if(vbuf->tail == vbuf->end) {
+		const size_t nused = vbuf->end - vbuf->array;
+		const size_t nnext = nused * 2;
+
+		struct iovec* nvec = (struct iovec*)realloc(
+				vbuf->array, sizeof(struct iovec)*nnext);
+		if(nvec == NULL) {
+			return -1;
+		}
+
+		vbuf->array = nvec;
+		vbuf->end   = nvec + nnext;
+		vbuf->tail  = nvec + nused;
+	}
+
+	vbuf->tail->iov_base = (char*)buf;
+	vbuf->tail->iov_len  = len;
+	++vbuf->tail;
+
+	return 0;
+}
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer_chunk* chunk = vbuf->chunk;
+	size_t cur_size = vbuf->chunk_size;
+
+	if(chunk->free < len) {
+		cur_size = (cur_size > len) ? cur_size : len;
+
+		chunk = (msgpack_vrefbuffer_chunk*)malloc(
+				cur_size + sizeof(msgpack_vrefbuffer_chunk));
+		if(chunk == NULL) {
+			return -1;
+		}
+
+		chunk->free = cur_size;
+		chunk->next = vbuf->chunk;
+		vbuf->chunk = chunk;
+	}
+
+	char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk)
+		+ (cur_size - chunk->free);
+
+	memcpy(m, buf, len);
+	chunk->free -= len;
+
+	if(vbuf->tail != vbuf->array && m ==
+			(const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) {
+		(vbuf->tail-1)->iov_len += len;
+		return 0;
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, m, len);
+	}
+}
+

c/vrefbuffer.h

@@ -0,0 +1,97 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_VREFBUFFER_H__
+#define MSGPACK_VREFBUFFER_H__
+
+#include "msgpack/zone.h"
+#include <sys/uio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_VREFBUFFER_REF_SIZE
+#define MSGPACK_VREFBUFFER_REF_SIZE 32
+#endif
+
+#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
+#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048
+#endif
+
+typedef struct msgpack_vrefbuffer_chunk {
+	size_t free;
+	struct msgpack_vrefbuffer_chunk* next;
+	/* data ... */
+} msgpack_vrefbuffer_chunk;
+
+typedef struct msgpack_vrefbuffer {
+	size_t chunk_size;
+	size_t ref_size;
+
+	struct iovec* tail;
+	struct iovec* end;
+	struct iovec* array;
+
+	msgpack_vrefbuffer_chunk* chunk;
+} msgpack_vrefbuffer;
+
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size);
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
+
+static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len);
+
+static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref);
+static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref);
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+
+int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data;
+
+	if(len < vbuf->ref_size) {
+		return msgpack_vrefbuffer_append_copy(vbuf, buf, len);
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, buf, len);
+	}
+}
+
+const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref)
+{
+	return vref->array;
+}
+
+size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref)
+{
+	return vref->tail - vref->array;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/vrefbuffer.h */
+

c/zone.c

@@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s
 
 	msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc(
 			sizeof(msgpack_zone_chunk) * nfirst);
-	if(!array) {
+	if(array == NULL) {
 		return false;
 	}
 

c/zone.h

@@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size);
 void msgpack_zone_free(msgpack_zone* zone);
 
 static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size);
+static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size);
 
 static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone,
 		void (*func)(void* data), void* data);
@@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone);
 
 void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size);
 
-void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size)
 {
-	size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1);
-
 	msgpack_zone_chunk* chunk = zone->chunk_array.tail;
 
 	if(chunk->free < size) {
@@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
 	return ptr;
 }
 
+void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+{
+	return msgpack_zone_malloc_no_align(zone,
+			((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1));
+}
+
 
 bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone,
 		void (*func)(void* data), void* data);

configure.in

@@ -1,6 +1,6 @@
 AC_INIT(msgpack/unpack_template.h)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.3.0)
+AM_INIT_AUTOMAKE(msgpack, 0.3.3)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

cpp/Makefile.am

@@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \
 nobase_include_HEADERS = \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
+		msgpack/vrefbuffer.hpp \
 		msgpack/pack.hpp \
 		msgpack/unpack.hpp \
 		msgpack/object.hpp \

cpp/msgpack.hpp

@@ -20,4 +20,5 @@
 #include "msgpack/pack.hpp"
 #include "msgpack/unpack.hpp"
 #include "msgpack/sbuffer.hpp"
+#include "msgpack/vrefbuffer.hpp"
 #include "msgpack.h"

cpp/object.hpp

@@ -80,13 +80,13 @@ struct object {
 	type::object_type type;
 	union_type via;
 
-	bool is_nil() { return type == type::NIL; }
+	bool is_nil() const { return type == type::NIL; }
 
 	template <typename T>
-	T as();
+	T as() const;
 
 	template <typename T>
-	void convert(T* v);
+	void convert(T* v) const;
 
 	object();
 	object(msgpack_object obj);
@@ -96,7 +96,7 @@ private:
 	struct implicit_type;
 
 public:
-	implicit_type convert();
+	implicit_type convert() const;
 };
 
 struct object_kv {
@@ -201,25 +201,25 @@ inline object::operator msgpack_object()
 }
 
 
-inline object::implicit_type object::convert()
+inline object::implicit_type object::convert() const
 {
 	return implicit_type(*this);
 }
 
 template <typename T>
-inline T object::as()
+inline void object::convert(T* v) const
+{
+	*this >> *v;
+}
+
+template <typename T>
+inline T object::as() const
 {
 	T v;
 	convert(&v);
 	return v;
 }
 
-template <typename T>
-inline void object::convert(T* v)
-{
-	*this >> *v;
-}
-
 
 // obsolete
 template <typename T>

cpp/type/map.hpp

@@ -80,13 +80,13 @@ inline std::map<K, V> operator>> (object o, std::map<K, V>& v)
 	for(; p != pend; ++p) {
 		K key;
 		p->key.convert(&key);
-		typename std::map<K,V>::iterator it(v.find(key));
-		if(it != v.end()) {
+		typename std::map<K,V>::iterator it(v.lower_bound(key));
+		if(it != v.end() && !(key < it->first)) {
+			p->val.convert(&it->second);
+		} else {
 			V val;
 			p->val.convert(&val);
-			it->insert( std::pair<K,V>(key, val) );
-		} else {
-			p->val.convert(&it->second);
+			v.insert(it, std::pair<K,V>(key, val));
 		}
 	}
 	return v;

cpp/vrefbuffer.hpp

@@ -0,0 +1,85 @@
+//
+// MessagePack for C++ zero-copy buffer implementation
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_VREFBUFFER_HPP__
+#define MSGPACK_VREFBUFFER_HPP__
+
+#include "msgpack/vrefbuffer.h"
+#include <stdexcept>
+
+namespace msgpack {
+
+
+class vrefbuffer : public msgpack_vrefbuffer {
+public:
+	vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE,
+			size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE)
+	{
+		msgpack_vrefbuffer_init(this, ref_size, chunk_size);
+	}
+
+	~vrefbuffer()
+	{
+		msgpack_vrefbuffer_destroy(this);
+	}
+
+public:
+	void write(const char* buf, unsigned int len)
+	{
+		if(len < base::ref_size) {
+			append_copy(buf, len);
+		} else {
+			append_ref(buf, len);
+		}
+	}
+
+	void append_ref(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	void append_copy(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	const struct iovec* vector() const
+	{
+		return msgpack_vrefbuffer_vec(this);
+	}
+
+	size_t vector_size() const
+	{
+		return msgpack_vrefbuffer_veclen(this);
+	}
+
+private:
+	typedef msgpack_vrefbuffer base;
+
+private:
+	vrefbuffer(const vrefbuffer&);
+};
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/vrefbuffer.hpp */
+

cpp/zone.hpp.erb

@@ -34,6 +34,7 @@ public:
 
 public:
 	void* malloc(size_t size);
+	void* malloc_no_align(size_t size);
 
 	void push_finalizer(void (*func)(void*), void* data);
 
@@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size)
 	return ptr;
 }
 
+inline void* zone::malloc_no_align(size_t size)
+{
+	void* ptr = msgpack_zone_malloc_no_align(this, size);
+	if(!ptr) {
+		throw std::bad_alloc();
+	}
+	return ptr;
+}
+
 inline void zone::push_finalizer(void (*func)(void*), void* data)
 {
 	if(!msgpack_zone_push_finalizer(this, func, data)) {

msgpack/pack_template.h

@@ -24,29 +24,79 @@
 #endif
 #endif
 
+
 #ifdef __LITTLE_ENDIAN__
 
-#define STORE_BE16(d) \
-	((char*)&d)[1], ((char*)&d)[0]
+#define STORE8_BE8(d) \
+	((uint8_t*)&d)[0]
 
-#define STORE_BE32(d) \
-	((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0]
 
-#define STORE_BE64(d) \
-	((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \
-	((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0]
+#define STORE16_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE16_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+
+#define STORE32_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE32_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE32_BE32(d) \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+
+#define STORE64_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE64_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE64_BE32(d) \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE64_BE64(d) \
+	((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
 
 #elif __BIG_ENDIAN__
 
-#define STORE_BE16(d) \
-	((char*)&d)[0], ((char*)&d)[1]
+#define STORE8_BE8(d) \
+	((uint8_t*)&d)[0]
 
-#define STORE_BE32(d) \
-	((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3]
 
-#define STORE_BE64(d) \
-	((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \
-	((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7]
+#define STORE16_BE8(d) \
+	((uint8_t*)&d)[1]
+
+#define STORE16_BE16(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1]
+
+
+#define STORE32_BE8(d) \
+	((uint8_t*)&d)[3]
+
+#define STORE32_BE16(d) \
+	((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
+
+#define STORE32_BE32(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
+
+
+#define STORE64_BE8(d) \
+	((uint8_t*)&d)[7]
+
+#define STORE64_BE16(d) \
+	((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
+
+#define STORE64_BE32(d) \
+	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
+
+#define STORE64_BE64(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \
+	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
 
 #endif
 
@@ -71,10 +121,10 @@
 do { \
 	if(d < (1<<7)) { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
 	} else { \
 		/* unsigned 8 */ \
-		const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+		const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \
 		msgpack_pack_append_buffer(x, buf, 2); \
 	} \
 } while(0)
@@ -83,14 +133,14 @@ do { \
 do { \
 	if(d < (1<<7)) { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
 	} else if(d < (1<<8)) { \
 		/* unsigned 8 */ \
-		const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+		const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
 		msgpack_pack_append_buffer(x, buf, 2); \
 	} else { \
 		/* unsigned 16 */ \
-		const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+		const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
 		msgpack_pack_append_buffer(x, buf, 3); \
 	} \
 } while(0)
@@ -100,20 +150,20 @@ do { \
 	if(d < (1<<8)) { \
 		if(d < (1<<7)) { \
 			/* fixnum */ \
-			msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+			msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
 		} else { \
 			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} \
 	} else { \
 		if(d < (1<<16)) { \
 			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
-			const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \
+			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -124,24 +174,24 @@ do { \
 	if(d < (1ULL<<8)) { \
 		if(d < (1<<7)) { \
 			/* fixnum */ \
-			msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+			msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
 		} else { \
 			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} \
 	} else { \
 		if(d < (1ULL<<16)) { \
 			/* signed 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else if(d < (1ULL<<32)) { \
 			/* signed 32 */ \
-			const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \
+			const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else { \
 			/* signed 64 */ \
-			const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \
+			const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
 			msgpack_pack_append_buffer(x, buf, 9); \
 		} \
 	} \
@@ -151,11 +201,11 @@ do { \
 do { \
 	if(d < -(1<<5)) { \
 		/* signed 8 */ \
-		const unsigned char buf[2] = {0xd0, d}; \
+		const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \
 		msgpack_pack_append_buffer(x, buf, 2); \
 	} else { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
 	} \
 } while(0)
 
@@ -164,24 +214,24 @@ do { \
 	if(d < -(1<<5)) { \
 		if(d < -(1<<7)) { \
 			/* signed 16 */ \
-			const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
-			const unsigned char buf[2] = {0xd0, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} \
 	} else if(d < (1<<7)) { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
 	} else { \
 		if(d < (1<<8)) { \
 			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} else { \
 			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} \
 	} \
@@ -192,32 +242,32 @@ do { \
 	if(d < -(1<<5)) { \
 		if(d < -(1<<15)) { \
 			/* signed 32 */ \
-			const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \
+			const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else if(d < -(1<<7)) { \
 			/* signed 16 */ \
-			const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
-			const unsigned char buf[2] = {0xd0, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} \
 	} else if(d < (1<<7)) { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
 	} else { \
 		if(d < (1<<8)) { \
 			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
 			msgpack_pack_append_buffer(x, buf, 2); \
 		} else if(d < (1<<16)) { \
 			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
-			const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \
+			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -229,46 +279,46 @@ do { \
 		if(d < -(1LL<<15)) { \
 			if(d < -(1LL<<31)) { \
 				/* signed 64 */ \
-				const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \
+				const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \
 				msgpack_pack_append_buffer(x, buf, 9); \
 			} else { \
 				/* signed 32 */ \
-				const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \
+				const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} \
 		} else { \
 			if(d < -(1<<7)) { \
 				/* signed 16 */ \
-				const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \
+				const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} else { \
 				/* signed 8 */ \
-				const unsigned char buf[2] = {0xd0, (uint8_t)d}; \
+				const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \
 				msgpack_pack_append_buffer(x, buf, 2); \
 			} \
 		} \
 	} else if(d < (1<<7)) { \
 		/* fixnum */ \
-		msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \
+		msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
 	} else { \
 		if(d < (1LL<<16)) { \
 			if(d < (1<<8)) { \
 				/* unsigned 8 */ \
-				const unsigned char buf[2] = {0xcc, (uint8_t)d}; \
+				const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
 				msgpack_pack_append_buffer(x, buf, 2); \
 			} else { \
 				/* unsigned 16 */ \
-				const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \
+				const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} \
 		} else { \
 			if(d < (1LL<<32)) { \
 				/* unsigned 32 */ \
-				const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \
+				const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} else { \
 				/* unsigned 64 */ \
-				const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \
+				const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
 				msgpack_pack_append_buffer(x, buf, 9); \
 			} \
 		} \
@@ -280,49 +330,49 @@ do { \
 
 msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
 {
-	const unsigned char buf[2] = {0xcc, d};
+	const unsigned char buf[2] = {0xcc, STORE8_BE8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
 msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
 {
-	const unsigned char buf[3] = {0xcd, STORE_BE16(d)};
+	const unsigned char buf[3] = {0xcd, STORE16_BE16(d)};
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
 msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
 {
-	const unsigned char buf[5] = {0xce, STORE_BE32(d)};
+	const unsigned char buf[5] = {0xce, STORE32_BE32(d)};
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
 msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
 {
-	const unsigned char buf[9] = {0xcf, STORE_BE64(d)};
+	const unsigned char buf[9] = {0xcf, STORE64_BE64(d)};
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
 msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
 {
-	const unsigned char buf[2] = {0xd0, d};
+	const unsigned char buf[2] = {0xd0, STORE8_BE8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
 msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
 {
-	const unsigned char buf[3] = {0xd1, STORE_BE16(d)};
+	const unsigned char buf[3] = {0xd1, STORE16_BE16(d)};
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
 msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
 {
-	const unsigned char buf[5] = {0xd2, STORE_BE32(d)};
+	const unsigned char buf[5] = {0xd2, STORE32_BE32(d)};
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
 msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
 {
-	const unsigned char buf[9] = {0xd3, STORE_BE64(d)};
+	const unsigned char buf[9] = {0xd3, STORE64_BE64(d)};
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
@@ -554,7 +604,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
 {
 	union { char buf[4]; uint32_t num; } f;
 	*((float*)&f.buf) = d;  // FIXME
-	const unsigned char buf[5] = {0xca, STORE_BE32(f.num)};
+	const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)};
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
@@ -562,7 +612,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
 {
 	union { char buf[8]; uint64_t num; } f;
 	*((double*)&f.buf) = d;  // FIXME
-	const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)};
+	const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)};
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
@@ -606,11 +656,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
 		msgpack_pack_append_buffer(x, &d, 1);
 	} else if(n < 65536) {
 		uint16_t d = (uint16_t)n;
-		unsigned char buf[3] = {0xdc, STORE_BE16(d)};
+		unsigned char buf[3] = {0xdc, STORE16_BE16(d)};
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		uint32_t d = (uint32_t)n;
-		unsigned char buf[5] = {0xdd, STORE_BE32(d)};
+		unsigned char buf[5] = {0xdd, STORE32_BE32(d)};
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -624,14 +674,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
 {
 	if(n < 16) {
 		unsigned char d = 0x80 | n;
-		msgpack_pack_append_buffer(x, &d, 1);
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
 	} else if(n < 65536) {
 		uint16_t d = (uint16_t)n;
-		unsigned char buf[3] = {0xde, STORE_BE16(d)};
+		unsigned char buf[3] = {0xde, STORE16_BE16(d)};
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		uint32_t d = (uint32_t)n;
-		unsigned char buf[5] = {0xdf, STORE_BE32(d)};
+		unsigned char buf[5] = {0xdf, STORE32_BE32(d)};
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -645,14 +695,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
 {
 	if(l < 32) {
 		unsigned char d = 0xa0 | l;
-		msgpack_pack_append_buffer(x, &d, 1);
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
 	} else if(l < 65536) {
 		uint16_t d = (uint16_t)l;
-		unsigned char buf[3] = {0xda, STORE_BE16(d)};
+		unsigned char buf[3] = {0xda, STORE16_BE16(d)};
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		uint32_t d = (uint32_t)l;
-		unsigned char buf[5] = {0xdb, STORE_BE32(d)};
+		unsigned char buf[5] = {0xdb, STORE32_BE32(d)};
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -666,9 +716,19 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l
 #undef msgpack_pack_user
 #undef msgpack_pack_append_buffer
 
-#undef STORE_BE16
-#undef STORE_BE32
-#undef STORE_BE64
+#undef STORE8_BE8
+
+#undef STORE16_BE8
+#undef STORE16_BE16
+
+#undef STORE32_BE8
+#undef STORE32_BE16
+#undef STORE32_BE32
+
+#undef STORE64_BE8
+#undef STORE64_BE16
+#undef STORE64_BE32
+#undef STORE64_BE64
 
 #undef msgpack_pack_real_uint8
 #undef msgpack_pack_real_uint16

perl/Makefile.PL

@@ -0,0 +1,34 @@
+use inc::Module::Install;
+name 'Data-MessagePack';
+all_from 'lib/Data/MessagePack.pm';
+
+perl_version '5.008005';
+license 'perl';
+can_cc or die "This module requires a C compiler";
+
+my $ccflags = '-I../ ';
+
+makemaker_args(
+    OBJECT => '$(O_FILES)',
+    LIBS => [''],
+    CCFLAGS => $ccflags,
+    clean => {
+        FILES => q{
+            *.stackdump
+            *.gcov *.gcda *.gcno
+            *.out
+            nytprof
+            cover_db
+        },
+    },
+);
+
+tests 't/*.t';
+author_tests('xt');
+
+auto_set_repository;
+build_requires 'Test::More';
+use_test_base;
+auto_include;
+WriteAll;
+

perl/MessagePack.c

@@ -0,0 +1,40 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#define NEED_newCONSTSUB
+#include "ppport.h"
+#ifdef __cplusplus
+};
+#endif
+
+XS(xs_pack);
+XS(xs_unpack);
+XS(xs_unpacker_new);
+XS(xs_unpacker_execute);
+XS(xs_unpacker_execute_limit);
+XS(xs_unpacker_is_finished);
+XS(xs_unpacker_data);
+XS(xs_unpacker_reset);
+
+XS(boot_Data__MessagePack) {
+    dXSARGS;
+    HV * stash;
+
+    newXS("Data::MessagePack::pack", xs_pack, __FILE__);
+    newXS("Data::MessagePack::unpack", xs_unpack, __FILE__);
+    stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE);
+    newCONSTSUB(stash, "true", &PL_sv_yes);
+    newCONSTSUB(stash, "false", &PL_sv_no);
+
+    newXS("Data::MessagePack::Unpacker::new",           xs_unpacker_new, __FILE__);
+    newXS("Data::MessagePack::Unpacker::execute",       xs_unpacker_execute, __FILE__);
+    newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__);
+    newXS("Data::MessagePack::Unpacker::is_finished",   xs_unpacker_is_finished, __FILE__);
+    newXS("Data::MessagePack::Unpacker::data",          xs_unpacker_data, __FILE__);
+    newXS("Data::MessagePack::Unpacker::reset",         xs_unpacker_reset, __FILE__);
+
+}
+

perl/benchmark/deserialize.pl

@@ -0,0 +1,20 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use JSON::XS;
+use Benchmark ':all';
+
+my $a = [0..2**24];
+my $j = JSON::XS::encode_json($a);
+my $m = Data::MessagePack->pack($a);
+
+print "-- deserialize\n";
+print "JSON::XS: $JSON::XS::VERSION\n";
+print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+cmpthese(
+    -1 => {
+        json => sub { JSON::XS::decode_json($j)     },
+        mp   => sub { Data::MessagePack->unpack($m) },
+    }
+);
+

perl/benchmark/serialize.pl

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use JSON::XS;
+use Benchmark ':all';
+
+my $a = [0..2**24];
+
+print "-- serialize\n";
+print "JSON::XS: $JSON::XS::VERSION\n";
+print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+cmpthese(
+    -1 => {
+        json => sub { JSON::XS::encode_json($a)   },
+        mp   => sub { Data::MessagePack->pack($a) },
+    }
+);
+

perl/lib/Data/MessagePack.pm

@@ -0,0 +1,33 @@
+package Data::MessagePack;
+use strict;
+use warnings;
+use XSLoader;
+
+our $VERSION = 0.01;
+
+XSLoader::load(__PACKAGE__, $VERSION);
+
+1;
+__END__
+
+=head1 NAME
+
+Data::MessagePack - messagepack
+
+=head1 SYNOPSIS
+
+    my $packed = Data::MessagePack->pack($dat);
+    my $unpacked = Data::MessagePack->unpack($dat);
+
+=head1 DESCRIPTION
+
+Data::MessagePack is a binary packer for perl.
+
+=head1 AUTHORS
+
+Tokuhiro Matsuno
+
+=head1 SEE ALSO
+
+L<http://msgpack.sourceforge.jp/>
+

perl/lib/Data/MessagePack/Unpacker.pod

@@ -0,0 +1,52 @@
+=head1 NAME
+
+Data::MessagePack::Unpacker - messagepack streaming deserializer
+
+=head1 SYNOPSIS
+
+    use Data::Dumper;
+    my $up = Data::MessagePack::Unpacker->new;
+    my $ret = $up->execute($v, 0);
+    if ($ret != length($v)) {
+        fail "extra bytes";
+    }
+    return Dumper($up->data);
+
+=head1 DESCRIPTION
+
+This is an streaming deserializer for messagepack.
+
+=head1 METHODS
+
+=over 4
+
+=item my $up = Data::MessagePack::Unpacker->new()
+
+create new stream deserializer
+
+=item $up->execute()
+
+=item $up->execute_limit()
+
+=item $up->is_finished()
+
+is this deserializer finished?
+
+=item $up->data()
+
+returns deserialized object.
+
+=item $up->reset()
+
+reset the stream deserializer, without memory zone.
+
+=back
+
+=head1 AUTHORS
+
+Tokuhiro Matsuno
+
+=head1 SEE ALSO
+
+L<Data::MessagePack>
+

perl/pack.c

@@ -0,0 +1,155 @@
+/*
+ * code is written by tokuhirom.
+ * buffer alocation technique is taken from JSON::XS. thanks to mlehmann.
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+#ifdef __cplusplus
+};
+#endif
+
+#include "msgpack/pack_define.h"
+
+#define msgpack_pack_inline_func(name) \
+    static inline void msgpack_pack ## name
+
+#define msgpack_pack_inline_func_cint(name) \
+    static inline void msgpack_pack ## name
+
+typedef struct {
+    char *cur; /* SvPVX (sv) + current output position */
+    char *end; /* SvEND (sv) */
+    SV *sv;    /* result scalar */
+} enc_t;
+void need(enc_t *enc, STRLEN len);
+
+#define msgpack_pack_user enc_t*
+
+#define msgpack_pack_append_buffer(enc, buf, len) \
+    need(enc, len); \
+    memcpy(enc->cur, buf, len); \
+    enc->cur += len;
+
+#include "msgpack/pack_template.h"
+
+#define _PACK_WRAPPER(t) msgpack_pack_##t
+#define PACK_WRAPPER(t) _PACK_WRAPPER(t)
+#define INIT_SIZE   32 /* initial scalar size to be allocated */
+
+void need(enc_t *enc, STRLEN len)
+{
+    if (enc->cur + len >= enc->end) {
+        STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
+        SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
+        enc->cur = SvPVX (enc->sv) + cur;
+        enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1;
+    }
+}
+
+static void _msgpack_pack_sv(enc_t *enc, SV* val) {
+    if (val==NULL) {
+        msgpack_pack_nil(enc);
+        return;
+    }
+
+    switch (SvTYPE(val)) {
+    case SVt_NULL:
+        msgpack_pack_nil(enc);
+        break;
+    case SVt_IV:
+        if (SvIOK_UV(val)) {
+            msgpack_pack_uint32(enc, SvUV(val));
+        } else {
+            PACK_WRAPPER(IVTYPE)(enc, SvIV(val));
+        }
+        break;
+    case SVt_PVNV:
+        {
+            STRLEN len = 0;
+            char *pv = SvPV(val, len);
+            if (len == 1 && *pv == '1') {
+                msgpack_pack_true(enc);
+            } else if (len == 0 && *pv==0) {
+                msgpack_pack_false(enc);
+            } else {
+                msgpack_pack_nil(enc);
+            }
+        }
+        break;
+    case SVt_NV:
+        PACK_WRAPPER(NVTYPE)(enc, SvNV(val));
+        break;
+    case SVt_PVAV:
+        {
+            AV* ary = (AV*)val;
+            int len = av_len(ary) + 1;
+            int i;
+            msgpack_pack_array(enc, len);
+            for (i=0; i<len; i++) {
+                SV** svp = av_fetch(ary, i, 0);
+                if (svp) {
+                    _msgpack_pack_sv(enc, *svp);
+                } else {
+                    msgpack_pack_nil(enc);
+                }
+            }
+        }
+        break;
+    case SVt_PVHV:
+        {
+            HV* hval = (HV*)val;
+            int count = hv_iterinit(hval);
+            HE* he;
+
+            msgpack_pack_map(enc, count);
+
+            while (he = hv_iternext(hval)) {
+                _msgpack_pack_sv(enc, hv_iterkeysv(he));
+                _msgpack_pack_sv(enc, HeVAL(he));
+            }
+        }
+        break;
+    case SVt_RV:
+        _msgpack_pack_sv(enc, SvRV(val));
+        break;
+    default:
+        if (SvPOKp(val)) {
+            STRLEN len;
+            char * cval = SvPV(val, len);
+            msgpack_pack_raw(enc, len);
+            msgpack_pack_raw_body(enc, cval, len);
+            return;
+        } else {
+            sv_dump(val);
+            Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val));
+        }
+    }
+}
+
+XS(xs_pack) {
+    dXSARGS;
+    if (items != 2) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)");
+    }
+
+    SV* val = ST(1);
+
+    enc_t enc;
+    enc.sv        = sv_2mortal(NEWSV(0, INIT_SIZE));
+    enc.cur       = SvPVX(enc.sv);
+    enc.end       = SvEND(enc.sv);
+    SvPOK_only(enc.sv);
+
+    _msgpack_pack_sv(&enc, val);
+
+    SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
+    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
+
+    ST(0) = enc.sv;
+    XSRETURN(1);
+}

perl/t/00_compile.t

@@ -0,0 +1,6 @@
+use strict;
+use warnings;
+use Test::More tests => 1;
+
+use_ok 'Data::MessagePack';
+

perl/t/01_pack.t

@@ -0,0 +1,60 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+
+sub packit {
+    local $_ = unpack("H*", Data::MessagePack->pack($_[0]));
+    s/(..)/$1 /g;
+    s/ $//;
+    $_;
+}
+
+sub pis ($$) {
+    is packit($_[0]), $_[1], 'dump ' . $_[1];
+}
+
+my @dat = (
+    0,     '00',
+    1,     '01',
+    127,   '7f',
+    128,   'cc 80',
+    255,   'cc ff',
+    256,   'cd 01 00',
+    65535, 'cd ff ff',
+    65536, 'ce 00 01 00 00',
+    -1,    'ff',
+    -32,   'e0',
+    -33,   'd0 df',
+    -128,  'd0 80',
+    -129,  'd1 ff 7f',
+    -32768, 'd1 80 00',
+    -32769, 'd2 ff ff 7f ff',
+    1.0,   'cb 3f f0 00 00 00 00 00 00',
+    "",    'a0',
+    "a",   'a1 61',
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61',
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61',
+    undef, 'c0',
+    Data::MessagePack::true(), 'c3',
+    Data::MessagePack::false(), 'c2',
+    [], '90',
+    [+[]], '91 90',
+    [[], undef], '92 90 c0',
+    {'a', 0}, '81 a1 61 00',
+    8388608, 'ce 00 80 00 00',
+
+    [undef, false, true], '93 c0 c2 c3',
+    ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66',
+    [[], [[undef]]], '92 90 91 91 c0',
+    [undef, false, true], '93 c0 c2 c3',
+    [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff',
+    [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff',
+    2147483648, 'ce 80 00 00 00',
+    -2147483648, 'd2 80 00 00 00',
+);
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/02_unpack.t

@@ -0,0 +1,24 @@
+use Test::More;
+use Data::MessagePack;
+use t::Util;
+no warnings 'uninitialized'; # i need this. i need this.
+
+sub unpackit {
+    my $v = $_[0];
+    $v =~ s/ //g;
+    $v = pack 'H*', $v;
+    return Data::MessagePack->unpack($v);
+}
+
+sub pis ($$) {
+    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/03_stream_unpack.t

@@ -0,0 +1,32 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+
+no warnings 'uninitialized'; # i need this. i need this.
+
+my $up = Data::MessagePack::Unpacker->new;
+sub unpackit {
+    my $v = $_[0];
+    $v =~ s/ //g;
+    $v = pack 'H*', $v;
+    $up->reset;
+    my $ret = $up->execute($v, 0);
+    if ($ret != length($v)) {
+        fail "extra bytes";
+    }
+    return $up->data;
+}
+
+sub pis ($$) {
+    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2) + 1;
+
+isa_ok $up, 'Data::MessagePack::Unpacker';
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/04_invert.t

@@ -0,0 +1,24 @@
+use Test::More;
+use Data::MessagePack;
+use t::Util;
+no warnings 'uninitialized'; # i need this. i need this.
+
+sub invert {
+    return Data::MessagePack->unpack(
+        Data::MessagePack->pack($_[0]),
+    );
+}
+
+sub pis ($) {
+    is_deeply invert($_[0]), $_[0], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    $i++;
+    pis $dat[$i++];
+}
+

perl/t/Util.pm

@@ -0,0 +1,20 @@
+package t::Util;
+use strict;
+use warnings;
+
+sub import {
+    my $pkg = caller(0);
+
+    strict->import;
+    warnings->import;
+
+    no strict 'refs';
+    *{"$pkg\::true"} = sub () {
+        Data::MessagePack::true()
+    };
+    *{"$pkg\::false"} = sub () {
+        Data::MessagePack::false()
+    };
+}
+
+1;

perl/t/data.pl

@@ -0,0 +1,15 @@
+no warnings 'uninitialized'; # i need this. i need this.
+(
+    '93 c0 c2 c3' => [undef, false, true],
+    '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"],
+    '92 90 91 91 c0', [[], [[undef]]],
+    '93 c0 c2 c3', [undef, false, true],
+    'ce 80 00 00 00', 2147483648,
+    '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
+    '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]],
+    '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]],
+    '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"],
+    '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1],
+    '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}},
+    '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}],
+)

perl/unpack.c

@@ -0,0 +1,290 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#define NEED_newRV_noinc
+#define NEED_sv_2pv_flags
+#include "ppport.h"
+
+#ifdef __cplusplus
+};
+#endif
+
+typedef struct {
+    int finished;
+    SV* source;
+} unpack_user;
+
+#include "msgpack/unpack_define.h"
+
+#define msgpack_unpack_struct(name) \
+    struct template ## name
+
+#define msgpack_unpack_func(ret, name) \
+    ret template ## name
+
+#define msgpack_unpack_callback(name) \
+    template_callback ## name
+
+#define msgpack_unpack_object SV*
+
+#define msgpack_unpack_user unpack_user
+
+struct template_context;
+typedef struct template_context msgpack_unpack_t;
+
+static void template_init(msgpack_unpack_t* u);
+
+static SV* template_data(msgpack_unpack_t* u);
+
+static int template_execute(msgpack_unpack_t* u,
+    const char* data, size_t len, size_t* off);
+
+static inline SV* template_callback_root(unpack_user* u)
+{ return &PL_sv_undef; }
+
+static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o)
+{ *o = newSViv(d); return 0; }
+
+static inline int template_callback_float(unpack_user* u, float d, SV** o)
+{ *o = newSVnv(d); return 0; }
+
+static inline int template_callback_double(unpack_user* u, double d, SV** o)
+{ *o = newSVnv(d); return 0; }
+
+static inline int template_callback_nil(unpack_user* u, SV** o)
+{ *o = &PL_sv_undef; return 0; }
+
+static inline int template_callback_true(unpack_user* u, SV** o)
+{ *o = &PL_sv_yes; return 0; }
+
+static inline int template_callback_false(unpack_user* u, SV** o)
+{ *o = &PL_sv_no; return 0;}
+
+static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o)
+{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; }
+
+static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o)
+{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
+
+static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o)
+{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; }
+
+static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
+{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
+
+static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
+{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; }
+
+#define UNPACKER(from, name) \
+	msgpack_unpack_t *name; \
+    name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \
+	if(name == NULL) { \
+		Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
+	}
+
+#include "msgpack/unpack_template.h"
+
+SV* _msgpack_unpack(SV* data, int limit) {
+    msgpack_unpack_t mp;
+    unpack_user u = {0, &PL_sv_undef};
+	int ret;
+	size_t from = 0;
+    STRLEN dlen;
+    const char * dptr = SvPV_const(data, dlen);
+
+	template_init(&mp);
+    mp.user = u;
+
+	mp.user.source = data;
+	ret = template_execute(&mp, dptr, (size_t)dlen, &from);
+	mp.user.source = &PL_sv_undef;
+
+	if(ret < 0) {
+        Perl_croak(aTHX_ "parse error.");
+	} else if(ret == 0) {
+        Perl_croak(aTHX_ "insufficient bytes.");
+	} else {
+		if(from < dlen) {
+            Perl_croak(aTHX_ "extra bytes.");
+		}
+        return template_data(&mp);
+	}
+}
+
+XS(xs_unpack_limit) {
+    dXSARGS;
+
+    if (items != 3) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)");
+    }
+
+    {
+        int limit = SvIV(ST(2));
+        ST(0) = _msgpack_unpack(ST(1), limit);
+    }
+    XSRETURN(1);
+}
+
+
+XS(xs_unpack) {
+    dXSARGS;
+    msgpack_unpack_t mp;
+
+    if (items != 2) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')");
+    }
+
+    {
+        ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1)));
+    }
+
+    XSRETURN(1);
+}
+
+/* ------------------------------ stream -- */
+
+static void _reset(SV* self) {
+	UNPACKER(self, mp);
+	template_init(mp);
+	unpack_user u = {0, &PL_sv_undef};
+	mp->user = u;
+}
+
+XS(xs_unpacker_new) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()");
+    }
+
+    SV* self = sv_newmortal();
+	msgpack_unpack_t *mp;
+
+    Newx(mp, 1, msgpack_unpack_t);
+
+    sv_setref_pv(self, "Data::MessagePack::Unpacker", mp);
+    _reset(self);
+
+    ST(0) = self;
+    XSRETURN(1);
+}
+
+static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
+    UNPACKER(self, mp);
+
+    size_t from = off;
+	const char* dptr = SvPV_nolen_const(data);
+	long dlen = limit;
+	int ret;
+
+	if(from >= dlen) {
+        Perl_croak(aTHX_ "offset is bigger than data buffer size.");
+	}
+
+	mp->user.source = data;
+	ret = template_execute(mp, dptr, (size_t)dlen, &from);
+	mp->user.source = &PL_sv_undef;
+
+	if(ret < 0) {
+		Perl_croak(aTHX_ "parse error.");
+	} else if(ret > 0) {
+		mp->user.finished = 1;
+		return newSVuv(from);
+	} else {
+		mp->user.finished = 0;
+		return newSVuv(from);
+	}
+}
+
+XS(xs_unpacker_execute) {
+    dXSARGS;
+    if (items != 3) {
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)");
+    }
+
+    {
+        SV* self = ST(0);
+        SV* data = ST(1);
+        IV  off  = SvIV(ST(2));
+
+        ST(0) = _execute_impl(self, data, off, sv_len(data));
+    }
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_execute_limit) {
+    dXSARGS;
+    if (items != 4) {
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)");
+    }
+
+    SV* self = ST(0);
+    SV* data = ST(1);
+    IV off   = SvIV(ST(2));
+    IV limit = SvIV(ST(3));
+
+    ST(0) = _execute_impl(self, data, off, limit);
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_is_finished) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->is_finished()");
+    }
+
+	UNPACKER(ST(0), mp);
+    ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no;
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_data) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->data()");
+    }
+
+	UNPACKER(ST(0), mp);
+	ST(0) = template_data(mp);
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_reset) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->reset()");
+    }
+
+    _reset(ST(0));
+
+    XSRETURN(0);
+}
+

perl/xt/99_pod.t

@@ -0,0 +1,4 @@
+use Test::More;
+eval "use Test::Pod 1.00";
+plan skip_all => "Test::Pod 1.00 required for testing POD" if $@;
+all_pod_files_ok();

ruby/gem/Rakefile

@@ -17,7 +17,7 @@ DESCRIPTION       = "Binary-based efficient data interchange format."
 RUBYFORGE_PROJECT = "msgpack"
 HOMEPATH          = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
 BIN_FILES         = %w(  )
-VERS              = "0.3.0"
+VERS              = "0.3.1"
 
 #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
 REV = nil

ruby/msgpack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.platform = Gem::Platform::CURRENT
   s.name = "msgpack"
-  s.version = "0.3.0"
+  s.version = "0.3.1"
   s.summary = "MessagePack"
   s.author = "FURUHASHI Sadayuki"
   s.email = "frsyuki@users.sourceforge.jp"

ruby/test_case.rb

@@ -113,6 +113,80 @@ class MessagePackTestFormat < Test::Unit::TestCase
 		#check_array 5, (1<<32)-1  # memory error
 	end
 
+	it "nil" do
+		match nil, "\xc0"
+	end
+
+	it "false" do
+		match false, "\xc2"
+	end
+
+	it "true" do
+		match true, "\xc3"
+	end
+
+	it "0" do
+		match 0, "\x00"
+	end
+
+	it "127" do
+		match 127, "\x7f"
+	end
+
+	it "128" do
+		match 128, "\xcc\x80"
+	end
+
+	it "256" do
+		match 256, "\xcd\x01\x00"
+	end
+
+	it "-1" do
+		match -1, "\xff"
+	end
+
+	it "-33" do
+		match -33, "\xd0\xdf"
+	end
+
+	it "-129" do
+		match -129, "\xd1\xff\x7f"
+	end
+
+	it "{1=>1}" do
+		match ({1=>1}), "\x81\x01\x01"
+	end
+
+	it "1.0" do
+		match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"
+	end
+
+	it "[]" do
+		match [], "\x90"
+	end
+
+	it "[0, 1, ..., 14]" do
+		match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"
+	end
+
+	it "[0, 1, ..., 15]" do
+		match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+	end
+
+	it "{}" do
+		match ({}), "\x80"
+	end
+
+	it "{0=>0, 1=>1, ..., 14=>14}" do
+		a = (0..14).to_a;
+		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
+	end
+
+	it "{0=>0, 1=>1, ..., 15=>15}" do
+		a = (0..15).to_a;
+		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
+	end
+
 #	it "fixmap" do
 #		check_map 1, 0
 #		check_map 1, (1<<4)-1
@@ -143,8 +217,9 @@ class MessagePackTestFormat < Test::Unit::TestCase
 		check num+overhead, Array.new(num)
 	end
 
-	def check_map(overhead, num)
-		# FIXME
+	def match(obj, buf)
+		assert_equal(obj.to_msgpack, buf)
+		assert_equal(MessagePack::unpack(buf), obj)
 	end
 end