c++: fix object >> std::string

diff --git a/perl/Changes b/perl/Changes
index 4e7f1d7..c6e370c 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.08
+
+    - fixed PVNV issue...
+
 0.07

     - do not use switch (SvTYPE(val)).

Makefile.am

@@ -11,20 +11,3 @@ nobase_include_HEADERS = \
 		msgpack/unpack_define.h \
 		msgpack/unpack_template.h
 
-EXTRA_DIST = \
-	ruby/bench.rb \
-	ruby/extconf.rb \
-	ruby/gem/README \
-	ruby/gem/Rakefile \
-	ruby/gem/test/test_helper.rb \
-	ruby/gengem.sh \
-	ruby/msgpack.gemspec \
-	ruby/pack.c \
-	ruby/pack.h \
-	ruby/rbinit.c \
-	ruby/test_case.rb \
-	ruby/test_format.rb \
-	ruby/test_pack.rb \
-	ruby/unpack.c \
-	ruby/unpack.h
-

README

@@ -8,7 +8,7 @@ Binary-based efficient data interchange format.
   MessagePack is only tested on Linux and Mac OS X, but it may run on other
   UNIX-like platforms.
 
-  Following programs is required to build:
+  Following programs are required to build:
     - gcc >= 4.1 with C++ support
     - ruby >= 1.8   (ruby is used as a preprocessor)
 

c/Makefile.am

@@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la
 libmsgpackc_la_SOURCES = \
 		unpack.c \
 		object.c \
+		vrefbuffer.c \
 		zone.c
 
 nobase_include_HEADERS = \
 		msgpack.h \
 		msgpack/sbuffer.h \
+		msgpack/vrefbuffer.h \
 		msgpack/pack.h \
 		msgpack/unpack.h \
 		msgpack/object.h \

c/msgpack.h

@@ -20,3 +20,4 @@
 #include "msgpack/pack.h"
 #include "msgpack/unpack.h"
 #include "msgpack/sbuffer.h"
+#include "msgpack/vrefbuffer.h"

c/vrefbuffer.c

@@ -0,0 +1,135 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#include "msgpack/vrefbuffer.h"
+#include <stdlib.h>
+#include <string.h>
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size)
+{
+	if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) {
+		chunk_size = 72;
+	} else {
+		chunk_size -= sizeof(msgpack_vrefbuffer_chunk);
+	}
+
+	vbuf->chunk_size = chunk_size;
+	vbuf->ref_size = ref_size;
+
+	// glibcは72バイト以下のmallocが高速
+	size_t nfirst = (sizeof(struct iovec) < 72/2) ?
+			72 / sizeof(struct iovec) : 8;
+
+	struct iovec* array = (struct iovec*)malloc(
+			sizeof(struct iovec) * nfirst);
+	if(array == NULL) {
+		return false;
+	}
+
+	vbuf->tail  = array;
+	vbuf->end   = array + nfirst;
+	vbuf->array = array;
+
+	vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc(
+			chunk_size + sizeof(msgpack_vrefbuffer_chunk));
+	if(vbuf->chunk == NULL) {
+		free(array);
+		return false;
+	}
+
+	vbuf->chunk->next = NULL;
+	vbuf->chunk->free = chunk_size;
+
+	return true;
+}
+
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf)
+{
+	msgpack_vrefbuffer_chunk* c = vbuf->chunk;
+	while(true) {
+		msgpack_vrefbuffer_chunk* n = c->next;
+		free(c);
+		if(n) {
+			c = n;
+		} else {
+			break;
+		}
+	}
+	free(vbuf->array);
+}
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	if(vbuf->tail == vbuf->end) {
+		const size_t nused = vbuf->end - vbuf->array;
+		const size_t nnext = nused * 2;
+
+		struct iovec* nvec = (struct iovec*)realloc(
+				vbuf->array, sizeof(struct iovec)*nnext);
+		if(nvec == NULL) {
+			return -1;
+		}
+
+		vbuf->array = nvec;
+		vbuf->end   = nvec + nnext;
+		vbuf->tail  = nvec + nused;
+	}
+
+	vbuf->tail->iov_base = (char*)buf;
+	vbuf->tail->iov_len  = len;
+	++vbuf->tail;
+
+	return 0;
+}
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer_chunk* chunk = vbuf->chunk;
+	size_t cur_size = vbuf->chunk_size;
+
+	if(chunk->free < len) {
+		cur_size = (cur_size > len) ? cur_size : len;
+
+		chunk = (msgpack_vrefbuffer_chunk*)malloc(
+				cur_size + sizeof(msgpack_vrefbuffer_chunk));
+		if(chunk == NULL) {
+			return -1;
+		}
+
+		chunk->free = cur_size;
+		chunk->next = vbuf->chunk;
+		vbuf->chunk = chunk;
+	}
+
+	char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk)
+		+ (cur_size - chunk->free);
+
+	memcpy(m, buf, len);
+	chunk->free -= len;
+
+	if(vbuf->tail != vbuf->array && m ==
+			(const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) {
+		(vbuf->tail-1)->iov_len += len;
+		return 0;
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, m, len);
+	}
+}
+

c/vrefbuffer.h

@@ -0,0 +1,97 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_VREFBUFFER_H__
+#define MSGPACK_VREFBUFFER_H__
+
+#include "msgpack/zone.h"
+#include <sys/uio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_VREFBUFFER_REF_SIZE
+#define MSGPACK_VREFBUFFER_REF_SIZE 32
+#endif
+
+#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
+#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048
+#endif
+
+typedef struct msgpack_vrefbuffer_chunk {
+	size_t free;
+	struct msgpack_vrefbuffer_chunk* next;
+	/* data ... */
+} msgpack_vrefbuffer_chunk;
+
+typedef struct msgpack_vrefbuffer {
+	size_t chunk_size;
+	size_t ref_size;
+
+	struct iovec* tail;
+	struct iovec* end;
+	struct iovec* array;
+
+	msgpack_vrefbuffer_chunk* chunk;
+} msgpack_vrefbuffer;
+
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size);
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
+
+static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len);
+
+static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref);
+static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref);
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+
+int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data;
+
+	if(len < vbuf->ref_size) {
+		return msgpack_vrefbuffer_append_copy(vbuf, buf, len);
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, buf, len);
+	}
+}
+
+const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref)
+{
+	return vref->array;
+}
+
+size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref)
+{
+	return vref->tail - vref->array;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/vrefbuffer.h */
+

c/zone.c

@@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s
 
 	msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc(
 			sizeof(msgpack_zone_chunk) * nfirst);
-	if(!array) {
+	if(array == NULL) {
 		return false;
 	}
 

c/zone.h

@@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size);
 void msgpack_zone_free(msgpack_zone* zone);
 
 static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size);
+static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size);
 
 static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone,
 		void (*func)(void* data), void* data);
@@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone);
 
 void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size);
 
-void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size)
 {
-	size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1);
-
 	msgpack_zone_chunk* chunk = zone->chunk_array.tail;
 
 	if(chunk->free < size) {
@@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
 	return ptr;
 }
 
+void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+{
+	return msgpack_zone_malloc_no_align(zone,
+			((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1));
+}
+
 
 bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone,
 		void (*func)(void* data), void* data);

configure.in

@@ -1,6 +1,6 @@
 AC_INIT(msgpack/unpack_template.h)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.3.1)
+AM_INIT_AUTOMAKE(msgpack, 0.3.6)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

cpp/Makefile.am

@@ -6,37 +6,50 @@ libmsgpack_la_SOURCES = \
 nobase_include_HEADERS = \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
+		msgpack/vrefbuffer.hpp \
 		msgpack/pack.hpp \
 		msgpack/unpack.hpp \
 		msgpack/object.hpp \
 		msgpack/zone.hpp \
 		msgpack/type.hpp \
-		msgpack/type/array.hpp \
-		msgpack/type/boolean.hpp \
+		msgpack/type/bool.hpp \
 		msgpack/type/float.hpp \
-		msgpack/type/integer.hpp \
+		msgpack/type/int.hpp \
+		msgpack/type/list.hpp \
+		msgpack/type/deque.hpp \
 		msgpack/type/map.hpp \
 		msgpack/type/nil.hpp \
+		msgpack/type/pair.hpp \
 		msgpack/type/raw.hpp \
-		msgpack/type/tuple.hpp
+		msgpack/type/set.hpp \
+		msgpack/type/string.hpp \
+		msgpack/type/vector.hpp \
+		msgpack/type/tuple.hpp \
+		msgpack/type/define.hpp
 
 # FIXME
-object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp
+object.lo: msgpack/type/tuple.hpp msgpack/type/define.hpp msgpack/zone.hpp
 
 msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb
 	$(ERB) $< > $@.tmp
 	mv $@.tmp $@
 
+msgpack/type/define.hpp: msgpack/type/define.hpp.erb
+	$(ERB) $< > $@.tmp
+	mv $@.tmp $@
+
 msgpack/zone.hpp: msgpack/zone.hpp.erb
 	$(ERB) $< > $@.tmp
 	mv $@.tmp $@
 
 MOSTLYCLEANFILES = \
 		msgpack/type/tuple.hpp \
+		msgpack/type/define.hpp \
 		msgpack/zone.hpp
 
 EXTRA_DIST = \
 		msgpack/type/tuple.hpp.erb \
+		msgpack/type/define.hpp.erb \
 		msgpack/zone.hpp.erb
 
 libmsgpack_la_LIBADD = -L../c -lmsgpackc

cpp/msgpack.hpp

@@ -20,4 +20,5 @@
 #include "msgpack/pack.hpp"
 #include "msgpack/unpack.hpp"
 #include "msgpack/sbuffer.hpp"
+#include "msgpack/vrefbuffer.hpp"
 #include "msgpack.h"

cpp/object.hpp

@@ -80,13 +80,13 @@ struct object {
 	type::object_type type;
 	union_type via;
 
-	bool is_nil() { return type == type::NIL; }
+	bool is_nil() const { return type == type::NIL; }
 
 	template <typename T>
-	T as();
+	T as() const;
 
 	template <typename T>
-	void convert(T* v);
+	void convert(T* v) const;
 
 	object();
 	object(msgpack_object obj);
@@ -96,7 +96,7 @@ private:
 	struct implicit_type;
 
 public:
-	implicit_type convert();
+	implicit_type convert() const;
 };
 
 struct object_kv {
@@ -129,6 +129,7 @@ private:
 };
 
 
+// obsolete
 template <typename Type>
 class define : public Type {
 public:
@@ -201,25 +202,25 @@ inline object::operator msgpack_object()
 }
 
 
-inline object::implicit_type object::convert()
+inline object::implicit_type object::convert() const
 {
 	return implicit_type(*this);
 }
 
 template <typename T>
-inline T object::as()
+inline void object::convert(T* v) const
+{
+	*this >> *v;
+}
+
+template <typename T>
+inline T object::as() const
 {
 	T v;
 	convert(&v);
 	return v;
 }
 
-template <typename T>
-inline void object::convert(T* v)
-{
-	*this >> *v;
-}
-
 
 // obsolete
 template <typename T>

cpp/type.hpp

@@ -1,9 +1,15 @@
-#include "msgpack/type/array.hpp"
-#include "msgpack/type/boolean.hpp"
+#include "msgpack/type/bool.hpp"
 #include "msgpack/type/float.hpp"
-#include "msgpack/type/integer.hpp"
+#include "msgpack/type/int.hpp"
+#include "msgpack/type/list.hpp"
+#include "msgpack/type/deque.hpp"
 #include "msgpack/type/map.hpp"
 #include "msgpack/type/nil.hpp"
+#include "msgpack/type/pair.hpp"
 #include "msgpack/type/raw.hpp"
+#include "msgpack/type/set.hpp"
+#include "msgpack/type/string.hpp"
+#include "msgpack/type/vector.hpp"
 #include "msgpack/type/tuple.hpp"
+#include "msgpack/type/define.hpp"
 

cpp/type/bool.hpp

@@ -15,8 +15,8 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 //
-#ifndef MSGPACK_TYPE_BOOLEAN_HPP__
-#define MSGPACK_TYPE_BOOLEAN_HPP__
+#ifndef MSGPACK_TYPE_BOOL_HPP__
+#define MSGPACK_TYPE_BOOL_HPP__
 
 #include "msgpack/object.hpp"
 #include <vector>
@@ -31,7 +31,6 @@ inline bool& operator>> (object o, bool& v)
 	return v;
 }
 
-
 template <typename Stream>
 inline packer<Stream>& operator<< (packer<Stream>& o, const bool& v)
 {

cpp/type/define.hpp.erb

@@ -0,0 +1,98 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_DEFINE_HPP__
+#define MSGPACK_TYPE_DEFINE_HPP__
+
+#define MSGPACK_DEFINE(...) \
+	template <typename Packer> \
+	void msgpack_pack(Packer& pk) const \
+	{ \
+		msgpack::type::make_define(__VA_ARGS__).msgpack_pack(pk); \
+	} \
+	void msgpack_unpack(msgpack::object o) \
+	{ \
+		msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \
+	}
+
+namespace msgpack {
+namespace type {
+
+
+<% GENERATION_LIMIT = 15 %>
+template <typename A0 = void<%1.upto(GENERATION_LIMIT+1) {|i|%>, typename A<%=i%> = void<%}%>>
+struct define;
+
+
+template <>
+struct define<> {
+	typedef define<> value_type;
+	typedef tuple<> tuple_type;
+	template <typename Packer>
+	void msgpack_pack(Packer& pk) const
+	{
+		pk.pack_array(1);
+	}
+	void msgpack_unpack(msgpack::object o)
+	{
+		if(o.type != type::ARRAY) { throw type_error(); }
+	}
+};
+<%0.upto(GENERATION_LIMIT) {|i|%>
+template <typename A0<%1.upto(i) {|j|%>, typename A<%=j%><%}%>>
+struct define<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> {
+	typedef define<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> value_type;
+	typedef tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> tuple_type;
+	define(A0& _a0<%1.upto(i) {|j|%>, A<%=j%>& _a<%=j%><%}%>) :
+		a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {}
+	template <typename Packer>
+	void msgpack_pack(Packer& pk) const
+	{
+		pk.pack_array(<%=i+1%>);
+		<%0.upto(i) {|j|%>
+		pk.pack(a<%=j%>);<%}%>
+	}
+	void msgpack_unpack(msgpack::object o)
+	{
+		if(o.type != type::ARRAY) { throw type_error(); }
+		const size_t size = o.via.array.size;
+		<%0.upto(i) {|j|%>
+		if(size <= <%=j%>) { return; } o.via.array.ptr[<%=j%>].convert(&a<%=j%>);<%}%>
+	}
+	<%0.upto(i) {|j|%>
+	A<%=j%>& a<%=j%>;<%}%>
+};
+<%}%>
+
+inline define<> make_define()
+{
+	return define<>();
+}
+<%0.upto(GENERATION_LIMIT) {|i|%>
+template <typename A0<%1.upto(i) {|j|%>, typename A<%=j%><%}%>>
+define<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, A<%=j%>& a<%=j%><%}%>)
+{
+	return define<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>);
+}
+<%}%>
+
+}  // namespace type
+}  // namespace msgpack
+
+
+#endif /* msgpack/type/define.hpp */
+

cpp/type/deque.hpp

@@ -0,0 +1,56 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_DEQUE_HPP__
+#define MSGPACK_TYPE_DEQUE_HPP__
+
+#include "msgpack/object.hpp"
+#include <deque>
+
+namespace msgpack {
+
+
+template <typename T>
+inline std::deque<T>& operator>> (object o, std::deque<T>& v)
+{
+	if(o.type != type::ARRAY) { throw type_error(); }
+	v.resize(o.via.array.size);
+	object* p = o.via.array.ptr;
+	object* const pend = o.via.array.ptr + o.via.array.size;
+	typename std::deque<T>::iterator it = v.begin();
+	for(; p < pend; ++p, ++it) {
+		p->convert(&*it);
+	}
+	return v;
+}
+
+template <typename Stream, typename T>
+inline packer<Stream>& operator<< (packer<Stream>& o, const std::deque<T>& v)
+{
+	o.pack_array(v.size());
+	for(typename std::deque<T>::const_iterator it(v.begin()), it_end(v.end());
+			it != it_end; ++it) {
+		o.pack(*it);
+	}
+	return o;
+}
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/deque.hpp */
+

cpp/type/float.hpp

@@ -34,7 +34,6 @@ inline float& operator>> (object o, float& v)
 	return v;
 }
 
-
 template <typename Stream>
 inline packer<Stream>& operator<< (packer<Stream>& o, const float& v)
 {
@@ -50,7 +49,6 @@ inline double& operator>> (object o, double& v)
 	return v;
 }
 
-
 template <typename Stream>
 inline packer<Stream>& operator<< (packer<Stream>& o, const double& v)
 {

cpp/type/int.hpp

@@ -15,8 +15,8 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 //
-#ifndef MSGPACK_TYPE_INTEGER_HPP__
-#define MSGPACK_TYPE_INTEGER_HPP__
+#ifndef MSGPACK_TYPE_INT_HPP__
+#define MSGPACK_TYPE_INT_HPP__
 
 #include "msgpack/object.hpp"
 #include <limits>
@@ -143,5 +143,5 @@ inline packer<Stream>& operator<< (packer<Stream>& o, const unsigned long long&
 
 }  // namespace msgpack
 
-#endif /* msgpack/type/integer.hpp */
+#endif /* msgpack/type/int.hpp */
 

cpp/type/list.hpp

@@ -0,0 +1,56 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_LIST_HPP__
+#define MSGPACK_TYPE_LIST_HPP__
+
+#include "msgpack/object.hpp"
+#include <list>
+
+namespace msgpack {
+
+
+template <typename T>
+inline std::list<T>& operator>> (object o, std::list<T>& v)
+{
+	if(o.type != type::ARRAY) { throw type_error(); }
+	v.resize(o.via.array.size);
+	object* p = o.via.array.ptr;
+	object* const pend = o.via.array.ptr + o.via.array.size;
+	typename std::list<T>::iterator it = v.begin();
+	for(; p < pend; ++p, ++it) {
+		p->convert(&*it);
+	}
+	return v;
+}
+
+template <typename Stream, typename T>
+inline packer<Stream>& operator<< (packer<Stream>& o, const std::list<T>& v)
+{
+	o.pack_array(v.size());
+	for(typename std::list<T>::const_iterator it(v.begin()), it_end(v.end());
+			it != it_end; ++it) {
+		o.pack(*it);
+	}
+	return o;
+}
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/list.hpp */
+

cpp/type/map.hpp

@@ -80,13 +80,13 @@ inline std::map<K, V> operator>> (object o, std::map<K, V>& v)
 	for(; p != pend; ++p) {
 		K key;
 		p->key.convert(&key);
-		typename std::map<K,V>::iterator it(v.find(key));
-		if(it != v.end()) {
+		typename std::map<K,V>::iterator it(v.lower_bound(key));
+		if(it != v.end() && !(key < it->first)) {
+			p->val.convert(&it->second);
+		} else {
 			V val;
 			p->val.convert(&val);
-			it->insert( std::pair<K,V>(key, val) );
-		} else {
-			p->val.convert(&it->second);
+			v.insert(it, std::pair<K,V>(key, val));
 		}
 	}
 	return v;

cpp/type/pair.hpp

@@ -0,0 +1,50 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_PAIR_HPP__
+#define MSGPACK_TYPE_PAIR_HPP__
+
+#include "msgpack/object.hpp"
+#include <utility>
+
+namespace msgpack {
+
+
+template <typename T1, typename T2>
+inline std::pair<T1, T2>& operator>> (object o, std::pair<T1, T2>& v)
+{
+	if(o.type != type::ARRAY) { throw type_error(); }
+	if(o.via.array.size != 2) { throw type_error(); }
+	o.via.array.ptr[0].convert(&v.first);
+	o.via.array.ptr[1].convert(&v.second);
+	return v;
+}
+
+template <typename Stream, typename T1, typename T2>
+inline packer<Stream>& operator<< (packer<Stream>& o, const std::pair<T1, T2>& v)
+{
+	o.pack_array(2);
+	o.pack(v.first);
+	o.pack(v.second);
+	return o;
+}
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/pair.hpp */
+

cpp/type/raw.hpp

@@ -69,16 +69,6 @@ inline type::raw_ref& operator>> (object o, type::raw_ref& v)
 	return v;
 }
 
-
-inline std::string& operator>> (object o, std::string& v)
-{
-	type::raw_ref r;
-	o >> r;
-	v.assign(r.ptr, r.size);
-	return v;
-}
-
-
 template <typename Stream>
 inline packer<Stream>& operator<< (packer<Stream>& o, const type::raw_ref& v)
 {
@@ -88,15 +78,6 @@ inline packer<Stream>& operator<< (packer<Stream>& o, const type::raw_ref& v)
 }
 
 
-template <typename Stream>
-inline packer<Stream>& operator<< (packer<Stream>& o, const std::string& v)
-{
-	o.pack_raw(v.size());
-	o.pack_raw_body(v.data(), v.size());
-	return o;
-}
-
-
 }  // namespace msgpack
 
 #endif /* msgpack/type/raw.hpp */

cpp/type/set.hpp

@@ -0,0 +1,55 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_SET_HPP__
+#define MSGPACK_TYPE_SET_HPP__
+
+#include "msgpack/object.hpp"
+#include <set>
+
+namespace msgpack {
+
+
+template <typename T>
+inline std::set<T>& operator>> (object o, std::set<T>& v)
+{
+	if(o.type != type::ARRAY) { throw type_error(); }
+	object* p = o.via.array.ptr + o.via.array.size;
+	object* const pbegin = o.via.array.ptr;
+	while(p > pbegin) {
+		--p;
+		v.insert(p->as<T>());
+	}
+	return v;
+}
+
+template <typename Stream, typename T>
+inline packer<Stream>& operator<< (packer<Stream>& o, const std::set<T>& v)
+{
+	o.pack_array(v.size());
+	for(typename std::set<T>::const_iterator it(v.begin()), it_end(v.end());
+			it != it_end; ++it) {
+		o.pack(*it);
+	}
+	return o;
+}
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/set.hpp */
+

cpp/type/string.hpp

@@ -0,0 +1,46 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_STRING_HPP__
+#define MSGPACK_TYPE_STRING_HPP__
+
+#include "msgpack/object.hpp"
+#include <string>
+
+namespace msgpack {
+
+
+inline std::string& operator>> (object o, std::string& v)
+{
+	if(o.type != type::RAW) { throw type_error(); }
+	v.assign(o.via.raw.ptr, o.via.raw.size);
+	return v;
+}
+
+template <typename Stream>
+inline packer<Stream>& operator<< (packer<Stream>& o, const std::string& v)
+{
+	o.pack_raw(v.size());
+	o.pack_raw_body(v.data(), v.size());
+	return o;
+}
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/string.hpp */
+

cpp/type/tuple.hpp.erb

@@ -64,7 +64,6 @@ struct tuple_type<const T&> {
 	typedef const T& transparent_reference;
 };
 
-
 <%0.upto(GENERATION_LIMIT) {|i|%>
 <%0.upto(i) {|j|%>
 template <typename A0<%1.upto(i) {|k|%>, typename A<%=k%><%}%>>
@@ -78,7 +77,6 @@ private:
 <%}%>
 <%}%>
 
-
 <%0.upto(GENERATION_LIMIT) {|i|%>
 <%0.upto(i) {|j|%>
 template <typename A0<%1.upto(i) {|k|%>, typename A<%=k%><%}%>>
@@ -91,18 +89,9 @@ private:
 <%}%>
 <%}%>
 
-
-<%0.upto(GENERATION_LIMIT) {|i|%>
-template <typename A0<%1.upto(i) {|j|%>, typename A<%=j%><%}%>>
-tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> make_tuple(typename tuple_type<A0>::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type<A<%=j%>>::transparent_reference a<%=j%><%}%>)
-{
-	return tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>);
-}
-<%}%>
-
-
 template <>
 struct tuple<> {
+	typedef tuple<> value_type;
 };
 <%0.upto(GENERATION_LIMIT) {|i|%>
 template <typename A0<%1.upto(i) {|j|%>, typename A<%=j%><%}%>>
@@ -121,6 +110,18 @@ struct tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> {
 };
 <%}%>
 
+inline tuple<> make_tuple()
+{
+	return tuple<>();
+}
+<%0.upto(GENERATION_LIMIT) {|i|%>
+template <typename A0<%1.upto(i) {|j|%>, typename A<%=j%><%}%>>
+tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>> make_tuple(typename tuple_type<A0>::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type<A<%=j%>>::transparent_reference a<%=j%><%}%>)
+{
+	return tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>);
+}
+<%}%>
+
 }  // namespace type
 
 
@@ -143,7 +144,6 @@ type::tuple<A0<%1.upto(i) {|j|%>, A<%=j%><%}%>>& operator>> (
 }
 <%}%>
 
-
 template <typename Stream>
 const packer<Stream>& operator<< (
 		packer<Stream>& o,
@@ -163,7 +163,6 @@ const packer<Stream>& operator<< (
 }
 <%}%>
 
-
 }  // namespace msgpack
 
 #endif /* msgpack/type/tuple.hpp */

cpp/type/vector.hpp

@@ -15,8 +15,8 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 //
-#ifndef MSGPACK_TYPE_ARRAY_HPP__
-#define MSGPACK_TYPE_ARRAY_HPP__
+#ifndef MSGPACK_TYPE_VECTOR_HPP__
+#define MSGPACK_TYPE_VECTOR_HPP__
 
 #include "msgpack/object.hpp"
 #include <vector>
@@ -25,7 +25,7 @@ namespace msgpack {
 
 
 template <typename T>
-inline std::vector<T> operator>> (object o, std::vector<T>& v)
+inline std::vector<T>& operator>> (object o, std::vector<T>& v)
 {
 	if(o.type != type::ARRAY) { throw type_error(); }
 	v.resize(o.via.array.size);
@@ -38,7 +38,6 @@ inline std::vector<T> operator>> (object o, std::vector<T>& v)
 	return v;
 }
 
-
 template <typename Stream, typename T>
 inline packer<Stream>& operator<< (packer<Stream>& o, const std::vector<T>& v)
 {
@@ -53,5 +52,5 @@ inline packer<Stream>& operator<< (packer<Stream>& o, const std::vector<T>& v)
 
 }  // namespace msgpack
 
-#endif /* msgpack/type/array.hpp */
+#endif /* msgpack/type/vector.hpp */
 

cpp/vrefbuffer.hpp

@@ -0,0 +1,85 @@
+//
+// MessagePack for C++ zero-copy buffer implementation
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_VREFBUFFER_HPP__
+#define MSGPACK_VREFBUFFER_HPP__
+
+#include "msgpack/vrefbuffer.h"
+#include <stdexcept>
+
+namespace msgpack {
+
+
+class vrefbuffer : public msgpack_vrefbuffer {
+public:
+	vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE,
+			size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE)
+	{
+		msgpack_vrefbuffer_init(this, ref_size, chunk_size);
+	}
+
+	~vrefbuffer()
+	{
+		msgpack_vrefbuffer_destroy(this);
+	}
+
+public:
+	void write(const char* buf, unsigned int len)
+	{
+		if(len < base::ref_size) {
+			append_copy(buf, len);
+		} else {
+			append_ref(buf, len);
+		}
+	}
+
+	void append_ref(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	void append_copy(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	const struct iovec* vector() const
+	{
+		return msgpack_vrefbuffer_vec(this);
+	}
+
+	size_t vector_size() const
+	{
+		return msgpack_vrefbuffer_veclen(this);
+	}
+
+private:
+	typedef msgpack_vrefbuffer base;
+
+private:
+	vrefbuffer(const vrefbuffer&);
+};
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/vrefbuffer.hpp */
+

cpp/zone.hpp.erb

@@ -34,6 +34,7 @@ public:
 
 public:
 	void* malloc(size_t size);
+	void* malloc_no_align(size_t size);
 
 	void push_finalizer(void (*func)(void*), void* data);
 
@@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size)
 	return ptr;
 }
 
+inline void* zone::malloc_no_align(size_t size)
+{
+	void* ptr = msgpack_zone_malloc_no_align(this, size);
+	if(!ptr) {
+		throw std::bad_alloc();
+	}
+	return ptr;
+}
+
 inline void zone::push_finalizer(void (*func)(void*), void* data)
 {
 	if(!msgpack_zone_push_finalizer(this, func, data)) {

example/custom.cc

@@ -0,0 +1,58 @@
+#include <msgpack.hpp>
+#include <string>
+#include <iostream>
+
+class old_class {
+public:
+	old_class() : value("default") { }
+
+	std::string value;
+
+	MSGPACK_DEFINE(value);
+};
+
+class new_class {
+public:
+	new_class() : value("default"), flag(-1) { }
+
+	std::string value;
+	int flag;
+
+	MSGPACK_DEFINE(value, flag);
+};
+
+int main(void)
+{
+	{
+		old_class oc;
+		new_class nc;
+
+		msgpack::sbuffer sbuf;
+		msgpack::pack(sbuf, oc);
+
+		msgpack::zone zone;
+		msgpack::object obj;
+		msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj);
+
+		obj.convert(&nc);
+
+		std::cout << obj << " value=" << nc.value << " flag=" << nc.flag << std::endl;
+	}
+
+	{
+		new_class nc;
+		old_class oc;
+
+		msgpack::sbuffer sbuf;
+		msgpack::pack(sbuf, nc);
+
+		msgpack::zone zone;
+		msgpack::object obj;
+		msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj);
+
+		obj.convert(&oc);
+
+		std::cout << obj << " value=" << oc.value << std::endl;
+	}
+}
+

example/stream.cc

@@ -22,14 +22,14 @@ public:
 		ssize_t count =
 			read(m_sock, m_pac.buffer(), m_pac.buffer_capacity());
 
-		if(count < 0) {
+		if(count <= 0) {
+			if(count == 0) {
+				throw std::runtime_error("connection closed");
+			}
 			if(errno == EAGAIN || errno == EINTR) {
 				return;
-			} else {
-				throw std::runtime_error(strerror(errno));
 			}
-		} else if(count == 0) {
-			throw std::runtime_error("connection closed");
+			throw std::runtime_error(strerror(errno));
 		}
 
 		m_pac.buffer_consumed(count);

perl/.gitignore

@@ -0,0 +1,13 @@
+META.yml
+Makefile
+Makefile.old
+MessagePack.bs
+MessagePack.o
+blib/
+inc/
+msgpack/
+pack.o
+pm_to_blib
+unpack.o
+MANIFEST
+ppport.h

perl/.shipit

@@ -0,0 +1,2 @@
+steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN
+git.tagpattern = perl-%v

perl/Changes

@@ -0,0 +1,37 @@
+0.08
+
+    - fixed PVNV issue...
+
+0.07
+
+    - do not use switch (SvTYPE(val)).
+
+0.06
+
+    - use SvNOK.
+
+0.05
+
+    - change type detection for old perl
+
+0.04
+
+    - check SvROK first(reported by yappo++)
+    - PreferInteger: faster string to integer conversion; support negative value
+      (frsyuki++)
+    - make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv
+      (frsyuki++)
+
+0.03
+
+    - performance tuning for too long string
+    - fixed memory leaks in stream unpacker
+
+0.02
+
+    - added $Data::MessagePack::PreferInteger
+      (requested by yappo++)
+
+0.01
+
+    - initial release to CPAN

perl/MANIFEST.SKIP

@@ -0,0 +1,23 @@
+\bRCS\b
+\bCVS\b
+^MANIFEST\.
+^Makefile$
+~$
+^#
+\.old$
+^blib/
+^pm_to_blib
+^MakeMaker-\d
+\.gz$
+\.cvsignore
+^t/9\d_.*\.t
+^t/perlcritic
+^tools/
+\.svn/
+^[^/]+\.yaml$
+^[^/]+\.pl$
+^\.shipit$
+^\.git/
+\.sw[pon]$
+^\.gitignore$
+ppport.h

perl/Makefile.PL

@@ -0,0 +1,39 @@
+use inc::Module::Install;
+name 'Data-MessagePack';
+all_from 'lib/Data/MessagePack.pm';
+
+perl_version '5.008005';
+license 'perl';
+can_cc or die "This module requires a C compiler";
+
+tests 't/*.t';
+author_tests('xt');
+use_ppport 3.19;
+
+clean_files qw{
+    *.stackdump
+    *.gcov *.gcda *.gcno
+    *.out
+    nytprof
+    cover_db
+};
+
+if ($ENV{DEBUG}) {
+    cc_append_to_ccflags '-g';
+}
+
+# copy modules
+if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) {
+    mkdir 'msgpack' unless -d 'msgpack';
+    require File::Copy;
+    for my $src (<../msgpack/*.h>) {
+        File::Copy::copy($src, 'msgpack/') or die "copy failed: $!";
+    }
+}
+
+auto_set_repository;
+build_requires 'Test::More';
+use_test_base;
+auto_include;
+WriteAll;
+

perl/MessagePack.c

@@ -0,0 +1,43 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#define NEED_newCONSTSUB
+#include "ppport.h"
+#ifdef __cplusplus
+};
+#endif
+
+XS(xs_pack);
+XS(xs_unpack);
+XS(xs_unpacker_new);
+XS(xs_unpacker_execute);
+XS(xs_unpacker_execute_limit);
+XS(xs_unpacker_is_finished);
+XS(xs_unpacker_data);
+XS(xs_unpacker_reset);
+XS(xs_unpacker_destroy);
+
+void boot_Data__MessagePack_pack(void);
+
+XS(boot_Data__MessagePack) {
+    dXSARGS;
+    HV * stash;
+
+    boot_Data__MessagePack_pack();
+
+    newXS("Data::MessagePack::pack", xs_pack, __FILE__);
+    newXS("Data::MessagePack::unpack", xs_unpack, __FILE__);
+    stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE);
+
+    newXS("Data::MessagePack::Unpacker::new",           xs_unpacker_new, __FILE__);
+    newXS("Data::MessagePack::Unpacker::execute",       xs_unpacker_execute, __FILE__);
+    newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__);
+    newXS("Data::MessagePack::Unpacker::is_finished",   xs_unpacker_is_finished, __FILE__);
+    newXS("Data::MessagePack::Unpacker::data",          xs_unpacker_data, __FILE__);
+    newXS("Data::MessagePack::Unpacker::reset",         xs_unpacker_reset, __FILE__);
+    newXS("Data::MessagePack::Unpacker::DESTROY",       xs_unpacker_destroy, __FILE__);
+}
+

perl/benchmark/deserialize.pl

@@ -0,0 +1,20 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use JSON::XS;
+use Benchmark ':all';
+
+my $a = [0..2**24];
+my $j = JSON::XS::encode_json($a);
+my $m = Data::MessagePack->pack($a);
+
+print "-- deserialize\n";
+print "JSON::XS: $JSON::XS::VERSION\n";
+print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+cmpthese(
+    -1 => {
+        json => sub { JSON::XS::decode_json($j)     },
+        mp   => sub { Data::MessagePack->unpack($m) },
+    }
+);
+

perl/benchmark/serialize.pl

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use JSON::XS;
+use Benchmark ':all';
+
+my $a = [0..2**24];
+
+print "-- serialize\n";
+print "JSON::XS: $JSON::XS::VERSION\n";
+print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+cmpthese(
+    -1 => {
+        json => sub { JSON::XS::encode_json($a)   },
+        mp   => sub { Data::MessagePack->pack($a) },
+    }
+);
+

perl/benchmark/size.pl

@@ -0,0 +1,37 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use Data::MessagePack;
+use Storable;
+use Text::SimpleTable;
+
+my @entries = (
+    '1',
+    '3.14',
+    '{}',
+    '[]',
+    "[('a')x10]",
+    "{('a')x10}",
+    "+{1,+{1,+{}}}",
+    "+[+[+[]]]",
+);
+
+my $table = Text::SimpleTable->new([15, 'src'], [9, 'storable'], [7, 'msgpack']);
+
+for my $src (@entries) {
+    my $e = eval $src;
+    die $@ if $@;
+
+    $table->row(
+        $src,
+        length(Storable::nfreeze(ref $e ? $e : \$e)),
+        length(Data::MessagePack->pack($e)),
+    );
+}
+
+print "perl: $]\n";
+print "Storable: $Storable::VERSION\n";
+print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+print "\n";
+print $table->draw;
+

perl/lib/Data/MessagePack.pm

@@ -0,0 +1,50 @@
+package Data::MessagePack;
+use strict;
+use warnings;
+use XSLoader;
+use 5.008001;
+
+our $VERSION = '0.08';
+our $PreferInteger = 0;
+
+our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
+our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
+sub true  () { $true  }
+sub false () { $false }
+
+XSLoader::load(__PACKAGE__, $VERSION);
+
+1;
+__END__
+
+=head1 NAME
+
+Data::MessagePack - messagepack
+
+=head1 SYNOPSIS
+
+    my $packed = Data::MessagePack->pack($dat);
+    my $unpacked = Data::MessagePack->unpack($dat);
+
+=head1 DESCRIPTION
+
+Data::MessagePack is a binary packer for perl.
+
+=head1 Configuration Variables
+
+=over 4
+
+=item $Data::MessagePack::PreferInteger
+
+Pack the string as int when the value looks like int(EXPERIMENTAL).
+
+=back
+
+=head1 AUTHORS
+
+Tokuhiro Matsuno
+
+=head1 SEE ALSO
+
+L<http://msgpack.sourceforge.jp/>
+

perl/lib/Data/MessagePack/Unpacker.pod

@@ -0,0 +1,52 @@
+=head1 NAME
+
+Data::MessagePack::Unpacker - messagepack streaming deserializer
+
+=head1 SYNOPSIS
+
+    use Data::Dumper;
+    my $up = Data::MessagePack::Unpacker->new;
+    my $ret = $up->execute($v, 0);
+    if ($ret != length($v)) {
+        fail "extra bytes";
+    }
+    return Dumper($up->data);
+
+=head1 DESCRIPTION
+
+This is an streaming deserializer for messagepack.
+
+=head1 METHODS
+
+=over 4
+
+=item my $up = Data::MessagePack::Unpacker->new()
+
+create new stream deserializer
+
+=item $up->execute()
+
+=item $up->execute_limit()
+
+=item $up->is_finished()
+
+is this deserializer finished?
+
+=item $up->data()
+
+returns deserialized object.
+
+=item $up->reset()
+
+reset the stream deserializer, without memory zone.
+
+=back
+
+=head1 AUTHORS
+
+Tokuhiro Matsuno
+
+=head1 SEE ALSO
+
+L<Data::MessagePack>
+

perl/pack.c

@@ -0,0 +1,256 @@
+/*
+ * code is written by tokuhirom.
+ * buffer alocation technique is taken from JSON::XS. thanks to mlehmann.
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+#ifdef __cplusplus
+};
+#endif
+
+#include "msgpack/pack_define.h"
+
+#define msgpack_pack_inline_func(name) \
+    static inline void msgpack_pack ## name
+
+#define msgpack_pack_inline_func_cint(name) \
+    static inline void msgpack_pack ## name
+
+typedef struct {
+    char *cur; /* SvPVX (sv) + current output position */
+    char *end; /* SvEND (sv) */
+    SV *sv;    /* result scalar */
+} enc_t;
+static void need(enc_t *enc, STRLEN len);
+
+#define msgpack_pack_user enc_t*
+
+#define msgpack_pack_append_buffer(enc, buf, len) \
+    need(enc, len); \
+    memcpy(enc->cur, buf, len); \
+    enc->cur += len;
+
+#include "msgpack/pack_template.h"
+
+#define _PACK_WRAPPER(t) msgpack_pack_##t
+#define PACK_WRAPPER(t) _PACK_WRAPPER(t)
+#define INIT_SIZE   32 /* initial scalar size to be allocated */
+
+static void need(enc_t *enc, STRLEN len)
+{
+    if (enc->cur + len >= enc->end) {
+        STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
+        SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
+        enc->cur = SvPVX (enc->sv) + cur;
+        enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1;
+    }
+}
+
+
+static int s_pref_int = 0;
+
+static int pref_int_set(pTHX_ SV* sv, MAGIC* mg) {
+    if (SvTRUE(sv)) {
+        s_pref_int = 1;
+    } else {
+        s_pref_int = 0;
+    }
+    return 0;
+}
+
+MGVTBL pref_int_vtbl = {
+    NULL,
+    pref_int_set,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+#ifdef MGf_LOCAL
+    NULL,
+#endif
+};
+
+void boot_Data__MessagePack_pack(void) {
+    SV* var = get_sv("Data::MessagePack::PreferInteger", 0);
+    sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0);
+    SvSETMAGIC(var);
+}
+
+
+static int try_int(enc_t* enc, const char *p, size_t len) {
+    int negative = 0;
+    const char* pe = p + len;
+    uint64_t num = 0;
+
+    if (len == 0) { return 0; }
+
+    if (*p == '-') {
+        /* length(-0x80000000) == 11 */
+        if (len <= 1 || len > 11) { return 0; }
+        negative = 1;
+        ++p;
+    } else {
+        /* length(0xFFFFFFFF) == 10 */
+        if (len > 10) { return 0; }
+    }
+
+#if '9'=='8'+1 && '8'=='7'+1 && '7'=='6'+1 && '6'=='5'+1 && '5'=='4'+1 \
+               && '4'=='3'+1 && '3'=='2'+1 && '2'=='1'+1 && '1'=='0'+1
+    do {
+        unsigned int c = ((int)*(p++)) - '0';
+        if (c > 9) { return 0; }
+        num = num * 10 + c;
+    } while(p < pe);
+#else
+    do {
+        switch (*(p++)) {
+        case '0': num = num * 10 + 0; break;
+        case '1': num = num * 10 + 1; break;
+        case '2': num = num * 10 + 2; break;
+        case '3': num = num * 10 + 3; break;
+        case '4': num = num * 10 + 4; break;
+        case '5': num = num * 10 + 5; break;
+        case '6': num = num * 10 + 6; break;
+        case '7': num = num * 10 + 7; break;
+        case '8': num = num * 10 + 8; break;
+        case '9': num = num * 10 + 9; break;
+        default: return 0;
+        }
+    } while(p < pe);
+#endif
+
+    if (negative) {
+        if (num > 0x80000000) { return 0; }
+        msgpack_pack_int32(enc, ((int32_t)num) * -1);
+    } else {
+        if (num > 0xFFFFFFFF) { return 0; }
+        msgpack_pack_uint32(enc, (uint32_t)num);
+    }
+
+    return 1;
+}
+
+
+static void _msgpack_pack_rv(enc_t *enc, SV* sv);
+
+static void _msgpack_pack_sv(enc_t *enc, SV* sv) {
+    SvGETMAGIC(sv);
+
+    if (sv==NULL) {
+        msgpack_pack_nil(enc);
+    } else if (SvPOKp(sv)) {
+        STRLEN len;
+        char * csv = SvPV(sv, len);
+
+        if (s_pref_int && try_int(enc, csv, len)) {
+            return;
+        } else {
+            msgpack_pack_raw(enc, len);
+            msgpack_pack_raw_body(enc, csv, len);
+        }
+    } else if (SvNOKp(sv)) {
+        PACK_WRAPPER(NVTYPE)(enc, SvNVX(sv));
+    } else if (SvIOK_UV(sv)) {
+        msgpack_pack_uint32(enc, SvUV(sv));
+    } else if (SvIOKp(sv)) {
+        PACK_WRAPPER(IVTYPE)(enc, SvIV(sv));
+    } else if (SvROK(sv)) {
+        _msgpack_pack_rv(enc, SvRV(sv));
+    } else if (!SvOK(sv)) {
+        msgpack_pack_nil(enc);
+    } else if (isGV(sv)) {
+        Perl_croak(aTHX_ "msgpack cannot pack the GV\n");
+    } else {
+        sv_dump(sv);
+        Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(sv));
+    }
+}
+
+static void _msgpack_pack_rv(enc_t *enc, SV* sv) {
+    svtype svt;
+    SvGETMAGIC(sv);
+    svt = SvTYPE(sv);
+
+    if (SvOBJECT (sv)) {
+        HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache?
+        if (SvSTASH (sv) == stash) {
+            if (SvIV(sv)) {
+                msgpack_pack_true(enc);
+            } else {
+                msgpack_pack_false(enc);
+            }
+        } else {
+            croak ("encountered object '%s', Data::MessagePack doesn't allow the object",
+                           SvPV_nolen(sv_2mortal(newRV_inc(sv))));
+        }
+    } else if (svt == SVt_PVHV) {
+        HV* hval = (HV*)sv;
+        int count = hv_iterinit(hval);
+        HE* he;
+
+        msgpack_pack_map(enc, count);
+
+        while (he = hv_iternext(hval)) {
+            _msgpack_pack_sv(enc, hv_iterkeysv(he));
+            _msgpack_pack_sv(enc, HeVAL(he));
+        }
+    } else if (svt == SVt_PVAV) {
+        AV* ary = (AV*)sv;
+        int len = av_len(ary) + 1;
+        int i;
+        msgpack_pack_array(enc, len);
+        for (i=0; i<len; i++) {
+            SV** svp = av_fetch(ary, i, 0);
+            if (svp) {
+                _msgpack_pack_sv(enc, *svp);
+            } else {
+                msgpack_pack_nil(enc);
+            }
+        }
+    } else if (svt < SVt_PVAV) {
+        STRLEN len = 0;
+        char *pv = svt ? SvPV (sv, len) : 0;
+
+        if (len == 1 && *pv == '1')
+            msgpack_pack_true(enc); 
+        else if (len == 1 && *pv == '0')
+            msgpack_pack_false(enc); 
+        else {
+            sv_dump(sv);
+            croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
+                    SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+        }
+    } else {
+        croak ("encountered %s, but msgpack can only represent references to arrays or hashes",
+                   SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+    }
+}
+
+XS(xs_pack) {
+    dXSARGS;
+    if (items != 2) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)");
+    }
+
+    SV* val = ST(1);
+
+    enc_t enc;
+    enc.sv        = sv_2mortal(NEWSV(0, INIT_SIZE));
+    enc.cur       = SvPVX(enc.sv);
+    enc.end       = SvEND(enc.sv);
+    SvPOK_only(enc.sv);
+
+    _msgpack_pack_sv(&enc, val);
+
+    SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
+    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
+
+    ST(0) = enc.sv;
+    XSRETURN(1);
+}

perl/t/00_compile.t

@@ -0,0 +1,6 @@
+use strict;
+use warnings;
+use Test::More tests => 1;
+
+use_ok 'Data::MessagePack';
+

perl/t/01_pack.t

@@ -0,0 +1,64 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+
+sub packit {
+    local $_ = unpack("H*", Data::MessagePack->pack($_[0]));
+    s/(..)/$1 /g;
+    s/ $//;
+    $_;
+}
+
+sub pis ($$) {
+    is packit($_[0]), $_[1], 'dump ' . $_[1];
+}
+
+my @dat = (
+    0,     '00',
+    (my $foo="0")+0, '00',
+    {2 => undef}, '81 a1 32 c0',
+    do {no warnings; my $foo = 10; "$foo"; $foo = undef; $foo} => 'c0', # PVIV but !POK && !IOK
+    1,     '01',
+    127,   '7f',
+    128,   'cc 80',
+    255,   'cc ff',
+    256,   'cd 01 00',
+    65535, 'cd ff ff',
+    65536, 'ce 00 01 00 00',
+    -1,    'ff',
+    -32,   'e0',
+    -33,   'd0 df',
+    -128,  'd0 80',
+    -129,  'd1 ff 7f',
+    -32768, 'd1 80 00',
+    -32769, 'd2 ff ff 7f ff',
+    1.0,   'cb 3f f0 00 00 00 00 00 00',
+    do { my $x=3.0;my $y = "$x";$x },   'a1 33', # PVNV
+    "",    'a0',
+    "a",   'a1 61',
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61',
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61',
+    undef, 'c0',
+    Data::MessagePack::true(), 'c3',
+    Data::MessagePack::false(), 'c2',
+    [], '90',
+    [+[]], '91 90',
+    [[], undef], '92 90 c0',
+    {'a', 0}, '81 a1 61 00',
+    8388608, 'ce 00 80 00 00',
+
+    [undef, false, true], '93 c0 c2 c3',
+    ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66',
+    [[], [[undef]]], '92 90 91 91 c0',
+    [undef, false, true], '93 c0 c2 c3',
+    [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff',
+    [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff',
+    2147483648, 'ce 80 00 00 00',
+    -2147483648, 'd2 80 00 00 00',
+);
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/02_unpack.t

@@ -0,0 +1,24 @@
+use Test::More;
+use Data::MessagePack;
+use t::Util;
+no warnings 'uninitialized'; # i need this. i need this.
+
+sub unpackit {
+    my $v = $_[0];
+    $v =~ s/ //g;
+    $v = pack 'H*', $v;
+    return Data::MessagePack->unpack($v);
+}
+
+sub pis ($$) {
+    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/03_stream_unpack.t

@@ -0,0 +1,32 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+
+no warnings 'uninitialized'; # i need this. i need this.
+
+my $up = Data::MessagePack::Unpacker->new;
+sub unpackit {
+    my $v = $_[0];
+    $v =~ s/ //g;
+    $v = pack 'H*', $v;
+    $up->reset;
+    my $ret = $up->execute($v, 0);
+    if ($ret != length($v)) {
+        fail "extra bytes";
+    }
+    return $up->data;
+}
+
+sub pis ($$) {
+    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2) + 1;
+
+isa_ok $up, 'Data::MessagePack::Unpacker';
+for (my $i=0; $i<scalar(@dat); ) {
+    pis $dat[$i++], $dat[$i++];
+}
+

perl/t/04_invert.t

@@ -0,0 +1,24 @@
+use Test::More;
+use Data::MessagePack;
+use t::Util;
+no warnings 'uninitialized'; # i need this. i need this.
+
+sub invert {
+    return Data::MessagePack->unpack(
+        Data::MessagePack->pack($_[0]),
+    );
+}
+
+sub pis ($) {
+    is_deeply invert($_[0]), $_[0], 'dump ' . $_[0];
+}
+
+my @dat = do 't/data.pl';
+
+plan tests => 1*(scalar(@dat)/2);
+
+for (my $i=0; $i<scalar(@dat); ) {
+    $i++;
+    pis $dat[$i++];
+}
+

perl/t/05_preferred_int.t

@@ -0,0 +1,57 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+use Data::Dumper;
+no warnings; # shut up "Integer overflow in hexadecimal number"
+
+sub packit {
+    local $_ = unpack("H*", Data::MessagePack->pack($_[0]));
+    s/(..)/$1 /g;
+    s/ $//;
+    $_;
+}
+
+sub pis ($$) {
+    is packit($_[0]), $_[1], 'dump ' . $_[1];
+    # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0]));
+}
+
+my @dat = (
+    '',      'a0',
+    '0',     '00',
+    '1',     '01',
+    '10',    '0a',
+    '-1',    'ff',
+    '-10',   'f6',
+    '-',     'a1 2d',
+    ''.0xEFFF => 'cd ef ff',
+    ''.0xFFFF => 'cd ff ff',
+    ''.0xFFFFFF => 'ce 00 ff ff ff',
+    ''.0xFFFFFFFF => 'ce ff ff ff ff',
+    ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35',
+    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34',
+    '-'.0x8000000 => 'd2 f8 00 00 00',
+    '-'.0x80000000 => 'd2 80 00 00 00',
+    '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38',
+    '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38',
+    '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35',
+    {'0' => '1'}, '81 00 01',
+    {'abc' => '1'}, '81 a3 61 62 63 01',
+);
+plan tests => 1*(scalar(@dat)/2) + 2;
+
+for (my $i=0; $i<scalar(@dat); ) {
+    local $Data::MessagePack::PreferInteger = 1;
+    my($x, $y) = ($i++, $i++);
+    pis $dat[$x], $dat[$y];
+}
+
+# flags working?
+{
+    local $Data::MessagePack::PreferInteger;
+    $Data::MessagePack::PreferInteger = 1;
+    pis '0',     '00';
+    $Data::MessagePack::PreferInteger = 0;
+    pis '0',     'a1 30';
+}
+

perl/t/Util.pm

@@ -0,0 +1,20 @@
+package t::Util;
+use strict;
+use warnings;
+
+sub import {
+    my $pkg = caller(0);
+
+    strict->import;
+    warnings->import;
+
+    no strict 'refs';
+    *{"$pkg\::true"} = sub () {
+        Data::MessagePack::true()
+    };
+    *{"$pkg\::false"} = sub () {
+        Data::MessagePack::false()
+    };
+}
+
+1;

perl/t/data.pl

@@ -0,0 +1,18 @@
+no warnings; # i need this, i need this.
+(
+    '93 c0 c2 c3' => [undef, false, true],
+    '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"],
+    '92 90 91 91 c0', [[], [[undef]]],
+    '93 c0 c2 c3', [undef, false, true],
+    'ce 80 00 00 00', 2147483648,
+    '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
+    '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]],
+    '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]],
+    '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"],
+    '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1],
+    '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}},
+    '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}],
+    'ce 00 ff ff ff' => ''.0xFFFFFF,
+    'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF,
+    'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF,
+)

perl/unpack.c

@@ -0,0 +1,317 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "util.h"
+#define NEED_newRV_noinc
+#define NEED_sv_2pv_flags
+#include "ppport.h"
+
+#ifdef __cplusplus
+};
+#endif
+
+typedef struct {
+    int finished;
+    SV* source;
+} unpack_user;
+
+#include "msgpack/unpack_define.h"
+
+#define msgpack_unpack_struct(name) \
+    struct template ## name
+
+#define msgpack_unpack_func(ret, name) \
+    ret template ## name
+
+#define msgpack_unpack_callback(name) \
+    template_callback ## name
+
+#define msgpack_unpack_object SV*
+
+#define msgpack_unpack_user unpack_user
+
+/* ---------------------------------------------------------------------- */
+/* utility functions                                                      */
+
+static INLINE SV *
+get_bool (const char *name) {
+    SV * sv = get_sv( name, 1 );
+
+    SvREADONLY_on(sv);
+    SvREADONLY_on( SvRV(sv) );
+
+    return sv;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct template_context;
+typedef struct template_context msgpack_unpack_t;
+
+static void template_init(msgpack_unpack_t* u);
+
+static SV* template_data(msgpack_unpack_t* u);
+
+static int template_execute(msgpack_unpack_t* u,
+    const char* data, size_t len, size_t* off);
+
+static INLINE SV* template_callback_root(unpack_user* u)
+{ return &PL_sv_undef; }
+
+static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
+{ *o = newSVuv(d); return 0; }
+
+static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
+{ *o = newSViv((long)d); return 0; }
+
+static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
+{ *o = newSViv(d); return 0; }
+
+static INLINE int template_callback_float(unpack_user* u, float d, SV** o)
+{ *o = newSVnv(d); return 0; }
+
+static INLINE int template_callback_double(unpack_user* u, double d, SV** o)
+{ *o = newSVnv(d); return 0; }
+
+static INLINE int template_callback_nil(unpack_user* u, SV** o)
+{ *o = &PL_sv_undef; return 0; }
+
+static INLINE int template_callback_true(unpack_user* u, SV** o)
+{ *o = get_bool("Data::MessagePack::true") ; return 0; }
+
+static INLINE int template_callback_false(unpack_user* u, SV** o)
+{ *o = get_bool("Data::MessagePack::false") ; return 0; }
+
+static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
+{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; }
+
+static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
+{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
+
+static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
+{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; }
+
+static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
+{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
+
+static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
+{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; }
+
+#define UNPACKER(from, name) \
+	msgpack_unpack_t *name; \
+    name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \
+	if(name == NULL) { \
+		Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
+	}
+
+#include "msgpack/unpack_template.h"
+
+SV* _msgpack_unpack(SV* data, int limit) {
+    msgpack_unpack_t mp;
+    unpack_user u = {0, &PL_sv_undef};
+	int ret;
+	size_t from = 0;
+    STRLEN dlen;
+    const char * dptr = SvPV_const(data, dlen);
+
+	template_init(&mp);
+    mp.user = u;
+
+	mp.user.source = data;
+	ret = template_execute(&mp, dptr, (size_t)dlen, &from);
+	mp.user.source = &PL_sv_undef;
+
+	if(ret < 0) {
+        Perl_croak(aTHX_ "parse error.");
+	} else if(ret == 0) {
+        Perl_croak(aTHX_ "insufficient bytes.");
+	} else {
+		if(from < dlen) {
+            Perl_croak(aTHX_ "extra bytes.");
+		}
+        return template_data(&mp);
+	}
+}
+
+XS(xs_unpack_limit) {
+    dXSARGS;
+
+    if (items != 3) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)");
+    }
+
+    {
+        int limit = SvIV(ST(2));
+        ST(0) = _msgpack_unpack(ST(1), limit);
+    }
+    XSRETURN(1);
+}
+
+
+XS(xs_unpack) {
+    dXSARGS;
+    msgpack_unpack_t mp;
+
+    if (items != 2) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')");
+    }
+
+    {
+        ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1)));
+    }
+
+    XSRETURN(1);
+}
+
+/* ------------------------------ stream -- */
+
+static void _reset(SV* self) {
+	UNPACKER(self, mp);
+	template_init(mp);
+	unpack_user u = {0, &PL_sv_undef};
+	mp->user = u;
+}
+
+XS(xs_unpacker_new) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()");
+    }
+
+    SV* self = sv_newmortal();
+	msgpack_unpack_t *mp;
+
+    Newx(mp, 1, msgpack_unpack_t);
+
+    sv_setref_pv(self, "Data::MessagePack::Unpacker", mp);
+    _reset(self);
+
+    ST(0) = self;
+    XSRETURN(1);
+}
+
+static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
+    UNPACKER(self, mp);
+
+    size_t from = off;
+	const char* dptr = SvPV_nolen_const(data);
+	long dlen = limit;
+	int ret;
+
+	if(from >= dlen) {
+        Perl_croak(aTHX_ "offset is bigger than data buffer size.");
+	}
+
+	mp->user.source = data;
+	ret = template_execute(mp, dptr, (size_t)dlen, &from);
+	mp->user.source = &PL_sv_undef;
+
+	if(ret < 0) {
+		Perl_croak(aTHX_ "parse error.");
+	} else if(ret > 0) {
+		mp->user.finished = 1;
+		return newSVuv(from);
+	} else {
+		mp->user.finished = 0;
+		return newSVuv(from);
+	}
+}
+
+XS(xs_unpacker_execute) {
+    dXSARGS;
+    if (items != 3) {
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)");
+    }
+
+    {
+        SV* self = ST(0);
+        SV* data = ST(1);
+        IV  off  = SvIV(ST(2));
+
+        ST(0) = _execute_impl(self, data, off, sv_len(data));
+    }
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_execute_limit) {
+    dXSARGS;
+    if (items != 4) {
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)");
+    }
+
+    SV* self = ST(0);
+    SV* data = ST(1);
+    IV off   = SvIV(ST(2));
+    IV limit = SvIV(ST(3));
+
+    ST(0) = _execute_impl(self, data, off, limit);
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_is_finished) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->is_finished()");
+    }
+
+	UNPACKER(ST(0), mp);
+    ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no;
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_data) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->data()");
+    }
+
+	UNPACKER(ST(0), mp);
+	ST(0) = template_data(mp);
+
+    XSRETURN(1);
+}
+
+XS(xs_unpacker_reset) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->reset()");
+    }
+
+    _reset(ST(0));
+
+    XSRETURN(0);
+}
+
+XS(xs_unpacker_destroy) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()");
+    }
+
+	UNPACKER(ST(0), mp);
+    Safefree(mp);
+
+    XSRETURN(0);
+}

perl/util.h

@@ -0,0 +1,11 @@
+#ifndef __PERL_MSGPACK_UTIL_H__
+#define __PERL_MSGPACK_UTIL_H__
+
+#if __GNUC__ >= 3
+# define INLINE                     inline
+#else
+# define INLINE                     
+#endif
+
+#endif // __PERL_MSGPACK_UTIL_H__
+

perl/xt/99_pod.t

@@ -0,0 +1,4 @@
+use Test::More;
+eval "use Test::Pod 1.00";
+plan skip_all => "Test::Pod 1.00 required for testing POD" if $@;
+all_pod_files_ok();

python/MANIFEST

@@ -0,0 +1,9 @@
+setup.py
+msgpack/pack.h
+msgpack/unpack.h
+msgpack/_msgpack.pyx
+msgpack/__init__.py
+msgpack/pack_define.h
+msgpack/pack_template.h
+msgpack/unpack_define.h
+msgpack/unpack_template.h

python/Makefile

@@ -0,0 +1,8 @@
+all:
+	python setup.py build_ext -i -f
+	python setup.py build
+	python setup.py sdist
+
+.PHONY: test
+test:
+	nosetests test

python/README

@@ -0,0 +1,38 @@
+===========================
+MessagePack Python Binding
+===========================
+
+:author: Naoki INADA
+:version: 0.1.0
+:date: 2009-07-12
+
+HOW TO USE
+-----------
+You can read document in docstring after `import msgpack`
+
+
+INSTALL
+---------
+Cython_ is required to build msgpack.
+
+.. _Cython: http://www.cython.org/
+
+posix
+''''''
+You can install msgpack in common way.
+
+    $ python setup.py install
+
+Windows
+''''''''
+MessagePack requires gcc currently. So you need to prepare
+MinGW GCC.
+
+    $ python setup.py install -c mingw32
+
+TEST
+----
+MessagePack uses nosetest for testing.
+Run test with following command:
+
+    $ nosetests test

python/msgpack/__init__.py

@@ -0,0 +1,3 @@
+# coding: utf-8
+from _msgpack import *
+

python/msgpack/_msgpack.pyx

@@ -0,0 +1,321 @@
+# coding: utf-8
+
+import cStringIO
+
+cdef extern from "Python.h":
+    ctypedef char* const_char_ptr "const char*"
+    ctypedef struct PyObject
+
+    cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
+    cdef PyObject* Py_True
+    cdef PyObject* Py_False
+
+    cdef char* PyString_AsString(object o)
+    cdef long long PyLong_AsLongLong(object o)
+    cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
+
+    cdef int PyMapping_Check(object o)
+    cdef int PySequence_Check(object o)
+    cdef int PyLong_Check(object o)
+    cdef int PyInt_Check(object o)
+    cdef int PyFloat_Check(object o)
+    cdef int PyString_Check(object o)
+    cdef int PyUnicode_Check(object o)
+
+cdef extern from "stdlib.h":
+    void* malloc(size_t)
+    void* realloc(void*, size_t)
+    void free(void*)
+
+cdef extern from "string.h":
+    void* memcpy(char* dst, char* src, size_t size)
+    void* memmove(char* dst, char* src, size_t size)
+
+cdef extern from "pack.h":
+    struct msgpack_packer:
+        char* buf
+        size_t length
+        size_t buf_size
+
+    int msgpack_pack_int(msgpack_packer* pk, int d)
+    int msgpack_pack_nil(msgpack_packer* pk)
+    int msgpack_pack_true(msgpack_packer* pk)
+    int msgpack_pack_false(msgpack_packer* pk)
+    int msgpack_pack_long(msgpack_packer* pk, long d)
+    int msgpack_pack_long_long(msgpack_packer* pk, long long d)
+    int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
+    int msgpack_pack_double(msgpack_packer* pk, double d)
+    int msgpack_pack_array(msgpack_packer* pk, size_t l)
+    int msgpack_pack_map(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
+
+
+cdef class Packer(object):
+    """MessagePack Packer
+    
+    usage:
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+    """
+    cdef msgpack_packer pk
+
+    def __cinit__(self):
+        cdef int buf_size = 1024*1024
+        self.pk.buf = <char*> malloc(buf_size);
+        self.pk.buf_size = buf_size
+        self.pk.length = 0
+
+    def __dealloc__(self):
+        free(self.pk.buf);
+
+    cdef int __pack(self, object o):
+        cdef long long llval
+        cdef unsigned long long ullval
+        cdef long longval
+        cdef double fval
+        cdef char* rawval 
+        cdef int ret
+
+        if o is None:
+            ret = msgpack_pack_nil(&self.pk)
+        elif <PyObject*>o == Py_True:
+            ret = msgpack_pack_true(&self.pk)
+        elif <PyObject*>o == Py_False:
+            ret = msgpack_pack_false(&self.pk)
+        elif PyLong_Check(o):
+            if o > 0:
+                ullval = PyLong_AsUnsignedLongLong(o)
+                ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
+            else:
+                llval = PyLong_AsLongLong(o)
+                ret = msgpack_pack_long_long(&self.pk, llval)
+        elif PyInt_Check(o):
+            longval = o
+            ret = msgpack_pack_long(&self.pk, longval)
+        elif PyFloat_Check(o):
+            fval = o
+            ret = msgpack_pack_double(&self.pk, fval)
+        elif PyString_Check(o):
+            rawval = o
+            ret = msgpack_pack_raw(&self.pk, len(o))
+            if ret == 0:
+                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+        elif PyUnicode_Check(o):
+            o = o.encode('utf-8')
+            rawval = o
+            ret = msgpack_pack_raw(&self.pk, len(o))
+            if ret == 0:
+                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+        elif PyMapping_Check(o):
+            ret = msgpack_pack_map(&self.pk, len(o))
+            if ret == 0:
+                for k,v in o.iteritems():
+                    ret = self.__pack(k)
+                    if ret != 0: break
+                    ret = self.__pack(v)
+                    if ret != 0: break
+        elif PySequence_Check(o):
+            ret = msgpack_pack_array(&self.pk, len(o))
+            if ret == 0:
+                for v in o:
+                    ret = self.__pack(v)
+                    if ret != 0: break
+        else:
+            # TODO: Serialize with defalt() like simplejson.
+            raise TypeError, "can't serialize %r" % (o,)
+        return ret
+
+    def pack(self, object obj):
+        cdef int ret
+        ret = self.__pack(obj)
+        if ret:
+            raise TypeError
+        buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
+
+
+def pack(object o, object stream):
+    """pack an object `o` and write it to stream)."""
+    packer = Packer()
+    stream.write(packer.pack(o))
+
+def packb(object o):
+    """pack o and return packed bytes."""
+    packer = Packer()
+    return packer.pack(o)
+
+packs = packb
+
+cdef extern from "unpack.h":
+    ctypedef struct template_context:
+        PyObject* obj
+        size_t count
+        unsigned int ct
+        PyObject* key
+
+    int template_execute(template_context* ctx, const_char_ptr data,
+            size_t len, size_t* off)
+    void template_init(template_context* ctx)
+    object template_data(template_context* ctx)
+
+
+def unpackb(object packed_bytes):
+    """Unpack packed_bytes to object. Returns an unpacked object."""
+    cdef const_char_ptr p = packed_bytes
+    cdef template_context ctx
+    cdef size_t off = 0
+    cdef int ret
+    template_init(&ctx)
+    ret = template_execute(&ctx, p, len(packed_bytes), &off)
+    if ret == 1:
+        return template_data(&ctx)
+    else:
+        return None
+
+unpacks = unpackb
+
+def unpack(object stream):
+    """unpack an object from stream."""
+    packed = stream.read()
+    return unpackb(packed)
+
+cdef class UnpackIterator(object):
+    cdef object unpacker
+
+    def __init__(self, unpacker):
+        self.unpacker = unpacker
+
+    def __next__(self):
+        return self.unpacker.unpack()
+
+    def __iter__(self):
+        return self
+
+cdef class Unpacker(object):
+    """Unpacker(file_like=None, read_size=1024*1024)
+
+    Streaming unpacker.
+    file_like must have read(n) method.
+    read_size is used like file_like.read(read_size)
+
+    If file_like is None, you can ``feed()`` bytes. ``feed()`` is
+    useful for unpacking from non-blocking stream.
+
+    exsample 1:
+        unpacker = Unpacker(afile)
+        for o in unpacker:
+            do_something(o)
+
+    example 2:
+        unpacker = Unpacker()
+        while 1:
+            buf = astream.read()
+            unpacker.feed(buf)
+            for o in unpacker:
+                do_something(o)
+    """
+
+    cdef template_context ctx
+    cdef char* buf
+    cdef size_t buf_size, buf_head, buf_tail
+    cdef object file_like
+    cdef int read_size
+    cdef object waiting_bytes
+
+    def __cinit__(self):
+        self.buf = NULL
+
+    def __dealloc__(self):
+        if self.buf:
+            free(self.buf);
+
+    def __init__(self, file_like=None, int read_size=1024*1024):
+        self.file_like = file_like
+        self.read_size = read_size
+        self.waiting_bytes = []
+        self.buf = <char*>malloc(read_size)
+        self.buf_size = read_size
+        self.buf_head = 0
+        self.buf_tail = 0
+        template_init(&self.ctx)
+
+    def feed(self, next_bytes):
+        if not isinstance(next_bytes, str):
+           raise ValueError, "Argument must be bytes object"
+        self.waiting_bytes.append(next_bytes)
+
+    cdef append_buffer(self):
+        cdef char* buf = self.buf
+        cdef Py_ssize_t tail = self.buf_tail
+        cdef Py_ssize_t l
+
+        for b in self.waiting_bytes:
+            l = len(b)
+            memcpy(buf + tail, PyString_AsString(b), l)
+            tail += l
+        self.buf_tail = tail
+        del self.waiting_bytes[:]
+
+    # prepare self.buf
+    cdef fill_buffer(self):
+        cdef Py_ssize_t add_size
+
+        if self.file_like is not None:
+            next_bytes = self.file_like.read(self.read_size)
+            if next_bytes:
+                self.waiting_bytes.append(next_bytes)
+            else:
+                self.file_like = None
+
+        if not self.waiting_bytes:
+            return
+
+        add_size = 0
+        for b in self.waiting_bytes:
+            add_size += len(b)
+
+        cdef char* buf = self.buf
+        cdef size_t head = self.buf_head
+        cdef size_t tail = self.buf_tail
+        cdef size_t size = self.buf_size
+
+        if self.buf_tail + add_size <= self.buf_size:
+            # do nothing.
+            pass
+        if self.buf_tail - self.buf_head + add_size < self.buf_size:
+            # move to front.
+            memmove(buf, buf + head, tail - head)
+            tail -= head
+            head = 0
+        else:
+            # expand buffer
+            size = tail + add_size
+            buf = <char*>realloc(<void*>buf, size)
+
+        self.buf = buf
+        self.buf_head = head
+        self.buf_tail = tail
+        self.buf_size = size
+
+        self.append_buffer()
+
+    cpdef unpack(self):
+        """unpack one object"""
+        cdef int ret
+        self.fill_buffer()
+        ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+        if ret == 1:
+            return template_data(&self.ctx)
+        elif ret == 0:
+            if self.file_like is not None:
+                return self.unpack()
+            raise StopIteration, "No more unpack data."
+        else:
+            raise ValueError, "Unpack failed."
+
+    def __iter__(self):
+        return UnpackIterator(self)

python/msgpack/pack.h

@@ -0,0 +1,102 @@
+/*
+ * MessagePack for Python packing routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "pack_define.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct msgpack_packer {
+    char *buf;
+    size_t length;
+    size_t buf_size;
+} msgpack_packer;
+
+typedef struct Packer Packer;
+
+static inline int msgpack_pack_short(msgpack_packer* pk, short d);
+static inline int msgpack_pack_int(msgpack_packer* pk, int d);
+static inline int msgpack_pack_long(msgpack_packer* pk, long d);
+static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
+static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
+static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
+static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
+static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
+
+static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
+static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
+static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
+static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
+static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
+static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
+static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
+static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
+
+static inline int msgpack_pack_float(msgpack_packer* pk, float d);
+static inline int msgpack_pack_double(msgpack_packer* pk, double d);
+
+static inline int msgpack_pack_nil(msgpack_packer* pk);
+static inline int msgpack_pack_true(msgpack_packer* pk);
+static inline int msgpack_pack_false(msgpack_packer* pk);
+
+static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
+
+static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
+{
+    char* buf = pk->buf;
+    size_t bs = pk->buf_size;
+    size_t len = pk->length;
+
+    if (len + l > bs) {
+        bs = (len + l) * 2;
+        buf = realloc(buf, bs);
+        if (!buf) return -1;
+    }
+    memcpy(buf + len, data, l);
+    len += l;
+
+    pk->buf = buf;
+    pk->buf_size = bs;
+    pk->length = len;
+    return 0;
+}
+
+#define msgpack_pack_inline_func(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_inline_func_cint(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_user msgpack_packer*
+
+#define msgpack_pack_append_buffer(user, buf, len) \
+        return msgpack_pack_write(user, (const char*)buf, len)
+
+#include "pack_template.h"
+
+#ifdef __cplusplus
+}
+#endif

python/msgpack/pack_define.h

@@ -0,0 +1,26 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_PACK_DEFINE_H__
+#define MSGPACK_PACK_DEFINE_H__
+
+#include <stddef.h>
+#include <stdint.h>
+#include <limits.h>
+
+#endif /* msgpack/pack_define.h */
+

python/msgpack/pack_template.h

@@ -0,0 +1,741 @@
+/*
+ * MessagePack packing routine template
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#endif
+#endif
+
+
+#ifdef __LITTLE_ENDIAN__
+
+#define STORE8_BE8(d) \
+	((uint8_t*)&d)[0]
+
+
+#define STORE16_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE16_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+
+#define STORE32_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE32_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE32_BE32(d) \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+
+#define STORE64_BE8(d) \
+	((uint8_t*)&d)[0]
+
+#define STORE64_BE16(d) \
+	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE64_BE32(d) \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+#define STORE64_BE64(d) \
+	((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \
+	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
+
+
+#elif __BIG_ENDIAN__
+
+#define STORE8_BE8(d) \
+	((uint8_t*)&d)[0]
+
+
+#define STORE16_BE8(d) \
+	((uint8_t*)&d)[1]
+
+#define STORE16_BE16(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1]
+
+
+#define STORE32_BE8(d) \
+	((uint8_t*)&d)[3]
+
+#define STORE32_BE16(d) \
+	((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
+
+#define STORE32_BE32(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
+
+
+#define STORE64_BE8(d) \
+	((uint8_t*)&d)[7]
+
+#define STORE64_BE16(d) \
+	((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
+
+#define STORE64_BE32(d) \
+	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
+
+#define STORE64_BE64(d) \
+	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \
+	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
+
+#endif
+
+#ifndef msgpack_pack_inline_func
+#error msgpack_pack_inline_func template is not defined
+#endif
+
+#ifndef msgpack_pack_user
+#error msgpack_pack_user type is not defined
+#endif
+
+#ifndef msgpack_pack_append_buffer
+#error msgpack_pack_append_buffer callback is not defined
+#endif
+
+
+/*
+ * Integer
+ */
+
+#define msgpack_pack_real_uint8(x, d) \
+do { \
+	if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
+	} else { \
+		/* unsigned 8 */ \
+		const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint16(x, d) \
+do { \
+	if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
+	} else if(d < (1<<8)) { \
+		/* unsigned 8 */ \
+		const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} else { \
+		/* unsigned 16 */ \
+		const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
+		msgpack_pack_append_buffer(x, buf, 3); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint32(x, d) \
+do { \
+	if(d < (1<<8)) { \
+		if(d < (1<<7)) { \
+			/* fixnum */ \
+			msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
+		} else { \
+			/* unsigned 8 */ \
+			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else { \
+		if(d < (1<<16)) { \
+			/* unsigned 16 */ \
+			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* unsigned 32 */ \
+			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint64(x, d) \
+do { \
+	if(d < (1ULL<<8)) { \
+		if(d < (1<<7)) { \
+			/* fixnum */ \
+			msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
+		} else { \
+			/* unsigned 8 */ \
+			const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else { \
+		if(d < (1ULL<<16)) { \
+			/* signed 16 */ \
+			const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else if(d < (1ULL<<32)) { \
+			/* signed 32 */ \
+			const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} else { \
+			/* signed 64 */ \
+			const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
+			msgpack_pack_append_buffer(x, buf, 9); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int8(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		/* signed 8 */ \
+		const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} else { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int16(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		if(d < -(1<<7)) { \
+			/* signed 16 */ \
+			const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* signed 8 */ \
+			const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
+	} else { \
+		if(d < (1<<8)) { \
+			/* unsigned 8 */ \
+			const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} else { \
+			/* unsigned 16 */ \
+			const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int32(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		if(d < -(1<<15)) { \
+			/* signed 32 */ \
+			const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} else if(d < -(1<<7)) { \
+			/* signed 16 */ \
+			const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* signed 8 */ \
+			const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
+	} else { \
+		if(d < (1<<8)) { \
+			/* unsigned 8 */ \
+			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} else if(d < (1<<16)) { \
+			/* unsigned 16 */ \
+			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* unsigned 32 */ \
+			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int64(x, d) \
+do { \
+	if(d < -(1LL<<5)) { \
+		if(d < -(1LL<<15)) { \
+			if(d < -(1LL<<31)) { \
+				/* signed 64 */ \
+				const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \
+				msgpack_pack_append_buffer(x, buf, 9); \
+			} else { \
+				/* signed 32 */ \
+				const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \
+				msgpack_pack_append_buffer(x, buf, 5); \
+			} \
+		} else { \
+			if(d < -(1<<7)) { \
+				/* signed 16 */ \
+				const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \
+				msgpack_pack_append_buffer(x, buf, 3); \
+			} else { \
+				/* signed 8 */ \
+				const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \
+				msgpack_pack_append_buffer(x, buf, 2); \
+			} \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
+	} else { \
+		if(d < (1LL<<16)) { \
+			if(d < (1<<8)) { \
+				/* unsigned 8 */ \
+				const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
+				msgpack_pack_append_buffer(x, buf, 2); \
+			} else { \
+				/* unsigned 16 */ \
+				const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
+				msgpack_pack_append_buffer(x, buf, 3); \
+			} \
+		} else { \
+			if(d < (1LL<<32)) { \
+				/* unsigned 32 */ \
+				const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
+				msgpack_pack_append_buffer(x, buf, 5); \
+			} else { \
+				/* unsigned 64 */ \
+				const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
+				msgpack_pack_append_buffer(x, buf, 9); \
+			} \
+		} \
+	} \
+} while(0)
+
+
+#ifdef msgpack_pack_inline_func_fastint
+
+msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
+{
+	const unsigned char buf[2] = {0xcc, STORE8_BE8(d)};
+	msgpack_pack_append_buffer(x, buf, 2);
+}
+
+msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
+{
+	const unsigned char buf[3] = {0xcd, STORE16_BE16(d)};
+	msgpack_pack_append_buffer(x, buf, 3);
+}
+
+msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
+{
+	const unsigned char buf[5] = {0xce, STORE32_BE32(d)};
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
+{
+	const unsigned char buf[9] = {0xcf, STORE64_BE64(d)};
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
+{
+	const unsigned char buf[2] = {0xd0, STORE8_BE8(d)};
+	msgpack_pack_append_buffer(x, buf, 2);
+}
+
+msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
+{
+	const unsigned char buf[3] = {0xd1, STORE16_BE16(d)};
+	msgpack_pack_append_buffer(x, buf, 3);
+}
+
+msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
+{
+	const unsigned char buf[5] = {0xd2, STORE32_BE32(d)};
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
+{
+	const unsigned char buf[9] = {0xd3, STORE64_BE64(d)};
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+#undef msgpack_pack_inline_func_fastint
+#endif
+
+
+msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d)
+{
+	msgpack_pack_real_uint8(x, d);
+}
+
+msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d)
+{
+	msgpack_pack_real_uint16(x, d);
+}
+
+msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d)
+{
+	msgpack_pack_real_uint32(x, d);
+}
+
+msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d)
+{
+	msgpack_pack_real_uint64(x, d);
+}
+
+msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d)
+{
+	msgpack_pack_real_int8(x, d);
+}
+
+msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d)
+{
+	msgpack_pack_real_int16(x, d);
+}
+
+msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d)
+{
+	msgpack_pack_real_int32(x, d);
+}
+
+msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
+{
+	msgpack_pack_real_int64(x, d);
+}
+
+
+#ifdef msgpack_pack_inline_func_cint
+
+msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
+{
+#if defined(SIZEOF_SHORT) || defined(SHRT_MAX)
+#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+#else
+if(sizeof(short) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(short) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
+{
+#if defined(SIZEOF_INT) || defined(INT_MAX)
+#if SIZEOF_INT == 2 || INT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+#else
+if(sizeof(int) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(int) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
+{
+#if defined(SIZEOF_LONG) || defined(LONG_MAX)
+#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+#else
+if(sizeof(long) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(long) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
+{
+#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX)
+#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+#else
+if(sizeof(long long) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(long long) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
+{
+#if defined(SIZEOF_SHORT) || defined(USHRT_MAX)
+#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+#else
+if(sizeof(unsigned short) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned short) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
+{
+#if defined(SIZEOF_INT) || defined(UINT_MAX)
+#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+#else
+if(sizeof(unsigned int) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned int) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
+{
+#if defined(SIZEOF_LONG) || defined(ULONG_MAX)
+#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+#else
+if(sizeof(unsigned int) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned int) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
+{
+#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX)
+#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+#else
+if(sizeof(unsigned long long) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long long) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+#undef msgpack_pack_inline_func_cint
+#endif
+
+
+
+/*
+ * Float
+ */
+
+msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
+{
+	union { char buf[4]; uint32_t num; } f;
+	*((float*)&f.buf) = d;  // FIXME
+	const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)};
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
+{
+	union { char buf[8]; uint64_t num; } f;
+	*((double*)&f.buf) = d;  // FIXME
+	const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)};
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+
+/*
+ * Nil
+ */
+
+msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc0;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Boolean
+ */
+
+msgpack_pack_inline_func(_true)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc3;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+msgpack_pack_inline_func(_false)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc2;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Array
+ */
+
+msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
+{
+	if(n < 16) {
+		unsigned char d = 0x90 | n;
+		msgpack_pack_append_buffer(x, &d, 1);
+	} else if(n < 65536) {
+		uint16_t d = (uint16_t)n;
+		unsigned char buf[3] = {0xdc, STORE16_BE16(d)};
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		uint32_t d = (uint32_t)n;
+		unsigned char buf[5] = {0xdd, STORE32_BE32(d)};
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+
+/*
+ * Map
+ */
+
+msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
+{
+	if(n < 16) {
+		unsigned char d = 0x80 | n;
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
+	} else if(n < 65536) {
+		uint16_t d = (uint16_t)n;
+		unsigned char buf[3] = {0xde, STORE16_BE16(d)};
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		uint32_t d = (uint32_t)n;
+		unsigned char buf[5] = {0xdf, STORE32_BE32(d)};
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+
+/*
+ * Raw
+ */
+
+msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
+{
+	if(l < 32) {
+		unsigned char d = 0xa0 | l;
+		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
+	} else if(l < 65536) {
+		uint16_t d = (uint16_t)l;
+		unsigned char buf[3] = {0xda, STORE16_BE16(d)};
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		uint32_t d = (uint32_t)l;
+		unsigned char buf[5] = {0xdb, STORE32_BE32(d)};
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l)
+{
+	msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+}
+
+#undef msgpack_pack_inline_func
+#undef msgpack_pack_user
+#undef msgpack_pack_append_buffer
+
+#undef STORE8_BE8
+
+#undef STORE16_BE8
+#undef STORE16_BE16
+
+#undef STORE32_BE8
+#undef STORE32_BE16
+#undef STORE32_BE32
+
+#undef STORE64_BE8
+#undef STORE64_BE16
+#undef STORE64_BE32
+#undef STORE64_BE64
+
+#undef msgpack_pack_real_uint8
+#undef msgpack_pack_real_uint16
+#undef msgpack_pack_real_uint32
+#undef msgpack_pack_real_uint64
+#undef msgpack_pack_real_int8
+#undef msgpack_pack_real_int16
+#undef msgpack_pack_real_int32
+#undef msgpack_pack_real_int64
+

python/msgpack/unpack.h

@@ -0,0 +1,177 @@
+/*
+ * MessagePack for Python unpacking routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#define MSGPACK_MAX_STACK_SIZE  (1024)
+#include "unpack_define.h"
+
+typedef struct unpack_user {
+} unpack_user;
+
+
+#define msgpack_unpack_struct(name) \
+	struct template ## name
+
+#define msgpack_unpack_func(ret, name) \
+	static inline ret template ## name
+
+#define msgpack_unpack_callback(name) \
+	template_callback ## name
+
+#define msgpack_unpack_object PyObject*
+
+#define msgpack_unpack_user unpack_user
+
+
+struct template_context;
+typedef struct template_context template_context;
+
+static inline msgpack_unpack_object template_callback_root(unpack_user* u)
+{
+    return NULL;
+}
+
+static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong((long)d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+{
+    return template_callback_uint16(u, d, o);
+}
+
+
+static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (d > LONG_MAX) {
+        p = PyLong_FromUnsignedLong((unsigned long)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyLong_FromUnsignedLongLong(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+{
+    return template_callback_int32(u, d, o);
+}
+
+static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+{
+    return template_callback_int32(u, d, o);
+}
+
+static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyLong_FromLongLong(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyFloat_FromDouble(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+{
+    return template_callback_double(u, d, o);
+}
+
+static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_None); *o = Py_None; return 0; }
+
+static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_True); *o = Py_True; return 0; }
+
+static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_False); *o = Py_False; return 0; }
+
+static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    PyObject *p = PyList_New(n);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
+{ PyList_SET_ITEM(*c, current, o); return 0; }
+
+static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    PyObject *p = PyDict_New();
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+{
+    if (PyDict_SetItem(*c, k, v) == 0) {
+        Py_DECREF(k);
+        Py_DECREF(v);
+        return 0;
+    }
+    return -1;
+}
+
+static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    py = PyString_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+#include "unpack_template.h"

python/msgpack/unpack_define.h

@@ -0,0 +1,142 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_UNPACK_DEFINE_H__
+#define MSGPACK_UNPACK_DEFINE_H__
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+#ifndef __WIN32__
+#include <arpa/inet.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_MAX_STACK_SIZE
+#define MSGPACK_MAX_STACK_SIZE 16
+#endif
+
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#endif
+#endif
+
+#ifdef __WIN32__
+static inline uint16_t msgpack_betoh16(uint16_t x) {
+	return  ((x <<  8) & 0xff00U) |
+			((x >>  8) & 0x00ffU);
+}
+static inline uint32_t msgpack_betoh32(uint32_t x) {
+	return	((x << 24) & 0xff000000UL ) |
+			((x <<  8) & 0x00ff0000UL ) |
+			((x >>  8) & 0x0000ff00UL ) |
+			((x >> 24) & 0x000000ffUL );
+}
+#else
+#define msgpack_betoh16(x) ntohs(x)
+#define msgpack_betoh32(x) ntohl(x)
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#if defined(__bswap_64)
+#  define msgpack_betoh64(x) __bswap_64(x)
+#elif defined(__DARWIN_OSSwapInt64)
+#  define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x)
+#else
+static inline uint64_t msgpack_betoh64(uint64_t x) {
+	return	((x << 56) & 0xff00000000000000ULL ) |
+			((x << 40) & 0x00ff000000000000ULL ) |
+			((x << 24) & 0x0000ff0000000000ULL ) |
+			((x <<  8) & 0x000000ff00000000ULL ) |
+			((x >>  8) & 0x00000000ff000000ULL ) |
+			((x >> 24) & 0x0000000000ff0000ULL ) |
+			((x >> 40) & 0x000000000000ff00ULL ) |
+			((x >> 56) & 0x00000000000000ffULL ) ;
+}
+#endif
+#else
+#define msgpack_betoh64(x) (x)
+#endif
+
+
+typedef enum {
+	CS_HEADER            = 0x00,  // nil
+
+	//CS_                = 0x01,
+	//CS_                = 0x02,  // false
+	//CS_                = 0x03,  // true
+
+	//CS_                = 0x04,
+	//CS_                = 0x05,
+	//CS_                = 0x06,
+	//CS_                = 0x07,
+
+	//CS_                = 0x08,
+	//CS_                = 0x09,
+	CS_FLOAT             = 0x0a,
+	CS_DOUBLE            = 0x0b,
+	CS_UINT_8            = 0x0c,
+	CS_UINT_16           = 0x0d,
+	CS_UINT_32           = 0x0e,
+	CS_UINT_64           = 0x0f,
+	CS_INT_8             = 0x10,
+	CS_INT_16            = 0x11,
+	CS_INT_32            = 0x12,
+	CS_INT_64            = 0x13,
+
+	//CS_                = 0x14,
+	//CS_                = 0x15,
+	//CS_BIG_INT_16        = 0x16,
+	//CS_BIG_INT_32        = 0x17,
+	//CS_BIG_FLOAT_16      = 0x18,
+	//CS_BIG_FLOAT_32      = 0x19,
+	CS_RAW_16            = 0x1a,
+	CS_RAW_32            = 0x1b,
+	CS_ARRAY_16          = 0x1c,
+	CS_ARRAY_32          = 0x1d,
+	CS_MAP_16            = 0x1e,
+	CS_MAP_32            = 0x1f,
+
+	//ACS_BIG_INT_VALUE,
+	//ACS_BIG_FLOAT_VALUE,
+	ACS_RAW_VALUE,
+} msgpack_unpack_state;
+
+
+typedef enum {
+	CT_ARRAY_ITEM,
+	CT_MAP_KEY,
+	CT_MAP_VALUE,
+} msgpack_container_type;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/unpack_define.h */
+

python/msgpack/unpack_template.h

@@ -0,0 +1,366 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#ifndef msgpack_unpack_func
+#error msgpack_unpack_func template is not defined
+#endif
+
+#ifndef msgpack_unpack_callback
+#error msgpack_unpack_callback template is not defined
+#endif
+
+#ifndef msgpack_unpack_struct
+#error msgpack_unpack_struct template is not defined
+#endif
+
+#ifndef msgpack_unpack_struct_decl
+#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name)
+#endif
+
+#ifndef msgpack_unpack_object
+#error msgpack_unpack_object type is not defined
+#endif
+
+#ifndef msgpack_unpack_user
+#error msgpack_unpack_user type is not defined
+#endif
+
+
+msgpack_unpack_struct_decl(_stack) {
+	msgpack_unpack_object obj;
+	size_t count;
+	unsigned int ct;
+
+    union {
+        size_t curr;
+        msgpack_unpack_object map_key;
+    };
+};
+
+msgpack_unpack_struct_decl(_context) {
+	msgpack_unpack_user user;
+	unsigned int cs;
+	unsigned int trail;
+	unsigned int top;
+	msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE];
+};
+
+
+msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
+{
+	ctx->cs = CS_HEADER;
+	ctx->trail = 0;
+	ctx->top = 0;
+	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
+}
+
+msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
+{
+	return (ctx)->stack[0].obj;
+}
+
+
+msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+{
+	assert(len >= *off);
+
+	const unsigned char* p = (unsigned char*)data + *off;
+	const unsigned char* const pe = (unsigned char*)data + len;
+	const void* n = NULL;
+
+	unsigned int trail = ctx->trail;
+	unsigned int cs = ctx->cs;
+	unsigned int top = ctx->top;
+	msgpack_unpack_struct(_stack)* stack = ctx->stack;
+	msgpack_unpack_user* user = &ctx->user;
+
+	msgpack_unpack_object obj;
+	msgpack_unpack_struct(_stack)* c = NULL;
+
+	int ret;
+
+#define push_simple_value(func) \
+	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
+	goto _push
+#define push_fixed_value(func, arg) \
+	if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
+	goto _push
+#define push_variable_value(func, base, pos, len) \
+	if(msgpack_unpack_callback(func)(user, \
+		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+	goto _push
+
+#define again_fixed_trail(_cs, trail_len) \
+	trail = trail_len; \
+	cs = _cs; \
+	goto _fixed_trail_again
+#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
+	trail = trail_len; \
+	if(trail == 0) { goto ifzero; } \
+	cs = _cs; \
+	goto _fixed_trail_again
+
+#define start_container(func, count_, ct_) \
+	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+	if((count_) == 0) { obj = stack[top].obj; goto _push; } \
+	if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \
+	stack[top].ct = ct_; \
+    stack[top].curr = 0; \
+	stack[top].count = count_; \
+	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
+	/*printf("stack push %d\n", top);*/ \
+	++top; \
+	goto _header_again
+
+#define NEXT_CS(p) \
+	((unsigned int)*p & 0x1f)
+
+#define PTR_CAST_8(ptr)   (*(uint8_t*)ptr)
+#define PTR_CAST_16(ptr)  msgpack_betoh16(*(uint16_t*)ptr)
+#define PTR_CAST_32(ptr)  msgpack_betoh32(*(uint32_t*)ptr)
+#define PTR_CAST_64(ptr)  msgpack_betoh64(*(uint64_t*)ptr)
+
+	if(p == pe) { goto _out; }
+	do {
+		switch(cs) {
+		case CS_HEADER:
+			switch(*p) {
+			case 0x00 ... 0x7f:  // Positive Fixnum
+				push_fixed_value(_uint8, *(uint8_t*)p);
+			case 0xe0 ... 0xff:  // Negative Fixnum
+				push_fixed_value(_int8, *(int8_t*)p);
+			case 0xc0 ... 0xdf:  // Variable
+				switch(*p) {
+				case 0xc0:  // nil
+					push_simple_value(_nil);
+				//case 0xc1:  // string
+				//	again_terminal_trail(NEXT_CS(p), p+1);
+				case 0xc2:  // false
+					push_simple_value(_false);
+				case 0xc3:  // true
+					push_simple_value(_true);
+				//case 0xc4:
+				//case 0xc5:
+				//case 0xc6:
+				//case 0xc7:
+				//case 0xc8:
+				//case 0xc9:
+				case 0xca:  // float
+				case 0xcb:  // double
+				case 0xcc:  // unsigned int  8
+				case 0xcd:  // unsigned int 16
+				case 0xce:  // unsigned int 32
+				case 0xcf:  // unsigned int 64
+				case 0xd0:  // signed int  8
+				case 0xd1:  // signed int 16
+				case 0xd2:  // signed int 32
+				case 0xd3:  // signed int 64
+					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+				//case 0xd4:
+				//case 0xd5:
+				//case 0xd6:  // big integer 16
+				//case 0xd7:  // big integer 32
+				//case 0xd8:  // big float 16
+				//case 0xd9:  // big float 32
+				case 0xda:  // raw 16
+				case 0xdb:  // raw 32
+				case 0xdc:  // array 16
+				case 0xdd:  // array 32
+				case 0xde:  // map 16
+				case 0xdf:  // map 32
+					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
+				default:
+					goto _failed;
+				}
+			case 0xa0 ... 0xbf:  // FixRaw
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
+			case 0x90 ... 0x9f:  // FixArray
+				start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
+			case 0x80 ... 0x8f:  // FixMap
+				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
+
+			default:
+				goto _failed;
+			}
+			// end CS_HEADER
+
+
+		_fixed_trail_again:
+			++p;
+
+		default:
+			if((size_t)(pe - p) < trail) { goto _out; }
+			n = p;  p += trail - 1;
+			switch(cs) {
+			//case CS_
+			//case CS_
+			case CS_FLOAT: {
+					union { uint32_t num; char buf[4]; } f;
+					f.num = PTR_CAST_32(n);  // FIXME
+					push_fixed_value(_float, *((float*)f.buf)); }
+			case CS_DOUBLE: {
+					union { uint64_t num; char buf[8]; } f;
+					f.num = PTR_CAST_64(n);  // FIXME
+					push_fixed_value(_double, *((double*)f.buf)); }
+			case CS_UINT_8:
+				push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n));
+			case CS_UINT_16:
+				push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n));
+			case CS_UINT_32:
+				push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n));
+			case CS_UINT_64:
+				push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n));
+
+			case CS_INT_8:
+				push_fixed_value(_int8, (int8_t)PTR_CAST_8(n));
+			case CS_INT_16:
+				push_fixed_value(_int16, (int16_t)PTR_CAST_16(n));
+			case CS_INT_32:
+				push_fixed_value(_int32, (int32_t)PTR_CAST_32(n));
+			case CS_INT_64:
+				push_fixed_value(_int64, (int64_t)PTR_CAST_64(n));
+
+			//case CS_
+			//case CS_
+			//case CS_BIG_INT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero);
+			//case CS_BIG_INT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero);
+			//case ACS_BIG_INT_VALUE:
+			//_big_int_zero:
+			//	// FIXME
+			//	push_variable_value(_big_int, data, n, trail);
+
+			//case CS_BIG_FLOAT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero);
+			//case CS_BIG_FLOAT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero);
+			//case ACS_BIG_FLOAT_VALUE:
+			//_big_float_zero:
+			//	// FIXME
+			//	push_variable_value(_big_float, data, n, trail);
+
+			case CS_RAW_16:
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero);
+			case CS_RAW_32:
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero);
+			case ACS_RAW_VALUE:
+			_raw_zero:
+				push_variable_value(_raw, data, n, trail);
+
+			case CS_ARRAY_16:
+				start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM);
+			case CS_ARRAY_32:
+				/* FIXME security guard */
+				start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM);
+
+			case CS_MAP_16:
+				start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY);
+			case CS_MAP_32:
+				/* FIXME security guard */
+				start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY);
+
+			default:
+				goto _failed;
+			}
+		}
+
+_push:
+	if(top == 0) { goto _finish; }
+	c = &stack[top-1];
+	switch(c->ct) {
+	case CT_ARRAY_ITEM:
+		if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; }
+		if(++c->curr == c->count) {
+			obj = c->obj;
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
+		}
+		goto _header_again;
+	case CT_MAP_KEY:
+		c->map_key = obj;
+		c->ct = CT_MAP_VALUE;
+		goto _header_again;
+	case CT_MAP_VALUE:
+		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(--c->count == 0) {
+			obj = c->obj;
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
+		}
+		c->ct = CT_MAP_KEY;
+		goto _header_again;
+
+	default:
+		goto _failed;
+	}
+
+_header_again:
+		cs = CS_HEADER;
+		++p;
+	} while(p != pe);
+	goto _out;
+
+
+_finish:
+	stack[0].obj = obj;
+	++p;
+	ret = 1;
+	/*printf("-- finish --\n"); */
+	goto _end;
+
+_failed:
+	/*printf("** FAILED **\n"); */
+	ret = -1;
+	goto _end;
+
+_out:
+	ret = 0;
+	goto _end;
+
+_end:
+	ctx->cs = cs;
+	ctx->trail = trail;
+	ctx->top = top;
+	*off = p - (const unsigned char*)data;
+
+	return ret;
+}
+
+
+#undef msgpack_unpack_func
+#undef msgpack_unpack_callback
+#undef msgpack_unpack_struct
+#undef msgpack_unpack_object
+#undef msgpack_unpack_user
+
+#undef push_simple_value
+#undef push_fixed_value
+#undef push_variable_value
+#undef again_fixed_trail
+#undef again_fixed_trail_if_zero
+#undef start_container
+
+#undef NEXT_CS
+#undef PTR_CAST_8
+#undef PTR_CAST_16
+#undef PTR_CAST_32
+#undef PTR_CAST_64
+

python/setup.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from distutils.core import setup, Extension
+#from Cython.Distutils import build_ext
+import os
+
+version = '0.1.1'
+
+msgpack_mod = Extension('msgpack._msgpack',
+                        #sources=['msgpack/_msgpack.pyx']
+                        sources=['msgpack/_msgpack.c']
+                        )
+
+desc = 'MessagePack (de)serializer.'
+long_desc = desc + """
+
+MessagePack_ (de)serializer for Python.
+
+.. _MessagePack: http://msgpack.sourceforge.jp/
+
+What's MessagePack? (from http://msgpack.sourceforge.jp/)
+
+ MessagePack is a binary-based efficient data interchange format that is
+ focused on high performance. It is like JSON, but very fast and small.
+"""
+
+setup(name='msgpack',
+      author='Naoki INADA',
+      author_email='songofacandy@gmail.com',
+      version=version,
+      #cmdclass={'build_ext': build_ext},
+      ext_modules=[msgpack_mod],
+      packages=['msgpack'],
+      description=desc,
+      long_description=long_desc,
+      classifiers=[
+          'Development Status :: 4 - Beta',
+          'Intended Audience :: Developers',
+          'License :: OSI Approved :: Apache Software License',
+          ]
+      )

python/setup_dev.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from distutils.core import setup, Extension
+from Cython.Distutils import build_ext
+import os
+
+version = '0.1.1dev'
+
+msgpack_mod = Extension('msgpack._msgpack',
+                        sources=['msgpack/_msgpack.pyx']
+                        )
+
+desc = 'MessagePack (de)serializer.'
+long_desc = desc + """
+
+MessagePack_ (de)serializer for Python.
+
+.. _MessagePack: http://msgpack.sourceforge.jp/
+
+What's MessagePack? (from http://msgpack.sourceforge.jp/)
+
+ MessagePack is a binary-based efficient data interchange format that is
+ focused on high performance. It is like JSON, but very fast and small.
+"""
+
+setup(name='msgpack',
+      author='Naoki INADA',
+      author_email='songofacandy@gmail.com',
+      version=version,
+      cmdclass={'build_ext': build_ext},
+      ext_modules=[msgpack_mod],
+      packages=['msgpack'],
+      description=desc,
+      long_description=long_desc,
+      classifiers=[
+          'Development Status :: 4 - Beta',
+          'Intended Audience :: Developers',
+          'License :: OSI Approved :: Apache Software License',
+          ]
+      )

python/test/test_case.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packs, unpacks
+
+def check(length, obj):
+    v = packs(obj)
+    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
+    assert_equal(unpacks(v), obj)
+
+def test_1():
+    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
+              -((1<<5)-1), -(1<<5)]:
+        check(1, o)
+
+def test_2():
+    for o in [1 << 7, (1 << 8) - 1,
+              -((1<<5)+1), -(1<<7)
+             ]:
+        check(2, o)
+
+def test_3():
+    for o in [1 << 8, (1 << 16) - 1,
+              -((1<<7)+1), -(1<<15)]:
+        check(3, o)
+
+def test_5():
+    for o in [1 << 16, (1 << 32) - 1,
+              -((1<<15)+1), -(1<<31)]:
+        check(5, o)
+
+def test_9():
+    for o in [1 << 32, (1 << 64) - 1,
+              -((1<<31)+1), -(1<<63),
+              1.0, 0.1, -0.1, -1.0]:
+        check(9, o)
+
+
+def check_raw(overhead, num):
+    check(num + overhead, " " * num)
+
+def test_fixraw():
+    check_raw(1, 0)
+    check_raw(1, (1<<5) - 1)
+
+def test_raw16():
+    check_raw(3, 1<<5)
+    check_raw(3, (1<<16) - 1)
+
+def test_raw32():
+    check_raw(5, 1<<16)
+
+
+def check_array(overhead, num):
+    check(num + overhead, [None] * num)
+
+def test_fixarray():
+    check_array(1, 0)
+    check_array(1, (1 << 4) - 1)
+
+def test_array16():
+    check_array(3, 1 << 4)
+    check_array(3, (1<<16)-1)
+
+def test_array32():
+    check_array(5, (1<<16))
+
+
+def match(obj, buf):
+    assert_equal(packs(obj), buf)
+    assert_equal(unpacks(buf), obj)
+
+def test_match():
+    cases = [
+        (None, '\xc0'),
+        (False, '\xc2'),
+        (True, '\xc3'),
+        (0, '\x00'),
+        (127, '\x7f'),
+        (128, '\xcc\x80'),
+        (256, '\xcd\x01\x00'),
+        (-1, '\xff'),
+        (-33, '\xd0\xdf'),
+        (-129, '\xd1\xff\x7f'),
+        ({1:1}, '\x81\x01\x01'),
+        (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ([], '\x90'),
+        (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
+        (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
+        ({}, '\x80'),
+        (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
+        (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
+        ]
+
+    for v, p in cases:
+        match(v, p)
+
+if __name__ == '__main__':
+    main()

python/test/test_format.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import unpacks
+
+def check(src, should):
+    assert_equal(unpacks(src), should)
+
+def testSimpleValue():
+    check("\x93\xc0\xc2\xc3", 
+            [None, False, True])
+
+def testFixnum():
+    check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
+          [[0,64,127], [-32,-16,-1]]
+          )
+
+def testFixArray():
+    check("\x92\x90\x91\x91\xc0",
+          [[],[[None]]],
+          )
+
+def testFixRaw():
+    check("\x94\xa0\xa1a\xa2bc\xa3def",
+          ["", "a", "bc", "def"],
+          )
+
+def testFixMap():
+    check(
+          "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          {False: {None: None}, True:{None:{}}},
+          )
+
+def testUnsignedInt():
+    check(
+          "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          "\xce\xff\xff\xff\xff",
+          [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
+          )
+
+def testSignedInt():
+    check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          "\xd2\xff\xff\xff\xff",
+          [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1])
+
+def testRaw():
+    check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        ["", "a", "ab", "", "a", "ab"])
+
+def testArray():
+    check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+        "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+        "\xc2\xc3",
+        [[], [None], [False,True], [], [None], [False,True]])
+
+def testMap():
+    check(
+        "\x96"
+            "\xde\x00\x00"
+            "\xde\x00\x01\xc0\xc2"
+            "\xde\x00\x02\xc0\xc2\xc3\xc2"
+            "\xdf\x00\x00\x00\x00"
+            "\xdf\x00\x00\x00\x01\xc0\xc2"
+            "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        [{}, {None: False}, {True: False, None: False}, {},
+            {None: False}, {True: False, None: False}])
+
+if __name__ == '__main__':
+    main()

python/test/test_pack.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def check(data):
+    re = unpacks(packs(data))
+    assert_equal(re, data)
+
+def testPack():
+    test_data = [
+            0, 1, 127, 128, 255, 256, 65535, 65536,
+            -1, -32, -33, -128, -129, -32768, -32769,
+            1.0,
+        "", "a", "a"*31, "a"*32,
+        None, True, False,
+        [], [[]], [[], None], 
+        {None: 0}, 
+        (1<<23), 
+        ]
+    for td in test_data:
+        check(td)
+
+if __name__ == '__main__':
+    main()

python/test/test_sequnpack.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals, print_function
+
+from msgpack import Unpacker
+
+def test_foobar():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord('f')
+    assert unpacker.unpack() == ord('o')
+    assert unpacker.unpack() == ord('o')
+    assert unpacker.unpack() == ord('b')
+    assert unpacker.unpack() == ord('a')
+    assert unpacker.unpack() == ord('r')
+    try:
+        o = unpacker.unpack()
+        print("Oops!", o)
+        assert 0
+    except StopIteration:
+        assert 1
+    else:
+        assert 0
+    unpacker.feed(b'foo')
+    unpacker.feed(b'bar')
+
+    k = 0
+    for o, e in zip(unpacker, b'foobarbaz'):
+        assert o == ord(e)
+        k += 1
+    assert k == len(b'foobar')
+
+if __name__ == '__main__':
+    test_foobar()
+

ruby/test_case.rb

@@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase
 
 	def match(obj, buf)
 		assert_equal(obj.to_msgpack, buf)
+		assert_equal(MessagePack::unpack(buf), obj)
 	end
 end