merge to trunk

README

@@ -8,7 +8,7 @@ Binary-based efficient data interchange format.
   MessagePack is only tested on Linux and Mac OS X, but it may run on other
   UNIX-like platforms.
 
-  Following programs is required to build:
+  Following programs are required to build:
     - gcc >= 4.1 with C++ support
     - ruby >= 1.8   (ruby is used as a preprocessor)
 

c/Makefile.am

@@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la
 libmsgpackc_la_SOURCES = \
 		unpack.c \
 		object.c \
+		vrefbuffer.c \
 		zone.c
 
 nobase_include_HEADERS = \
 		msgpack.h \
 		msgpack/sbuffer.h \
+		msgpack/vrefbuffer.h \
 		msgpack/pack.h \
 		msgpack/unpack.h \
 		msgpack/object.h \

c/vrefbuffer.c

@@ -0,0 +1,135 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#include "msgpack/vrefbuffer.h"
+#include <stdlib.h>
+#include <string.h>
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size)
+{
+	if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) {
+		chunk_size = 72;
+	} else {
+		chunk_size -= sizeof(msgpack_vrefbuffer_chunk);
+	}
+
+	vbuf->chunk_size = chunk_size;
+	vbuf->ref_size = ref_size;
+
+	// glibcは72バイト以下のmallocが高速
+	size_t nfirst = (sizeof(struct iovec) < 72/2) ?
+			72 / sizeof(struct iovec) : 8;
+
+	struct iovec* array = (struct iovec*)malloc(
+			sizeof(struct iovec) * nfirst);
+	if(array == NULL) {
+		return false;
+	}
+
+	vbuf->tail  = array;
+	vbuf->end   = array + nfirst;
+	vbuf->array = array;
+
+	vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc(
+			chunk_size + sizeof(msgpack_vrefbuffer_chunk));
+	if(vbuf->chunk == NULL) {
+		free(array);
+		return false;
+	}
+
+	vbuf->chunk->next = NULL;
+	vbuf->chunk->free = chunk_size;
+
+	return true;
+}
+
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf)
+{
+	msgpack_vrefbuffer_chunk* c = vbuf->chunk;
+	while(true) {
+		msgpack_vrefbuffer_chunk* n = c->next;
+		free(c);
+		if(n) {
+			c = n;
+		} else {
+			break;
+		}
+	}
+	free(vbuf->array);
+}
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	if(vbuf->tail == vbuf->end) {
+		const size_t nused = vbuf->end - vbuf->array;
+		const size_t nnext = nused * 2;
+
+		struct iovec* nvec = (struct iovec*)realloc(
+				vbuf->array, sizeof(struct iovec)*nnext);
+		if(nvec == NULL) {
+			return -1;
+		}
+
+		vbuf->array = nvec;
+		vbuf->end   = nvec + nnext;
+		vbuf->tail  = nvec + nused;
+	}
+
+	vbuf->tail->iov_base = (char*)buf;
+	vbuf->tail->iov_len  = len;
+	++vbuf->tail;
+
+	return 0;
+}
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer_chunk* chunk = vbuf->chunk;
+	size_t cur_size = vbuf->chunk_size;
+
+	if(chunk->free < len) {
+		cur_size = (cur_size > len) ? cur_size : len;
+
+		chunk = (msgpack_vrefbuffer_chunk*)malloc(
+				cur_size + sizeof(msgpack_vrefbuffer_chunk));
+		if(chunk == NULL) {
+			return -1;
+		}
+
+		chunk->free = cur_size;
+		chunk->next = vbuf->chunk;
+		vbuf->chunk = chunk;
+	}
+
+	char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk)
+		+ (cur_size - chunk->free);
+
+	memcpy(m, buf, len);
+	chunk->free -= len;
+
+	if(vbuf->tail != vbuf->array && m ==
+			(const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) {
+		(vbuf->tail-1)->iov_len += len;
+		return 0;
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, m, len);
+	}
+}
+

c/vrefbuffer.h

@@ -0,0 +1,97 @@
+/*
+ * MessagePack for C zero-copy buffer implementation
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_VREFBUFFER_H__
+#define MSGPACK_VREFBUFFER_H__
+
+#include "msgpack/zone.h"
+#include <sys/uio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_VREFBUFFER_REF_SIZE
+#define MSGPACK_VREFBUFFER_REF_SIZE 32
+#endif
+
+#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
+#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048
+#endif
+
+typedef struct msgpack_vrefbuffer_chunk {
+	size_t free;
+	struct msgpack_vrefbuffer_chunk* next;
+	/* data ... */
+} msgpack_vrefbuffer_chunk;
+
+typedef struct msgpack_vrefbuffer {
+	size_t chunk_size;
+	size_t ref_size;
+
+	struct iovec* tail;
+	struct iovec* end;
+	struct iovec* array;
+
+	msgpack_vrefbuffer_chunk* chunk;
+} msgpack_vrefbuffer;
+
+
+bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
+		size_t ref_size, size_t chunk_size);
+void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
+
+static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len);
+
+static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref);
+static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref);
+
+int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf,
+		const char* buf, unsigned int len);
+
+
+int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len)
+{
+	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data;
+
+	if(len < vbuf->ref_size) {
+		return msgpack_vrefbuffer_append_copy(vbuf, buf, len);
+	} else {
+		return msgpack_vrefbuffer_append_ref(vbuf, buf, len);
+	}
+}
+
+const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref)
+{
+	return vref->array;
+}
+
+size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref)
+{
+	return vref->tail - vref->array;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/vrefbuffer.h */
+

c/zone.c

@@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s
 
 	msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc(
 			sizeof(msgpack_zone_chunk) * nfirst);
-	if(!array) {
+	if(array == NULL) {
 		return false;
 	}
 

c/zone.h

@@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size);
 void msgpack_zone_free(msgpack_zone* zone);
 
 static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size);
+static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size);
 
 static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone,
 		void (*func)(void* data), void* data);
@@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone);
 
 void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size);
 
-void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size)
 {
-	size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1);
-
 	msgpack_zone_chunk* chunk = zone->chunk_array.tail;
 
 	if(chunk->free < size) {
@@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
 	return ptr;
 }
 
+void* msgpack_zone_malloc(msgpack_zone* zone, size_t size)
+{
+	return msgpack_zone_malloc_no_align(zone,
+			((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1));
+}
+
 
 bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone,
 		void (*func)(void* data), void* data);

configure.in

@@ -1,6 +1,6 @@
 AC_INIT(msgpack/unpack_template.h)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.3.1)
+AM_INIT_AUTOMAKE(msgpack, 0.3.2)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

cpp/Makefile.am

@@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \
 nobase_include_HEADERS = \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
+		msgpack/vrefbuffer.hpp \
 		msgpack/pack.hpp \
 		msgpack/unpack.hpp \
 		msgpack/object.hpp \

cpp/object.hpp

@@ -80,13 +80,13 @@ struct object {
 	type::object_type type;
 	union_type via;
 
-	bool is_nil() { return type == type::NIL; }
+	bool is_nil() const { return type == type::NIL; }
 
 	template <typename T>
-	T as();
+	T as() const;
 
 	template <typename T>
-	void convert(T* v);
+	void convert(T* v) const;
 
 	object();
 	object(msgpack_object obj);
@@ -96,7 +96,7 @@ private:
 	struct implicit_type;
 
 public:
-	implicit_type convert();
+	implicit_type convert() const;
 };
 
 struct object_kv {
@@ -201,25 +201,25 @@ inline object::operator msgpack_object()
 }
 
 
-inline object::implicit_type object::convert()
+inline object::implicit_type object::convert() const
 {
 	return implicit_type(*this);
 }
 
 template <typename T>
-inline T object::as()
+inline void object::convert(T* v) const
+{
+	*this >> *v;
+}
+
+template <typename T>
+inline T object::as() const
 {
 	T v;
 	convert(&v);
 	return v;
 }
 
-template <typename T>
-inline void object::convert(T* v)
-{
-	*this >> *v;
-}
-
 
 // obsolete
 template <typename T>

cpp/type/map.hpp

@@ -80,13 +80,13 @@ inline std::map<K, V> operator>> (object o, std::map<K, V>& v)
 	for(; p != pend; ++p) {
 		K key;
 		p->key.convert(&key);
-		typename std::map<K,V>::iterator it(v.find(key));
+		typename std::map<K,V>::iterator it(v.lower_bound(key));
-		if(it != v.end()) {
+		if(it != v.end() && !(key < it->first)) {
+			p->val.convert(&it->second);
+		} else {
 			V val;
 			p->val.convert(&val);
-			it->insert( std::pair<K,V>(key, val) );
+			v.insert(it, std::pair<K,V>(key, val));
-		} else {
-			p->val.convert(&it->second);
 		}
 	}
 	return v;

cpp/vrefbuffer.hpp

@@ -0,0 +1,85 @@
+//
+// MessagePack for C++ zero-copy buffer implementation
+//
+// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_VREFBUFFER_HPP__
+#define MSGPACK_VREFBUFFER_HPP__
+
+#include "msgpack/vrefbuffer.h"
+#include <stdexcept>
+
+namespace msgpack {
+
+
+class vrefbuffer : public msgpack_vrefbuffer {
+public:
+	vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE,
+			size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE)
+	{
+		msgpack_vrefbuffer_init(this, ref_size, chunk_size);
+	}
+
+	~vrefbuffer()
+	{
+		msgpack_vrefbuffer_destroy(this);
+	}
+
+public:
+	void write(const char* buf, unsigned int len)
+	{
+		if(len < base::ref_size) {
+			append_copy(buf, len);
+		} else {
+			append_ref(buf, len);
+		}
+	}
+
+	void append_ref(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	void append_copy(const char* buf, size_t len)
+	{
+		if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) {
+			throw std::bad_alloc();
+		}
+	}
+
+	const struct iovec* vector() const
+	{
+		return msgpack_vrefbuffer_vec(this);
+	}
+
+	size_t vector_size() const
+	{
+		return msgpack_vrefbuffer_veclen(this);
+	}
+
+private:
+	typedef msgpack_vrefbuffer base;
+
+private:
+	vrefbuffer(const vrefbuffer&);
+};
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/vrefbuffer.hpp */
+

cpp/zone.hpp.erb

@@ -34,6 +34,7 @@ public:
 
 public:
 	void* malloc(size_t size);
+	void* malloc_no_align(size_t size);
 
 	void push_finalizer(void (*func)(void*), void* data);
 
@@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size)
 	return ptr;
 }
 
+inline void* zone::malloc_no_align(size_t size)
+{
+	void* ptr = msgpack_zone_malloc_no_align(this, size);
+	if(!ptr) {
+		throw std::bad_alloc();
+	}
+	return ptr;
+}
+
 inline void zone::push_finalizer(void (*func)(void*), void* data)
 {
 	if(!msgpack_zone_push_finalizer(this, func, data)) {

ruby/test_case.rb

@@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase
 
 	def match(obj, buf)
 		assert_equal(obj.to_msgpack, buf)
+		assert_equal(MessagePack::unpack(buf), obj)
 	end
 end