diff --git a/python/msgpack.pyx b/python/msgpack.pyx index 68853176..ceae9b65 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -3,15 +3,17 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(int) void free(void*) + cdef extern from "string.h": int memcpy(char*dst, char*src, unsigned int size) -cdef extern from "msgpack/pack.h": +cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) struct msgpack_packer: @@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "msgpack/unpack.h": +cdef extern from "unpack.h": ctypedef struct msgpack_unpacker @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def __call__(self, object o): + def pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -109,6 +111,9 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, int): intval = o msgpack_pack_long_long(&self.pk, intval) @@ -120,24 +125,21 @@ cdef class Packer: msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, unicode): - # todo - pass + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, dict): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self(k) - self(v) - elif isinstance(o, tuple): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): msgpack_pack_array(&self.pk, len(o)) for v in o: - self(v) - elif isinstance(o, list): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self(v) - elif hasattr(o, "__msgpack__"): - o.__msgpack__(self) + self.pack(v) else: + # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): @@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 +cdef extern from "msgpack/zone.h": + ctypedef struct msgpack_zone + +cdef extern from "unpack.c": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + void template_init(template_context* ctx) + PyObject* template_data(template_context* ctx) + + cdef class Unpacker: def __init__(self): pass - def unpack(strm): - pass + + def unpack(self, bytes_): + cdef const_char_ptr p = bytes_ + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(bytes_), &off) + return template_data(&ctx) diff --git a/python/pack.h b/python/pack.h new file mode 100644 index 00000000..4a336d5d --- /dev/null +++ b/python/pack.h @@ -0,0 +1,130 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#elif +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" +#include "msgpack/object.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); + +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); + +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); + +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/pack.h */ + diff --git a/python/unpack.h b/python/unpack.h new file mode 100644 index 00000000..d5783602 --- /dev/null +++ b/python/unpack.h @@ -0,0 +1,397 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include + +#include "Python.h" + + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static void template_init(template_context* ctx); + +static msgpack_unpack_object template_data(template_context* ctx); + +static int template_execute(template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ PyObject *o = Py_None; Py_INCREF(o); return o; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_None; Py_INCREF(o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_True; Py_INCREF(o); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_False; Py_INCREF(o); return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" + + +#if 0 +#define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + + + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buffer); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buffer); + return false; + } + + mpac->buffer = buffer; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->parsed = 0; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->ctx = ctx; + + init_count(mpac->buffer); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = false; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buffer); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buffer, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buffer = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + free(tmp); + return false; + } + CTX_REFERENCED(mpac) = false; + } else { + decl_count(mpac->buffer); + } + + mpac->buffer = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), + mpac->buffer, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; +} + +msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = r; + + return old; +} + +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + return false; + } + CTX_REFERENCED(mpac) = false; + + incr_count(mpac->buffer); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag + mpac->parsed = 0; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_unpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + ctx.user.z = z; + ctx.user.referenced = false; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} +#endif