diff --git a/python/msgpack.pyx b/python/msgpack.pyx index ceae9b65..c454c5de 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -1,5 +1,6 @@ # coding: utf-8 +from cStringIO import StringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -32,13 +33,15 @@ cdef extern from "pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "unpack.h": - ctypedef struct msgpack_unpacker - cdef int BUFF_SIZE=2*1024 cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ cdef char* buff cdef unsigned int length cdef unsigned int allocated @@ -46,11 +49,6 @@ cdef class Packer: cdef object strm def __init__(self, strm, int size=0): - """Make packer that pack data into strm. - - strm must have `write(bytes)` method. - size specifies local buffer size. - """ if size <= 0: size = BUFF_SIZE @@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 -cdef extern from "msgpack/zone.h": - ctypedef struct msgpack_zone +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() -cdef extern from "unpack.c": +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": ctypedef struct template_context: pass - int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) void template_init(template_context* ctx) PyObject* template_data(template_context* ctx) -cdef class Unpacker: - def __init__(self): - pass +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) - def unpack(self, bytes_): - cdef const_char_ptr p = bytes_ - cdef template_context ctx - cdef size_t off = 0 - template_init(&ctx) - template_execute(&ctx, p, len(bytes_), &off) - return template_data(&ctx) +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/python/pack.h b/python/pack.h index 4a336d5d..f3935fbb 100644 --- a/python/pack.h +++ b/python/pack.h @@ -15,9 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_H__ -#define MSGPACK_PACK_H__ - #if _MSC_VER typedef signed char uint8_t; typedef unsigned char uint8_t; @@ -27,14 +24,13 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef long long int64_t; typedef unsigned long long uint64_t; -#elif +#else #include #endif #include #include #include "msgpack/pack_define.h" -#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -48,44 +44,42 @@ typedef struct msgpack_packer { msgpack_packer_write callback; } msgpack_packer; -static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static void msgpack_packer_free(msgpack_packer* pk); +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); -static int msgpack_pack_short(msgpack_packer* pk, short d); -static int msgpack_pack_int(msgpack_packer* pk, int d); -static int msgpack_pack_long(msgpack_packer* pk, long d); -static int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_float(msgpack_packer* pk, float d); -static int msgpack_pack_double(msgpack_packer* pk, double d); +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_nil(msgpack_packer* pk); -static int msgpack_pack_true(msgpack_packer* pk); -static int msgpack_pack_false(msgpack_packer* pk); +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk) #ifdef __cplusplus } #endif - -#endif /* msgpack/pack.h */ - diff --git a/python/unpack.h b/python/unpack.h index d5783602..eb72001d 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -15,12 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" #include "msgpack/unpack_define.h" -#include - -#include "Python.h" - typedef struct { int reserved; @@ -44,11 +39,11 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static void template_init(template_context* ctx); +static inline void template_init(template_context* ctx); -static msgpack_unpack_object template_data(template_context* ctx); +static inline msgpack_unpack_object template_data(template_context* ctx); -static int template_execute(template_context* ctx, +static inline int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); @@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } #include "msgpack/unpack_template.h" - - -#if 0 -#define CTX_CAST(m) ((template_context*)(m)) -#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - - - -bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) -{ - if(initial_buffer_size < COUNTER_SIZE) { - initial_buffer_size = COUNTER_SIZE; - } - - char* buffer = (char*)malloc(initial_buffer_size); - if(buffer == NULL) { - return false; - } - - void* ctx = malloc(sizeof(template_context)); - if(ctx == NULL) { - free(buffer); - return false; - } - - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { - free(ctx); - free(buffer); - return false; - } - - mpac->buffer = buffer; - mpac->used = COUNTER_SIZE; - mpac->free = initial_buffer_size - mpac->used; - mpac->off = COUNTER_SIZE; - mpac->parsed = 0; - mpac->initial_buffer_size = initial_buffer_size; - mpac->z = z; - mpac->ctx = ctx; - - init_count(mpac->buffer); - - template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = false; - - return true; -} - -void msgpack_unpacker_destroy(msgpack_unpacker* mpac) -{ - msgpack_zone_free(mpac->z); - free(mpac->ctx); - decl_count(mpac->buffer); -} - - -msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) -{ - msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); - if(mpac == NULL) { - return NULL; - } - - if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { - free(mpac); - return NULL; - } - - return mpac; -} - -void msgpack_unpacker_free(msgpack_unpacker* mpac) -{ - msgpack_unpacker_destroy(mpac); - free(mpac); -} - -bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) -{ - if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 - && !CTX_REFERENCED(mpac)) { - // rewind buffer - mpac->free += mpac->used - COUNTER_SIZE; - mpac->used = COUNTER_SIZE; - mpac->off = COUNTER_SIZE; - - if(mpac->free >= size) { - return true; - } - } - - if(mpac->off == COUNTER_SIZE) { - size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE - while(next_size < size + mpac->used) { - next_size *= 2; - } - - char* tmp = (char*)realloc(mpac->buffer, next_size); - if(tmp == NULL) { - return false; - } - - mpac->buffer = tmp; - mpac->free = next_size - mpac->used; - - } else { - size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = mpac->used - mpac->off; - while(next_size < size + not_parsed + COUNTER_SIZE) { - next_size *= 2; - } - - char* tmp = (char*)malloc(next_size); - if(tmp == NULL) { - return false; - } - - init_count(tmp); - - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - free(tmp); - return false; - } - CTX_REFERENCED(mpac) = false; - } else { - decl_count(mpac->buffer); - } - - mpac->buffer = tmp; - mpac->used = not_parsed + COUNTER_SIZE; - mpac->free = next_size - mpac->used; - mpac->off = COUNTER_SIZE; - } - - return true; -} - -int msgpack_unpacker_execute(msgpack_unpacker* mpac) -{ - size_t off = mpac->off; - int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buffer, mpac->used, &mpac->off); - if(mpac->off > off) { - mpac->parsed += mpac->off - off; - } - return ret; -} - -msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) -{ - return template_data(CTX_CAST(mpac->ctx)); -} - -msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) -{ - if(!msgpack_unpacker_flush_zone(mpac)) { - return false; - } - - msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(r == NULL) { - return NULL; - } - - msgpack_zone* old = mpac->z; - mpac->z = r; - - return old; -} - -void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) -{ - msgpack_zone_clear(mpac->z); -} - -bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) -{ - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - return false; - } - CTX_REFERENCED(mpac) = false; - - incr_count(mpac->buffer); - } - - return true; -} - -void msgpack_unpacker_reset(msgpack_unpacker* mpac) -{ - template_init(CTX_CAST(mpac->ctx)); - // don't reset referenced flag - mpac->parsed = 0; -} - - -msgpack_unpack_return -msgpack_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_unpack_object* result) -{ - template_context ctx; - template_init(&ctx); - - ctx.user.z = z; - ctx.user.referenced = false; - - size_t noff = 0; - if(off != NULL) { noff = *off; } - - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; - } - - if(off != NULL) { *off = noff; } - - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; - } - - *result = template_data(&ctx); - - if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; - } - - return MSGPACK_UNPACK_SUCCESS; -} -#endif