Merge branch 'master' of git://github.com/msgpack/msgpack

Conflicts:
	perl/t/05_preferred_int.t

perl/t/05_preferred_int.t

@@ -12,7 +12,11 @@ sub packit {
 }
 
 sub pis ($$) {
-    is packit($_[0]), $_[1], 'dump ' . $_[1];
+    if (ref $_[1]) {
+        like packit($_[0]), $_[1], 'dump ' . $_[1];
+    } else {
+        is packit($_[0]), $_[1], 'dump ' . $_[1];
+    }
     # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0]));
 }
 
@@ -30,16 +34,12 @@ my @dat = (
     ''.0xFFFFFF => 'ce 00 ff ff ff',
     ''.0xFFFFFFFF => 'ce ff ff ff ff',
     ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35',
-    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => $is_win ?
+    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$},
-                                            'b5 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 30 33 34'
-                                          : 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34',
     '-'.0x8000000 => 'd2 f8 00 00 00',
     '-'.0x80000000 => 'd2 80 00 00 00',
     '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38',
     '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38',
-    '-'.0x800000000000000000000000000000 => $is_win ?
+    '-'.0x800000000000000000000000000000 => qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)},
-                                            'b6 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 30 33 35'
-                                          : 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35',
     {'0' => '1'}, '81 00 01',
     {'abc' => '1'}, '81 a3 61 62 63 01',
 );

python/.gitignore

@@ -3,3 +3,4 @@ build/*
 dist/*
 *.pyc
 *.pyo
+msgpack/__version__.py

python/Makefile

@@ -1,7 +1,12 @@
+.PHONY: test all python3
+
 all:
 	python setup.py build_ext -i -f
 	python setup.py build sdist
 
-.PHONY: test
+python3:
+	python3 setup.py build_ext -i -f
+	python3 setup.py build sdist
+
 test:
 	nosetests test

python/msgpack/__init__.py

@@ -1,5 +1,6 @@
 # coding: utf-8
-from _msgpack import *
+from msgpack.__version__ import *
+from msgpack._msgpack import *
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack

python/msgpack/_msgpack.pyx

@@ -1,26 +1,24 @@
 # coding: utf-8
 
-import cStringIO
-
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef struct PyObject
 
-    cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
+    cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
     cdef PyObject* Py_True
     cdef PyObject* Py_False
 
-    cdef char* PyString_AsString(object o)
     cdef long long PyLong_AsLongLong(object o)
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
 
-    cdef int PyMapping_Check(object o)
+    cdef bint PyBool_Check(object o)
-    cdef int PySequence_Check(object o)
+    cdef bint PyDict_Check(object o)
-    cdef int PyLong_Check(object o)
+    cdef bint PySequence_Check(object o)
-    cdef int PyInt_Check(object o)
+    cdef bint PyLong_Check(object o)
-    cdef int PyFloat_Check(object o)
+    cdef bint PyInt_Check(object o)
-    cdef int PyString_Check(object o)
+    cdef bint PyFloat_Check(object o)
-    cdef int PyUnicode_Check(object o)
+    cdef bint PyBytes_Check(object o)
+    cdef bint PyUnicode_Check(object o)
 
 cdef extern from "stdlib.h":
     void* malloc(size_t)
@@ -71,20 +69,23 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int __pack(self, object o) except -1:
+    cdef int _pack(self, object o) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
         cdef double fval
         cdef char* rawval 
         cdef int ret
+        cdef dict d
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
-        elif <PyObject*>o == Py_True:
+            #elif PyBool_Check(o):
-            ret = msgpack_pack_true(&self.pk)
+        elif isinstance(o, bool):
-        elif <PyObject*>o == Py_False:
+            if o:
-            ret = msgpack_pack_false(&self.pk)
+                ret = msgpack_pack_true(&self.pk)
+            else:
+                ret = msgpack_pack_false(&self.pk)
         elif PyLong_Check(o):
             if o > 0:
                 ullval = PyLong_AsUnsignedLongLong(o)
@@ -98,7 +99,7 @@ cdef class Packer(object):
         elif PyFloat_Check(o):
             fval = o
             ret = msgpack_pack_double(&self.pk, fval)
-        elif PyString_Check(o):
+        elif PyBytes_Check(o):
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -109,19 +110,20 @@ cdef class Packer(object):
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyMapping_Check(o):
+        elif PyDict_Check(o):
-            ret = msgpack_pack_map(&self.pk, len(o))
+            d = o
+            ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in o.iteritems():
+                for k,v in d.items():
-                    ret = self.__pack(k)
+                    ret = self._pack(k)
                     if ret != 0: break
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         else:
             # TODO: Serialize with defalt() like simplejson.
@@ -130,10 +132,10 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self.__pack(obj)
+        ret = self._pack(obj)
         if ret:
             raise TypeError
-        buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0
         return buf
 
@@ -253,19 +255,18 @@ cdef class Unpacker(object):
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
 
-    def feed(self, next_bytes):
+    def feed(self, bytes next_bytes):
-        if not isinstance(next_bytes, str):
-           raise ValueError, "Argument must be bytes object"
         self.waiting_bytes.append(next_bytes)
 
     cdef append_buffer(self):
         cdef char* buf = self.buf
         cdef Py_ssize_t tail = self.buf_tail
         cdef Py_ssize_t l
+        cdef bytes b
 
         for b in self.waiting_bytes:
             l = len(b)
-            memcpy(buf + tail, PyString_AsString(b), l)
+            memcpy(buf + tail, <char*>(b), l)
             tail += l
         self.buf_tail = tail
         del self.waiting_bytes[:]

python/msgpack/unpack.h

@@ -175,7 +175,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
-    py = PyString_FromStringAndSize(p, l);
+    py = PyBytes_FromStringAndSize(p, l);
     if (!py)
         return -1;
     *o = py;

python/setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+version = (0, 1, 6, 'final')
 
 import os
 from glob import glob
@@ -14,7 +15,13 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     have_cython = False
 
-version = '0.1.4'
+# make msgpack/__verison__.py
+f = open('msgpack/__version__.py', 'w')
+f.write("version = %r\n" % (version,))
+f.close()
+version_str = '.'.join(str(x) for x in version[:3])
+if len(version) > 3 and version[3] != 'final':
+    version_str += version[3]
 
 # take care of extension modules.
 if have_cython:
@@ -53,7 +60,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/)
 setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
-      version=version,
+      version=version_str,
       cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],
@@ -62,6 +69,7 @@ setup(name='msgpack-python',
       url='http://msgpack.sourceforge.net/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
+          'Programming Language :: Python :: 3',
           'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',

python/test3/test_case.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packs, unpacks
+
+
+def check(length, obj):
+    v = packs(obj)
+    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
+    assert_equal(unpacks(v), obj)
+
+def test_1():
+    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
+              -((1<<5)-1), -(1<<5)]:
+        check(1, o)
+
+def test_2():
+    for o in [1 << 7, (1 << 8) - 1,
+              -((1<<5)+1), -(1<<7)
+             ]:
+        check(2, o)
+
+def test_3():
+    for o in [1 << 8, (1 << 16) - 1,
+              -((1<<7)+1), -(1<<15)]:
+        check(3, o)
+
+def test_5():
+    for o in [1 << 16, (1 << 32) - 1,
+              -((1<<15)+1), -(1<<31)]:
+        check(5, o)
+
+def test_9():
+    for o in [1 << 32, (1 << 64) - 1,
+              -((1<<31)+1), -(1<<63),
+              1.0, 0.1, -0.1, -1.0]:
+        check(9, o)
+
+
+def check_raw(overhead, num):
+    check(num + overhead, b" " * num)
+
+def test_fixraw():
+    check_raw(1, 0)
+    check_raw(1, (1<<5) - 1)
+
+def test_raw16():
+    check_raw(3, 1<<5)
+    check_raw(3, (1<<16) - 1)
+
+def test_raw32():
+    check_raw(5, 1<<16)
+
+
+def check_array(overhead, num):
+    check(num + overhead, (None,) * num)
+
+def test_fixarray():
+    check_array(1, 0)
+    check_array(1, (1 << 4) - 1)
+
+def test_array16():
+    check_array(3, 1 << 4)
+    check_array(3, (1<<16)-1)
+
+def test_array32():
+    check_array(5, (1<<16))
+
+
+def match(obj, buf):
+    assert_equal(packs(obj), buf)
+    assert_equal(unpacks(buf), obj)
+
+def test_match():
+    cases = [
+        (None, b'\xc0'),
+        (False, b'\xc2'),
+        (True, b'\xc3'),
+        (0, b'\x00'),
+        (127, b'\x7f'),
+        (128, b'\xcc\x80'),
+        (256, b'\xcd\x01\x00'),
+        (-1, b'\xff'),
+        (-33, b'\xd0\xdf'),
+        (-129, b'\xd1\xff\x7f'),
+        ({1:1}, b'\x81\x01\x01'),
+        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ((), b'\x90'),
+        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
+        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
+        ({}, b'\x80'),
+        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
+        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
+        ]
+
+    for v, p in cases:
+        match(v, p)
+
+if __name__ == '__main__':
+    main()

python/test3/test_except.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose.tools import *
+from msgpack import packs, unpacks
+
+import datetime
+
+def test_raise_on_find_unsupported_value():
+    assert_raises(TypeError, packs, datetime.datetime.now())
+
+if __name__ == '__main__':
+    from nose import main
+    main()

python/test3/test_format.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import unpacks
+
+def check(src, should):
+    assert_equal(unpacks(src), should)
+
+def testSimpleValue():
+    check(b"\x93\xc0\xc2\xc3", 
+            (None, False, True,))
+
+def testFixnum():
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
+          ((0,64,127,), (-32,-16,-1,),)
+          )
+
+def testFixArray():
+    check(b"\x92\x90\x91\x91\xc0",
+          ((),((None,),),),
+          )
+
+def testFixRaw():
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
+          (b"", b"a", b"bc", b"def",),
+          )
+
+def testFixMap():
+    check(
+          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          {False: {None: None}, True:{None:{}}},
+          )
+
+def testUnsignedInt():
+    check(
+          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          b"\xce\xff\xff\xff\xff",
+          (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
+          )
+
+def testSignedInt():
+    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          b"\xd2\xff\xff\xff\xff",
+          (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
+
+def testRaw():
+    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        (b"", b"a", b"ab", b"", b"a", b"ab"))
+
+def testArray():
+    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+        b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+        b"\xc2\xc3",
+        ((), (None,), (False,True), (), (None,), (False,True))
+        )
+
+def testMap():
+    check(
+        b"\x96"
+            b"\xde\x00\x00"
+            b"\xde\x00\x01\xc0\xc2"
+            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+            b"\xdf\x00\x00\x00\x00"
+            b"\xdf\x00\x00\x00\x01\xc0\xc2"
+            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        ({}, {None: False}, {True: False, None: False}, {},
+            {None: False}, {True: False, None: False}))
+
+if __name__ == '__main__':
+    main()

python/test3/test_pack.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def check(data):
+    re = unpacks(packs(data))
+    assert_equal(re, data)
+
+def testPack():
+    test_data = [
+            0, 1, 127, 128, 255, 256, 65535, 65536,
+            -1, -32, -33, -128, -129, -32768, -32769,
+            1.0,
+        b"", b"a", b"a"*31, b"a"*32,
+        None, True, False,
+        (), ((),), ((), None,), 
+        {None: 0}, 
+        (1<<23), 
+        ]
+    for td in test_data:
+        check(td)
+
+if __name__ == '__main__':
+    main()

python/test3/test_sequnpack.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+
+from msgpack import Unpacker
+
+def test_foobar():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
+    try:
+        o = unpacker.unpack()
+        print(("Oops!", o))
+        assert 0
+    except StopIteration:
+        assert 1
+    else:
+        assert 0
+    unpacker.feed(b'foo')
+    unpacker.feed(b'bar')
+
+    k = 0
+    for o, e in zip(unpacker, b'foobarbaz'):
+        assert o == e
+        k += 1
+    assert k == len(b'foobar')
+
+if __name__ == '__main__':
+    test_foobar()
+