From 5393a0df16f3bbdf296222b6337db5d57fe7c3a6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 01:27:09 +0900 Subject: [PATCH 01/24] import MessagePack for Java implementation plan 1 --- java-plan1/MessagePack.java | 31 +++ java-plan1/Packer.java | 266 +++++++++++++++++++++++++ java-plan1/Unpacker.java | 385 ++++++++++++++++++++++++++++++++++++ java-plan1/run.sh | 3 + java-plan1/test.java | 27 +++ 5 files changed, 712 insertions(+) create mode 100644 java-plan1/MessagePack.java create mode 100644 java-plan1/Packer.java create mode 100644 java-plan1/Unpacker.java create mode 100755 java-plan1/run.sh create mode 100644 java-plan1/test.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java new file mode 100644 index 00000000..143f6b51 --- /dev/null +++ b/java-plan1/MessagePack.java @@ -0,0 +1,31 @@ +import java.nio.ByteBuffer; +import java.io.InputStream; +import java.io.IOException; + +public class MessagePack { + + static public Object unpack(InputStream source) + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(byte[] source, int len) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer source) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer[] source) throws IOException + { + // FIXME not implemented yet + return null; + } +} + diff --git a/java-plan1/Packer.java b/java-plan1/Packer.java new file mode 100644 index 00000000..ea6b2d4d --- /dev/null +++ b/java-plan1/Packer.java @@ -0,0 +1,266 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + //public Packer pack(Object o) throws IOException +} + diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java new file mode 100644 index 00000000..be86344e --- /dev/null +++ b/java-plan1/Unpacker.java @@ -0,0 +1,385 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Unpacker { + protected static final int CS_HEADER = 0x00; + protected static final int CS_FLOAT = 0x0a; + protected static final int CS_DOUBLE = 0x0b; + protected static final int CS_UINT_8 = 0x0c; + protected static final int CS_UINT_16 = 0x0d; + protected static final int CS_UINT_32 = 0x0e; + protected static final int CS_UINT_64 = 0x0f; + protected static final int CS_INT_8 = 0x10; + protected static final int CS_INT_16 = 0x11; + protected static final int CS_INT_32 = 0x12; + protected static final int CS_INT_64 = 0x13; + protected static final int CS_RAW_16 = 0x1a; + protected static final int CS_RAW_32 = 0x1b; + protected static final int CS_ARRAY_16 = 0x1c; + protected static final int CS_ARRAY_32 = 0x1d; + protected static final int CS_MAP_16 = 0x1e; + protected static final int CS_MAP_32 = 0x1f; + protected static final int ACS_RAW_VALUE = 0x20; + protected static final int CT_ARRAY_ITEM = 0x00; + protected static final int CT_MAP_KEY = 0x01; + protected static final int CT_MAP_VALUE = 0x02; + + protected static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected Object[] stack_map_key = new Object[MAX_STACK_SIZE]; + //protected Object user; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + + public Object getData() + { + return stack_obj[0]; + } + + public boolean isFinished() + { + return finished; + } + + + @SuppressWarnings("unchecked") + public int execute(byte[] src, int off, int length) throws IOException + { + if(off >= length) { return off; } + + int limit = off + length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = null; + break _push; + case 0xc2: // false + obj = false; + break _push; + case 0xc3: // true + obj = true; + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new IOException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = castBuffer.getFloat(0); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = castBuffer.getDouble(0); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = ((int)src[n]) & 0xff; + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = ((int)castBuffer.getShort(0)) & 0xffff; + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + { + // FIXME always long? + int o = castBuffer.getInt(0); + if(o < 0) { + obj = ((long)o) & 0xffffffffL; + } else { + obj = o; + } + } + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always BigInteger? + long o = castBuffer.getLong(0); + if(o < 0) { + obj = BigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = o; + } + } + throw new IOException("uint 64 is not supported"); + case CS_INT_8: + obj = (int)src[n]; + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = (int)castBuffer.getShort(0); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = (int)castBuffer.getInt(0); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always long? + long o = castBuffer.getLong(0); + if(o <= 0x7fffffffL && o > -0x80000000L) { + obj = (int)o; + } else { + obj = o; + } + } + //System.out.println("long "+obj); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = ByteBuffer.wrap(src, n, trail); // FIXME プール? コピー + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new IOException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + stack_obj[0] = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayList)(stack_obj[top])).add(obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + stack_map_key[top] = obj; + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new IOException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i - off; + } +} + diff --git a/java-plan1/run.sh b/java-plan1/run.sh new file mode 100755 index 00000000..1539a37b --- /dev/null +++ b/java-plan1/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +javac MessagePack.java Packer.java Unpacker.java test.java +java test diff --git a/java-plan1/test.java b/java-plan1/test.java new file mode 100644 index 00000000..938a6871 --- /dev/null +++ b/java-plan1/test.java @@ -0,0 +1,27 @@ +import java.util.*; +import java.io.*; + +class OpenByteArrayOutputStream extends ByteArrayOutputStream { + int getCount() { return count; } + byte[] getBuffer() { return buf; } +} + +public class test { + public static void main(String[] args) throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + Packer pk = new Packer(out); + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.1); + + Unpacker pac = new Unpacker(); + int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { + System.out.println(pac.getData()); + } + } +} + From d39c016e1d84765a50ae3c7cbfc3d6869faa9c07 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 02:41:55 +0900 Subject: [PATCH 02/24] java: fix streaming de/serializer --- java-plan1/Unpacker.java | 22 ++++++++++-- java-plan1/test.java | 75 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java index be86344e..32945de9 100644 --- a/java-plan1/Unpacker.java +++ b/java-plan1/Unpacker.java @@ -50,13 +50,26 @@ public class Unpacker { return finished; } + public void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + stack_map_key[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + } @SuppressWarnings("unchecked") public int execute(byte[] src, int off, int length) throws IOException { if(off >= length) { return off; } - int limit = off + length; + int limit = length; int i = off; int count; @@ -64,7 +77,7 @@ public class Unpacker { _out: do { _header_again: { - //System.out.println("while i:"+i); + //System.out.println("while i:"+i+" limit:"+limit); int b = src[i]; @@ -349,6 +362,7 @@ public class Unpacker { ((ArrayList)(stack_obj[top])).add(obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_obj[top] = null; --top; break _push; } @@ -363,6 +377,8 @@ public class Unpacker { ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_map_key[top] = null; + stack_obj[top] = null; --top; break _push; } @@ -379,7 +395,7 @@ public class Unpacker { ++i; } while(i < limit); // _out - return i - off; + return i; } } diff --git a/java-plan1/test.java b/java-plan1/test.java index 938a6871..5b2349fd 100644 --- a/java-plan1/test.java +++ b/java-plan1/test.java @@ -6,21 +6,88 @@ class OpenByteArrayOutputStream extends ByteArrayOutputStream { byte[] getBuffer() { return buf; } } + public class test { + public static void main(String[] args) throws IOException + { + testSimple(); + testStreaming(); + } + + + public static void testSimple() throws IOException { OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); Packer pk = new Packer(out); pk.packArray(3) - .packInt(0) - .packByte((byte)1) - .packDouble(0.1); + .packInt(1) + .packByte((byte)2) + .packDouble(0.3); Unpacker pac = new Unpacker(); int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { - System.out.println(pac.getData()); + System.out.println("testSimple: "+pac.getData()); + } + } + + + public static void testStreaming() throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + //// + // sender + // + // initialize the streaming serializer + Packer pk = new Packer(out); + + // serialize 2 objects + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.2); + pk.packArray(3) + .packInt(3) + .packByte((byte)4) + .packDouble(0.5); + + // send it through the network + InputStream sock = new ByteArrayInputStream(out.getBuffer(), 0, out.getCount()); + + + //// + // receiver + // + // initialize the streaming deserializer + Unpacker pac = new Unpacker(); + int parsed = 0; + + byte[] buf = new byte[1024]; + int buflen = 0; + + while(true) { + // receive data from the network + int c = sock.read(); + if(c < 0) { return; } + + buf[buflen++] = (byte)c; + + // deserialize + parsed = pac.execute(buf, parsed, buflen); + if(pac.isFinished()) { + // get an object + Object msg = pac.getData(); + System.out.println("testStreaming: "+msg); + + // reset the streaming deserializer + pac.reset(); + buflen = 0; + parsed = 0; + } } } } From 4758d9f04b115bbccb568e262e49d53c5549edb5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:24:47 +0900 Subject: [PATCH 03/24] Fix to use MANIEFST.in. --- python/MANIFEST | 9 --------- python/MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 python/MANIFEST create mode 100644 python/MANIFEST.in diff --git a/python/MANIFEST b/python/MANIFEST deleted file mode 100644 index f6c43b02..00000000 --- a/python/MANIFEST +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -msgpack/pack.h -msgpack/unpack.h -msgpack/_msgpack.pyx -msgpack/__init__.py -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 00000000..6841ffef --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +recursive-include msgpack *.h *.c *.pyx From 93a95725fc45d7d0047578ecdc5b2ae4e900970f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:26:12 +0900 Subject: [PATCH 04/24] Fix Makefile --- python/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/Makefile b/python/Makefile index d90789ab..267415c5 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,6 @@ all: - python setup.py build_ext -i -f - python setup.py build - python setup.py sdist + python setup_dev.py build_ext -i -f + python setup.py build sdist .PHONY: test test: From e39e1d4f602b0202b830f8e672e2116bdb8b9f34 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 12 Nov 2009 01:10:36 +0900 Subject: [PATCH 05/24] import MessagePack for Java implementation plan 2 --- java-plan1/MessagePack.java | 6 + java-plan2/build.xml | 15 + java-plan2/src/org/msgpack/GenericArray.java | 25 ++ .../src/org/msgpack/GenericBoolean.java | 17 + java-plan2/src/org/msgpack/GenericLong.java | 17 + java-plan2/src/org/msgpack/GenericMap.java | 25 ++ java-plan2/src/org/msgpack/GenericObject.java | 52 +++ java-plan2/src/org/msgpack/GenericRaw.java | 59 +++ java-plan2/src/org/msgpack/GenericRawRef.java | 55 +++ java-plan2/src/org/msgpack/GenericShort.java | 29 ++ .../src/org/msgpack/MessageConvertable.java | 7 + .../src/org/msgpack/MessageMergeable.java | 7 + .../src/org/msgpack/MessagePackException.java | 10 + .../src/org/msgpack/MessagePackable.java | 9 + java-plan2/src/org/msgpack/Packer.java | 332 +++++++++++++++ .../src/org/msgpack/UnbufferedUnpacker.java | 74 ++++ .../src/org/msgpack/UnpackException.java | 35 ++ .../src/org/msgpack/UnpackIterator.java | 53 +++ java-plan2/src/org/msgpack/Unpacker.java | 269 ++++++++++++ .../src/org/msgpack/impl/ArrayBuilder.java | 7 + .../msgpack/impl/GenericObjectBuilder.java | 127 ++++++ .../impl/GenericZeroCopyObjectBuilder.java | 12 + .../src/org/msgpack/impl/MapBuilder.java | 8 + .../src/org/msgpack/impl/ObjectBuilder.java | 16 + .../msgpack/impl/SpecificObjectBuilder.java | 203 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 395 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 61 +++ .../org/msgpack/schema/ClassGenerator.java | 230 ++++++++++ .../src/org/msgpack/schema/ClassSchema.java | 39 ++ .../src/org/msgpack/schema/FieldSchema.java | 31 ++ .../msgpack/schema/GenericClassSchema.java | 89 ++++ .../msgpack/schema/GenericFieldSchema.java | 25 ++ .../src/org/msgpack/schema/IntSchema.java | 58 +++ .../src/org/msgpack/schema/LongSchema.java | 58 +++ .../src/org/msgpack/schema/MapSchema.java | 70 ++++ .../src/org/msgpack/schema/ObjectSchema.java | 33 ++ .../org/msgpack/schema/PrimitiveSchema.java | 21 + .../src/org/msgpack/schema/RawSchema.java | 44 ++ .../src/org/msgpack/schema/SSchemaParser.java | 246 +++++++++++ java-plan2/src/org/msgpack/schema/Schema.java | 93 +++++ .../src/org/msgpack/schema/ShortSchema.java | 46 ++ .../msgpack/schema/SpecificClassSchema.java | 156 +++++++ .../msgpack/schema/SpecificFieldSchema.java | 78 ++++ .../src/org/msgpack/schema/StringSchema.java | 48 +++ java-plan2/test/Generate.java | 18 + .../src/serializers/msgpack/MediaContent.java | 277 ++++++++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackSerializer.java | 70 ++++ 48 files changed, 3676 insertions(+) create mode 100644 java-plan2/build.xml create mode 100644 java-plan2/src/org/msgpack/GenericArray.java create mode 100644 java-plan2/src/org/msgpack/GenericBoolean.java create mode 100644 java-plan2/src/org/msgpack/GenericLong.java create mode 100644 java-plan2/src/org/msgpack/GenericMap.java create mode 100644 java-plan2/src/org/msgpack/GenericObject.java create mode 100644 java-plan2/src/org/msgpack/GenericRaw.java create mode 100644 java-plan2/src/org/msgpack/GenericRawRef.java create mode 100644 java-plan2/src/org/msgpack/GenericShort.java create mode 100644 java-plan2/src/org/msgpack/MessageConvertable.java create mode 100644 java-plan2/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan2/src/org/msgpack/MessagePackException.java create mode 100644 java-plan2/src/org/msgpack/MessagePackable.java create mode 100644 java-plan2/src/org/msgpack/Packer.java create mode 100644 java-plan2/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan2/src/org/msgpack/UnpackException.java create mode 100644 java-plan2/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan2/src/org/msgpack/Unpacker.java create mode 100644 java-plan2/src/org/msgpack/impl/ArrayBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/MapBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/ObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan2/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ObjectSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/PrimitiveSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan2/src/org/msgpack/schema/Schema.java create mode 100644 java-plan2/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan2/test/Generate.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java index 143f6b51..0a4f442c 100644 --- a/java-plan1/MessagePack.java +++ b/java-plan1/MessagePack.java @@ -29,3 +29,9 @@ public class MessagePack { } } +/* +public interface MessagePackable { + public void toMessagePack(Packer pk); +} +*/ + diff --git a/java-plan2/build.xml b/java-plan2/build.xml new file mode 100644 index 00000000..d382ce04 --- /dev/null +++ b/java-plan2/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan2/src/org/msgpack/GenericArray.java b/java-plan2/src/org/msgpack/GenericArray.java new file mode 100644 index 00000000..53799ca8 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericArray.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.List; +import java.util.ArrayList; + +public class GenericArray extends GenericObject { + private ArrayList array; + + public GenericArray(int length) + { + this.array = new ArrayList(length); + } + + public void add(GenericObject element) + { + array.add(element); + } + + @Override + public List asArray() + { + return array; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericBoolean.java b/java-plan2/src/org/msgpack/GenericBoolean.java new file mode 100644 index 00000000..908128f1 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericBoolean.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericBoolean extends GenericObject { + boolean value; + + public GenericBoolean(boolean value) + { + this.value = value; + } + + @Override + public boolean asBoolean() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericLong.java b/java-plan2/src/org/msgpack/GenericLong.java new file mode 100644 index 00000000..7cc9110d --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericLong.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericLong extends GenericObject { + long value; + + public GenericLong(long value) + { + this.value = value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericMap.java b/java-plan2/src/org/msgpack/GenericMap.java new file mode 100644 index 00000000..0a5512a6 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericMap.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.Map; +import java.util.HashMap; + +public class GenericMap extends GenericObject { + private HashMap map; + + public GenericMap(int length) + { + this.map = new HashMap(length); + } + + public void put(GenericObject key, GenericObject value) + { + map.put(key, value); + } + + @Override + public Map asMap() + { + return map; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericObject.java b/java-plan2/src/org/msgpack/GenericObject.java new file mode 100644 index 00000000..32eacd36 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericObject.java @@ -0,0 +1,52 @@ +package org.msgpack; + +import java.util.List; +import java.util.Map; + +public abstract class GenericObject { + public boolean asBoolean() + { + throw new RuntimeException("type error"); + } + + public byte asByte() + { + throw new RuntimeException("type error"); + } + + public short asShort() + { + throw new RuntimeException("type error"); + } + + public int asInt() + { + throw new RuntimeException("type error"); + } + + public long asLong() + { + throw new RuntimeException("type error"); + } + + public String asString() + { + throw new RuntimeException("type error"); + } + + public byte[] asBytes() + { + throw new RuntimeException("type error"); + } + + public List asArray() + { + throw new RuntimeException("type error"); + } + + public Map asMap() + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRaw.java b/java-plan2/src/org/msgpack/GenericRaw.java new file mode 100644 index 00000000..6de03b28 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRaw.java @@ -0,0 +1,59 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRaw extends GenericObject { + byte[] bytes; + String string; + + public GenericRaw() + { + this.bytes = new byte[0]; + this.string = null; + } + + public GenericRaw(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + public GenericRaw(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRawRef.java b/java-plan2/src/org/msgpack/GenericRawRef.java new file mode 100644 index 00000000..70078100 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRawRef.java @@ -0,0 +1,55 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRawRef extends GenericRaw { + int offset; + int length; + + public GenericRawRef(byte[] bytes, int offset, int length) + { + this.bytes = bytes; + this.offset = offset; + this.length = length; + this.string = null; + } + + public GenericRawRef(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericShort.java b/java-plan2/src/org/msgpack/GenericShort.java new file mode 100644 index 00000000..eb2463e3 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericShort.java @@ -0,0 +1,29 @@ +package org.msgpack; + +public class GenericShort extends GenericObject { + short value; + + public GenericShort(short value) + { + this.value = value; + } + + @Override + public short asShort() + { + return value; + } + + @Override + public int asInt() + { + return value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/MessageConvertable.java b/java-plan2/src/org/msgpack/MessageConvertable.java new file mode 100644 index 00000000..68c123ec --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageConvertable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageConvertable +{ + public void messageConvert(GenericObject obj); +} + diff --git a/java-plan2/src/org/msgpack/MessageMergeable.java b/java-plan2/src/org/msgpack/MessageMergeable.java new file mode 100644 index 00000000..dc507490 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageMergeable { + public void setField(int index, Object value); + public Object getField(int index); +} + diff --git a/java-plan2/src/org/msgpack/MessagePackException.java b/java-plan2/src/org/msgpack/MessagePackException.java new file mode 100644 index 00000000..dc379893 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackException.java @@ -0,0 +1,10 @@ +package org.msgpack; + +public class MessagePackException extends RuntimeException +{ + public MessagePackException(String message) + { + super(message); + } +} + diff --git a/java-plan2/src/org/msgpack/MessagePackable.java b/java-plan2/src/org/msgpack/MessagePackable.java new file mode 100644 index 00000000..1efff3f0 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackable.java @@ -0,0 +1,9 @@ +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable +{ + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan2/src/org/msgpack/Packer.java b/java-plan2/src/org/msgpack/Packer.java new file mode 100644 index 00000000..4545849e --- /dev/null +++ b/java-plan2/src/org/msgpack/Packer.java @@ -0,0 +1,332 @@ +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + public Packer packString(String s) throws IOException + { + byte[] b = ((String)s).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + return this; + } + + public Packer pack(MessagePackable o) throws IOException + { + o.messagePack(this); + return this; + } + + public Packer pack(Object o) throws IOException + { + if(o == null) { + packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + } else if(o instanceof Boolean) { + if((Boolean)o) { + packTrue(); + } else { + packFalse(); + } + } else if(o instanceof Integer) { + packInt((Integer)o); + } else if(o instanceof Long) { + packLong((Long)o); + } else if(o instanceof Short) { + packShort((Short)o); + } else if(o instanceof Byte) { + packByte((Byte)o); + } else if(o instanceof Float) { + packFloat((Float)o); + } else if(o instanceof Double) { + packDouble((Double)o); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + return this; + } +} + diff --git a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 00000000..5b5eb97f --- /dev/null +++ b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,74 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() + { + super(new GenericObjectBuilder()); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public Object getData() + { + return data; + } + + public boolean isFinished() + { + return finished; + } + + public void reset() + { + super.reset(); + this.offset = 0; + } + + int getOffset() + { + return offset; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException + { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackException.java b/java-plan2/src/org/msgpack/UnpackException.java new file mode 100644 index 00000000..2081fafe --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackException.java @@ -0,0 +1,35 @@ +package org.msgpack; + +public class UnpackException extends MessagePackException +{ + public static final int EXTRA_BYTES = 1; + public static final int INSUFFICIENT_BYTES = 0; + public static final int PARSE_ERROR = -1; + + private int errorCode; + private Object data; + + public UnpackException(String message, int errorCode) + { + super(message); + this.errorCode = errorCode; + } + + public UnpackException(String message, int errorCode, Object data) + { + super(message); + this.errorCode = errorCode; + this.data = data; + } + + public int getErrorCode() + { + return errorCode; + } + + public Object getData() + { + return data; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackIterator.java b/java-plan2/src/org/msgpack/UnpackIterator.java new file mode 100644 index 00000000..90101018 --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,53 @@ +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) + { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() + { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() + { + if(!have) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan2/src/org/msgpack/Unpacker.java b/java-plan2/src/org/msgpack/Unpacker.java new file mode 100644 index 00000000..5a58ce88 --- /dev/null +++ b/java-plan2/src/org/msgpack/Unpacker.java @@ -0,0 +1,269 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = null; + } + + public Unpacker(int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = null; + } + + public Unpacker(InputStream stream) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = stream; + } + + public Unpacker(InputStream stream, int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public void reserveBuffer(int size) + { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() + { + return buffer; + } + + public int getBufferOffset() + { + return used; + } + + public int getBufferCapacity() + { + return buffer.length - used; + } + + public void bufferConsumed(int size) + { + used += size; + } + + public void feed(byte[] buffer) + { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) + { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException + { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() + { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException + { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() + { + return super.getData(); + } + + public void reset() + { + super.reset(); + parsed = 0; + } + + public int getMessageSize() + { + return parsed - offset + used; + } + + public int getParsedSize() + { + return parsed; + } + + public int getNonParsedSize() + { + return used - offset; + } + + public void skipNonparsedBuffer(int size) + { + offset += size; + } + + public void removeNonparsedBuffer() + { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java new file mode 100644 index 00000000..9bb099b2 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java @@ -0,0 +1,7 @@ +package org.msgpack.impl; + +public interface ArrayBuilder { + public void add(Object element); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java new file mode 100644 index 00000000..814e3029 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java @@ -0,0 +1,127 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericObjectBuilder implements ObjectBuilder { + public Object createNil() + { + return null; + } + + @Override + public Object createBoolean(boolean v) + { + return new GenericBoolean(v); + } + + @Override + public Object createByte(byte v) + { + //return new GenericByte(v); + return null; // FIXME + } + + @Override + public Object createShort(short v) + { + return new GenericShort(v); + } + + @Override + public Object createInt(int v) + { + //return new GenericInt(v); + return null; // FIXME + } + + @Override + public Object createLong(long v) + { + return new GenericLong(v); + } + + @Override + public Object createFloat(float v) + { + //return new GenericFloat(v); + return null; // FIXME + } + + @Override + public Object createDouble(double v) + { + //return new GenericDouble(v); + return null; // FIXME + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] copy = new byte[length]; + System.arraycopy(b, offset, copy, 0, length); + return new GenericRaw(copy); + } + + @Override + public ArrayBuilder createArray(int length) + { + return new GenericArrayBuilder(length); + } + + @Override + public MapBuilder createMap(int length) + { + return new GenericMapBuilder(length); + } +} + +final class GenericArrayBuilder implements ArrayBuilder { + private GenericArray a; + + GenericArrayBuilder(int length) + { + this.a = new GenericArray(length); + } + + @Override + public void add(Object element) + { + a.add((GenericObject)element); + } + + @Override + public Object finish() + { + return a; + } +} + +final class GenericMapBuilder implements MapBuilder { + private GenericMap m; + private GenericObject key; + + GenericMapBuilder(int length) + { + this.m = new GenericMap(length); + } + + @Override + public void putKey(Object key) + { + this.key = (GenericObject)key; + } + + @Override + public void putValue(Object value) + { + m.put(this.key, (GenericObject)value); + this.key = null; + } + + @Override + public Object finish() + { + return m; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java new file mode 100644 index 00000000..5569fbf5 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java @@ -0,0 +1,12 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericZeroCopyObjectBuilder extends GenericObjectBuilder { + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return new GenericRawRef(b, offset, length); + } +} + diff --git a/java-plan2/src/org/msgpack/impl/MapBuilder.java b/java-plan2/src/org/msgpack/impl/MapBuilder.java new file mode 100644 index 00000000..57859a68 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/MapBuilder.java @@ -0,0 +1,8 @@ +package org.msgpack.impl; + +public interface MapBuilder { + public void putKey(Object key); + public void putValue(Object value); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java new file mode 100644 index 00000000..32689034 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java @@ -0,0 +1,16 @@ +package org.msgpack.impl; + +public interface ObjectBuilder { + public Object createNil(); + public Object createBoolean(boolean v); + public Object createByte(byte v); + public Object createShort(short v); + public Object createInt(int v); + public Object createLong(long v); + public Object createFloat(float v); + public Object createDouble(double v); + public Object createRaw(byte[] b, int offset, int length); + public ArrayBuilder createArray(int length); + public MapBuilder createMap(int length); +} + diff --git a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java new file mode 100644 index 00000000..7748844e --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java @@ -0,0 +1,203 @@ +package org.msgpack.impl; + +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import org.msgpack.schema.*; + +public final class SpecificObjectBuilder implements ObjectBuilder { + private int top; + private Schema schema; + + public SpecificObjectBuilder(Schema schema) + { + this.top = 0; + this.schema = schema; + } + + void setSchema(Schema s) + { + schema = s; + } + + Schema swapSchema(Schema s) + { + Schema old = schema; + schema = s; + return old; + } + + @Override + public Object createNil() + { + return schema.createNil(); + } + + @Override + public Object createBoolean(boolean v) + { + return schema.createBoolean(v); + } + + @Override + public Object createByte(byte v) + { + return schema.createByte(v); + } + + @Override + public Object createShort(short v) + { + return schema.createShort(v); + } + + @Override + public Object createInt(int v) + { + return schema.createInt(v); + } + + @Override + public Object createLong(long v) + { + return schema.createLong(v); + } + + @Override + public Object createFloat(float v) + { + return schema.createFloat(v); + } + + @Override + public Object createDouble(double v) + { + return schema.createDouble(v); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return schema.createRaw(b, offset, length); + } + + @Override + public ArrayBuilder createArray(int length) + { + if(schema instanceof ClassSchema) { + return new ClassBuilder(length, (ClassSchema)schema, this); + } + ArraySchema as = (ArraySchema)schema; + return new SpecificArrayBuilder(length, as.getElementType(), this); + } + + @Override + public MapBuilder createMap(int length) + { + MapSchema ms = (MapSchema)schema; + return new SpecificMapBuilder(length, ms.getKeyType(), ms.getValueType(), this); + } +} + +final class SpecificArrayBuilder implements ArrayBuilder { + private ArrayList a; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificArrayBuilder(int length, Schema elementSchema, SpecificObjectBuilder builder) + { + this.a = new ArrayList(length); + this.builder = builder; + this.parentSchema = builder.swapSchema(elementSchema); + } + + public void add(Object element) + { + a.add(element); + } + + public Object finish() + { + builder.swapSchema(parentSchema); + return a; + } +} + +final class SpecificMapBuilder implements MapBuilder { + private HashMap m; + private Object key; + private Schema keySchema; + private Schema valueSchema; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificMapBuilder(int length, Schema keySchema, Schema valueSchema, SpecificObjectBuilder builder) + { + this.m = new HashMap(length); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + this.builder = builder; + this.parentSchema = builder.swapSchema(keySchema); + } + + @Override + public void putKey(Object key) + { + this.key = key; + this.builder.setSchema(valueSchema); + } + + @Override + public void putValue(Object value) + { + m.put(this.key, value); + this.key = null; + this.builder.setSchema(keySchema); + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return m; + } +} + +final class ClassBuilder implements ArrayBuilder { + private Object object; + private int index; + private List fields; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + ClassBuilder(int length, ClassSchema schema, SpecificObjectBuilder builder) + { + this.object = schema.newInstance(); + this.index = 0; + this.fields = schema.getFields(); + this.builder = builder; + this.parentSchema = builder.swapSchema(fields.get(0).getType()); + // FIXME check length + } + + @Override + public void add(Object element) + { + FieldSchema f = fields.get(index++); // FIXME check fields.size + f.setFieldValue(object, element); // XXX FIXME debug + if(fields.size() > index) { + builder.setSchema( fields.get(index).getType() ); + } else { + builder.setSchema( null ); + // FIXME: builder.setSchema(new InvalidFieldSchema); + } + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return object; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 00000000..9f88072a --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,395 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.UnpackException; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected Object data = null; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + protected ObjectBuilder builder; + + protected UnpackerImpl(ObjectBuilder builder) + { + this.builder = builder; + } + + protected void setBuilder(ObjectBuilder builder) + { + this.builder = builder; + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + data = null; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = builder.createNil(); + break _push; + case 0xc2: // false + obj = builder.createBoolean(false); + break _push; + case 0xc3: // true + obj = builder.createBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = builder.createShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = builder.createLong( o ); + } + } + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported", UnpackException.PARSE_ERROR); + case CS_INT_8: + obj = builder.createByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = builder.createRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayBuilder)stack_obj[top]).add(obj); + if(--stack_count[top] == 0) { + obj = ((ArrayBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + MapBuilder mb = (MapBuilder)stack_obj[top]; + mb.putKey(obj); + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((MapBuilder)stack_obj[top]).putValue(obj); + if(--stack_count[top] == 0) { + obj = ((MapBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ArraySchema.java b/java-plan2/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 00000000..4b051904 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,61 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema { + private Schema elementType; + + public ArraySchema(Schema elementType) + { + super("array"); + this.elementType = elementType; + } + + public Schema getElementType() + { + return elementType; + } + + @Override + public String getFullName() + { + return "ArrayList<"+elementType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + List d = (List)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementType.pack(pk, e); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + List d = obj.asArray(); + List g = new ArrayList(); + for(GenericObject o : d) { + g.add( elementType.convert(o) ); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassGenerator.java b/java-plan2/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 00000000..25a96206 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,230 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.Writer; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) + { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException + { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException + { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getType()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) + { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getType()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementType()); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeyType()); + findSubclassSchema(dst, as.getValueType()); + } + } + + private void writeHeader() throws IOException + { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.*;"); + } + + private void writeFooter() throws IOException + { + } + + private void writeClass() throws IOException + { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException + { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException + { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException + { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getType().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException + { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeConvertFunction(); + } + + private void writeConstructors() throws IOException + { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException + { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException + { + line(); + line("@Override"); + line("public void messagePack(Packer pk) throws IOException"); + line("{"); + pushIndent(); + line("List _f = _SCHEMA.getFields();"); + line("pk.packArray("+schema.getFields().size()+");"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_f.get("+i+").getType().pack(pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeConvertFunction() throws IOException + { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageConvert(GenericObject obj)"); + line("{"); + pushIndent(); + line("List _l = obj.asArray();"); + line("List _f = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_l.size() <= "+i+") { return; } "+f.getName()+" = ("+f.getType().getFullName()+")_f.get("+i+").getType().convert(_l.get("+i+"));"); + ++i; + } + popIndent(); + line("}"); + line(); + line("public static "+schema.getName()+" convert(GenericObject obj)"); + line("{"); + pushIndent(); + line("return ("+schema.getName()+")_SCHEMA.convert(obj);"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException + { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException + { + writer.write("\n"); + } + + private void pushIndent() + { + indent += 1; + } + + private void popIndent() + { + indent -= 1; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassSchema.java b/java-plan2/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 00000000..3343fca8 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,39 @@ +package org.msgpack.schema; + +import java.util.List; + +public abstract class ClassSchema extends Schema { + protected String namespace; + protected List imports; + + public ClassSchema(String name, String namespace, List imports) + { + super(name); + this.namespace = namespace; + this.imports = imports; + } + + public String getNamespace() + { + return namespace; + } + + public List getImports() + { + return imports; + } + + void setNamespace(String namespace) + { + this.namespace = namespace; + } + + void setImports(List imports) + { + this.imports = imports; + } + + public abstract List getFields(); + public abstract Object newInstance(); +} + diff --git a/java-plan2/src/org/msgpack/schema/FieldSchema.java b/java-plan2/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 00000000..31a132cb --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,31 @@ +package org.msgpack.schema; + +public abstract class FieldSchema { + private String name; + private Schema type; + + public FieldSchema(String name, Schema type) + { + this.name = name; + this.type = type; + } + + public String getName() + { + return this.name; + } + + public Schema getType() + { + return type; + } + + public String getExpression() + { + return "(field "+name+" "+type.getExpression()+")"; + } + + public abstract Object getFieldValue(Object obj); + public abstract void setFieldValue(Object obj, Object value); +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 00000000..f1e2b446 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,89 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + + +public class GenericClassSchema extends ClassSchema { + private List fields; + + private String fqdn; + private Constructor constructorCache; + + public GenericClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + public String getNamespace() + { + return namespace; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(GenericFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + // FIXME + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + // FIXME + return obj; + } + + @Override + public Object newInstance() + { + return new HashMap(fields.size()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java new file mode 100644 index 00000000..507ee18e --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java @@ -0,0 +1,25 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.lang.reflect.Field; + +public final class GenericFieldSchema extends FieldSchema { + public GenericFieldSchema(String name, Schema type) + { + super(name, type); + } + + @Override + public Object getFieldValue(Object obj) + { + return ((Map)obj).get(getName()); + } + + @Override + @SuppressWarnings("unchecked") + public void setFieldValue(Object obj, Object value) + { + ((Map)obj).put(getName(), value); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/IntSchema.java b/java-plan2/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 00000000..69771c33 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() + { + super("Integer"); + } + + @Override + public String getExpression() + { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packInt((Integer)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asInt(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } + + @Override + public Object createInt(int v) + { + return (int)v; + } + + @Override + public Object createLong(long v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/LongSchema.java b/java-plan2/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 00000000..0ba30573 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() + { + super("Long"); + } + + @Override + public String getExpression() + { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packLong((Long)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asLong(); + } + + @Override + public Object createByte(byte v) + { + return (long)v; + } + + @Override + public Object createShort(short v) + { + return (long)v; + } + + @Override + public Object createInt(int v) + { + return (long)v; + } + + @Override + public Object createLong(long v) + { + return (long)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/MapSchema.java b/java-plan2/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 00000000..e72cf635 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,70 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema { + private Schema keyType; + private Schema valueType; + + public MapSchema(Schema keyType, Schema valueType) + { + super("map"); + this.keyType = keyType; + this.valueType = valueType; + } + + public Schema getKeyType() + { + return keyType; + } + + public Schema getValueType() + { + return valueType; + } + + @Override + public String getFullName() + { + return "HashList<"+keyType.getFullName()+", "+valueType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(map "+keyType.getExpression()+" "+valueType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keyType.pack(pk, e.getKey()); + valueType.pack(pk, e.getValue()); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + Map d = obj.asMap(); + Map g = new HashMap(); + for(Map.Entry e : d.entrySet()) { + g.put(keyType.convert(e.getKey()), valueType.convert(e.getValue())); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ObjectSchema.java b/java-plan2/src/org/msgpack/schema/ObjectSchema.java new file mode 100644 index 00000000..a9f30f5b --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ObjectSchema.java @@ -0,0 +1,33 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ObjectSchema extends Schema { + public ObjectSchema() + { + super("object"); + } + + public String getFullName() + { + return "GenericObject"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.pack(obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java new file mode 100644 index 00000000..023d81ba --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java @@ -0,0 +1,21 @@ +package org.msgpack.schema; + +public abstract class PrimitiveSchema extends Schema { + public static enum PrimitiveType { + BYTE, + SHORT, + INT, + LONG, + FLOAT, + DOUBLE, + } + + public final PrimitiveType type; + + protected PrimitiveSchema(String name, PrimitiveType type) + { + super(name); + this.type = type; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/RawSchema.java b/java-plan2/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 00000000..847ad294 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,44 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() + { + super("raw"); + } + + public String getFullName() + { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asBytes(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SSchemaParser.java b/java-plan2/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 00000000..bfe912f5 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,246 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +// FIXME exception class + +class SSchemaParser { + public static Schema parse(String source) + { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) + { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) + { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) + { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) + { + if(exp.isAtom()) { + String type = exp.getAtom(); + // FIXME + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("object")) { + return new ObjectSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) + { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, fields, namespace, imports); + } else { + return new GenericClassSchema(name, fields, namespace, imports); + } + } + + private ArraySchema parseArray(SExp exp) + { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + if(specificClass) { + return new SpecificFieldSchema(name, type); + } else { + return new GenericFieldSchema(name, type); + } + } +} + diff --git a/java-plan2/src/org/msgpack/schema/Schema.java b/java-plan2/src/org/msgpack/schema/Schema.java new file mode 100644 index 00000000..15b7e729 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/Schema.java @@ -0,0 +1,93 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.impl.*; +import org.msgpack.Packer; +import org.msgpack.GenericObject; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) + { + this.expression = expression; + this.name = name; + } + + public String getName() + { + return name; + } + + public String getFullName() + { + return name; + } + + public String getExpression() + { + return name; + } + + public static Schema parse(String source) + { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) + { + return SSchemaParser.load(source); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + public abstract Object convert(GenericObject obj); + //public abstract Object convertGeneric(GenericObject obj); + + + public Object createNil() + { + return null; + } + + public Object createBoolean(boolean v) + { + throw new RuntimeException("type error"); + } + + public Object createByte(byte v) + { + throw new RuntimeException("type error"); + } + + public Object createShort(short v) + { + throw new RuntimeException("type error"); + } + + public Object createInt(int v) + { + throw new RuntimeException("type error"); + } + + public Object createLong(long v) + { + throw new RuntimeException("type error"); + } + + public Object createFloat(float v) + { + throw new RuntimeException("type error"); + } + + public Object createDouble(double v) + { + throw new RuntimeException("type error"); + } + + public Object createRaw(byte[] b, int offset, int length) + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ShortSchema.java b/java-plan2/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 00000000..aa95f510 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,46 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() + { + super("Short"); + } + + @Override + public String getExpression() + { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packShort((Short)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asShort(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 00000000..75c474a7 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,156 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.io.IOException; +import java.util.Iterator; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private List fields; + private String namespace; + private List imports; + + private String fqdn; + private Constructor constructorCache; + + public SpecificClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(SpecificFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + + if(constructorCache == null) { + cacheConstructor(); + } + + pk.packArray(fields.size()); + for(SpecificFieldSchema f : fields) { + f.getType().pack(pk, f.getFieldValue(obj)); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + if(constructorCache == null) { + cacheConstructor(); + } + + List d = obj.asArray(); + + try { + Object g = constructorCache.newInstance((Object[])null); + + Iterator vi = d.iterator(); + Iterator fi = fields.iterator(); + while(fi.hasNext() && vi.hasNext()) { + SpecificFieldSchema f = fi.next(); + GenericObject v = vi.next(); + f.setFieldValue(g, f.getType().convert(v)); + } + // leave it as uninitialized + //while(fi.hasNext()) { + // SpecificFieldSchema f = fi.next(); + // g.put(f.getName(), null); + //} + + return g; + + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @Override + public Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + public boolean equals(SpecificClassSchema o) + { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java new file mode 100644 index 00000000..297df27e --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java @@ -0,0 +1,78 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.util.Arrays; +import java.lang.reflect.Field; +import org.msgpack.*; + +public class SpecificFieldSchema extends FieldSchema { + public Field fieldCache; + private int index; + + public SpecificFieldSchema(String name, Schema type) + { + super(name, type); + this.index = -1; + } + + @Override + public Object getFieldValue(Object obj) + { + if(index >= 0) { + return ((MessageMergeable)obj).getField(index); + } + + try { + return fieldCache.get(obj); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + @Override + public void setFieldValue(Object obj, Object value) + { + if(index >= 0) { + ((MessageMergeable)obj).setField(index, value); + return; + } + + try { + fieldCache.set(obj, value); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + void cacheField(Class c, int index) + { + for(Class i : c.getInterfaces()) { + if(i.equals(MessageMergeable.class)) { + this.index = index; + return; + } + } + + try { + fieldCache = c.getDeclaredField(getName()); + if(!fieldCache.isAccessible()) { + fieldCache.setAccessible(true); + } + } catch(NoSuchFieldException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } catch(SecurityException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } + } + + //public void setFieldInt(Object obj, int value) + //{ + // if(type instanceof PrimitiveSchema) { + // } + //} +} + diff --git a/java-plan2/src/org/msgpack/schema/StringSchema.java b/java-plan2/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 00000000..fc6855bf --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,48 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() + { + super("string"); + } + + public String getFullName() + { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + String s = (String)obj; + byte[] d = s.getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asString(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + try { + return new String(b, offset, length, "UTF-8"); // XXX FIXME debug + } catch (Exception e) { + // FIXME + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java new file mode 100644 index 00000000..2ac5878d --- /dev/null +++ b/java-plan2/test/Generate.java @@ -0,0 +1,18 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + Writer output = new OutputStreamWriter(System.out); + + Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); + ClassGenerator.write(s1, output); + + Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + ClassGenerator.write(s2, output); + } +} + diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 00000000..ecb64fd2 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,277 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public final class MediaContent implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public ArrayList image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(2); + _f.get(0).getType().pack(pk, image); + _f.get(1).getType().pack(pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } image = (ArrayList)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } media = (Media)_f.get(1).getType().convert(_l.get(1)); + } + + public static MediaContent convert(GenericObject obj) + { + return (MediaContent)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + image = (ArrayList)value; + break; + case 1: + media = (Media)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return image; + case 1: + return media; + } + return null; + } +} + +final class Image implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(5); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } size = (Integer)_f.get(4).getType().convert(_l.get(4)); + } + + public static Image convert(GenericObject obj) + { + return (Image)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + size = (Integer)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return size; + } + return null; + } +} + +final class Media implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public ArrayList person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(11); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, format); + _f.get(5).getType().pack(pk, duration); + _f.get(6).getType().pack(pk, size); + _f.get(7).getType().pack(pk, bitrate); + _f.get(8).getType().pack(pk, person); + _f.get(9).getType().pack(pk, player); + _f.get(10).getType().pack(pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } format = (String)_f.get(4).getType().convert(_l.get(4)); + if(_l.size() <= 5) { return; } duration = (Long)_f.get(5).getType().convert(_l.get(5)); + if(_l.size() <= 6) { return; } size = (Long)_f.get(6).getType().convert(_l.get(6)); + if(_l.size() <= 7) { return; } bitrate = (Integer)_f.get(7).getType().convert(_l.get(7)); + if(_l.size() <= 8) { return; } person = (ArrayList)_f.get(8).getType().convert(_l.get(8)); + if(_l.size() <= 9) { return; } player = (Integer)_f.get(9).getType().convert(_l.get(9)); + if(_l.size() <= 10) { return; } copyright = (String)_f.get(10).getType().convert(_l.get(10)); + } + + public static Media convert(GenericObject obj) + { + return (Media)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + format = (String)value; + break; + case 5: + duration = (Long)value; + break; + case 6: + size = (Long)value; + break; + case 7: + bitrate = (Integer)value; + break; + case 8: + person = (ArrayList)value; + break; + case 9: + player = (Integer)value; + break; + case 10: + copyright = (String)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return format; + case 5: + return duration; + case 6: + return size; + case 7: + return bitrate; + case 8: + return person; + case 9: + return player; + case 10: + return copyright; + } + return null; + } + +} diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 00000000..547ba487 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java new file mode 100644 index 00000000..acb5580b --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import java.nio.charset.Charset; + +import org.msgpack.*; +import org.msgpack.schema.*; +import serializers.msgpack.*; + +import serializers.ObjectSerializer; + +public class MessagePackSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + return content; + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } +} + From eb9e89249137cbed0ace62de9902e69bd082b2a3 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Nov 2009 11:22:08 +0900 Subject: [PATCH 06/24] import MessagePack for Java implementation plan 3 --- java-plan2/test/Generate.java | 2 +- java-plan3/build.xml | 15 + .../src/org/msgpack/MessageMergeable.java | 23 + .../src/org/msgpack/MessagePackable.java | 25 + .../src/org/msgpack/MessageTypeException.java | 39 ++ java-plan3/src/org/msgpack/Packer.java | 409 +++++++++++++++ java-plan3/src/org/msgpack/Schema.java | 133 +++++ .../src/org/msgpack/UnbufferedUnpacker.java | 82 +++ .../src/org/msgpack/UnpackException.java | 29 ++ .../src/org/msgpack/UnpackIterator.java | 66 +++ java-plan3/src/org/msgpack/Unpacker.java | 245 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 483 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 125 +++++ .../src/org/msgpack/schema/ByteSchema.java | 89 ++++ .../org/msgpack/schema/ClassGenerator.java | 241 +++++++++ .../src/org/msgpack/schema/ClassSchema.java | 95 ++++ .../src/org/msgpack/schema/DoubleSchema.java | 84 +++ .../src/org/msgpack/schema/FieldSchema.java | 43 ++ .../src/org/msgpack/schema/FloatSchema.java | 84 +++ .../msgpack/schema/GenericClassSchema.java | 91 ++++ .../src/org/msgpack/schema/GenericSchema.java | 192 +++++++ .../src/org/msgpack/schema/IArraySchema.java | 26 + .../src/org/msgpack/schema/IMapSchema.java | 27 + .../src/org/msgpack/schema/IntSchema.java | 89 ++++ .../src/org/msgpack/schema/LongSchema.java | 89 ++++ .../src/org/msgpack/schema/MapSchema.java | 102 ++++ .../src/org/msgpack/schema/RawSchema.java | 105 ++++ .../msgpack/schema/ReflectionClassSchema.java | 64 +++ .../src/org/msgpack/schema/SSchemaParser.java | 254 +++++++++ .../src/org/msgpack/schema/ShortSchema.java | 89 ++++ .../msgpack/schema/SpecificClassSchema.java | 122 +++++ .../src/org/msgpack/schema/StringSchema.java | 111 ++++ java-plan3/test/Generate.java | 38 ++ .../src/serializers/msgpack/MediaContent.java | 173 +++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackDynamicSerializer.java | 68 +++ .../msgpack/MessagePackGenericSerializer.java | 70 +++ .../MessagePackIndirectSerializer.java | 67 +++ .../MessagePackSpecificSerializer.java | 66 +++ 39 files changed, 4175 insertions(+), 1 deletion(-) create mode 100644 java-plan3/build.xml create mode 100644 java-plan3/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan3/src/org/msgpack/MessagePackable.java create mode 100644 java-plan3/src/org/msgpack/MessageTypeException.java create mode 100644 java-plan3/src/org/msgpack/Packer.java create mode 100644 java-plan3/src/org/msgpack/Schema.java create mode 100644 java-plan3/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan3/src/org/msgpack/UnpackException.java create mode 100644 java-plan3/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan3/src/org/msgpack/Unpacker.java create mode 100644 java-plan3/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan3/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ByteSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/DoubleSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FloatSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IMapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan3/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan3/test/Generate.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java index 2ac5878d..6b7800bb 100644 --- a/java-plan2/test/Generate.java +++ b/java-plan2/test/Generate.java @@ -11,7 +11,7 @@ public class Generate { Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); ClassGenerator.write(s1, output); - Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + Schema s2 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); ClassGenerator.write(s2, output); } } diff --git a/java-plan3/build.xml b/java-plan3/build.xml new file mode 100644 index 00000000..598a853d --- /dev/null +++ b/java-plan3/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan3/src/org/msgpack/MessageMergeable.java b/java-plan3/src/org/msgpack/MessageMergeable.java new file mode 100644 index 00000000..e11119cb --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,23 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +public interface MessageMergeable { + public void messageMerge(Object obj) throws MessageTypeException; +} + diff --git a/java-plan3/src/org/msgpack/MessagePackable.java b/java-plan3/src/org/msgpack/MessagePackable.java new file mode 100644 index 00000000..d8a7db9c --- /dev/null +++ b/java-plan3/src/org/msgpack/MessagePackable.java @@ -0,0 +1,25 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable { + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan3/src/org/msgpack/MessageTypeException.java b/java-plan3/src/org/msgpack/MessageTypeException.java new file mode 100644 index 00000000..09031b21 --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageTypeException.java @@ -0,0 +1,39 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class MessageTypeException extends IOException { + public MessageTypeException() { } + + public MessageTypeException(String s) { + super(s); + } + + public static MessageTypeException invalidConvert(Object from, Schema to) { + return new MessageTypeException(from.getClass().getName()+" cannot be convert to "+to.getExpression()); + } + + /* FIXME + public static MessageTypeException schemaMismatch(Schema to) { + return new MessageTypeException("schema mismatch "+to.getExpression()); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/Packer.java b/java-plan3/src/org/msgpack/Packer.java new file mode 100644 index 00000000..7f2508c3 --- /dev/null +++ b/java-plan3/src/org/msgpack/Packer.java @@ -0,0 +1,409 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) { + this.out = out; + } + + public Packer packByte(byte d) throws IOException { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException { + out.write(b, off, length); + return this; + } + + + public Packer packWithSchema(Object o, Schema s) throws IOException { + s.pack(this, o); + return this; + } + + + public Packer packString(String s) throws IOException { + byte[] b = ((String)s).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } + + + public Packer pack(String o) throws IOException { + if(o == null) { return packNil(); } + return packString(o); + } + + public Packer pack(MessagePackable o) throws IOException { + if(o == null) { return packNil(); } + o.messagePack(this); + return this; + } + + public Packer pack(byte[] o) throws IOException { + if(o == null) { return packNil(); } + packRaw(o.length); + return packRawBody(o); + } + + public Packer pack(List o) throws IOException { + if(o == null) { return packNil(); } + packArray(o.size()); + for(Object i : o) { pack(i); } + return this; + } + + @SuppressWarnings("unchecked") + public Packer pack(Map o) throws IOException { + if(o == null) { return packNil(); } + packMap(o.size()); + for(Map.Entry e : ((Map)o).entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } + + public Packer pack(Boolean o) throws IOException { + if(o == null) { return packNil(); } + if(o) { + return packTrue(); + } else { + return packFalse(); + } + } + + public Packer pack(Byte o) throws IOException { + if(o == null) { return packNil(); } + return packByte(o); + } + + public Packer pack(Short o) throws IOException { + if(o == null) { return packNil(); } + return packShort(o); + } + + public Packer pack(Integer o) throws IOException { + if(o == null) { return packNil(); } + return packInt(o); + } + + public Packer pack(Long o) throws IOException { + if(o == null) { return packNil(); } + return packLong(o); + } + + public Packer pack(Float o) throws IOException { + if(o == null) { return packNil(); } + return packFloat(o); + } + + public Packer pack(Double o) throws IOException { + if(o == null) { return packNil(); } + return packDouble(o); + } + + + @SuppressWarnings("unchecked") + public Packer pack(Object o) throws IOException { + if(o == null) { + return packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + return this; + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + return this; + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } else if(o instanceof Boolean) { + if((Boolean)o) { + return packTrue(); + } else { + return packFalse(); + } + } else if(o instanceof Integer) { + return packInt((Integer)o); + } else if(o instanceof Long) { + return packLong((Long)o); + } else if(o instanceof Short) { + return packShort((Short)o); + } else if(o instanceof Byte) { + return packByte((Byte)o); + } else if(o instanceof Float) { + return packFloat((Float)o); + } else if(o instanceof Double) { + return packDouble((Double)o); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + } +} + diff --git a/java-plan3/src/org/msgpack/Schema.java b/java-plan3/src/org/msgpack/Schema.java new file mode 100644 index 00000000..f99b3d0a --- /dev/null +++ b/java-plan3/src/org/msgpack/Schema.java @@ -0,0 +1,133 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.Writer; +import java.io.IOException; +import org.msgpack.schema.SSchemaParser; +import org.msgpack.schema.ClassGenerator; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) { + this.expression = expression; + this.name = name; + } + + public String getName() { + return name; + } + + public String getFullName() { + return name; + } + + public String getExpression() { + return name; + } + + public static Schema parse(String source) { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) { + return SSchemaParser.load(source); + } + + public void write(Writer output) throws IOException { + ClassGenerator.write(this, output); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + + public abstract Object convert(Object obj) throws MessageTypeException; + + + public Object createFromNil() { + return null; + } + + public Object createFromBoolean(boolean v) { + throw new RuntimeException("type error"); + } + + public Object createFromByte(byte v) { + throw new RuntimeException("type error"); + } + + public Object createFromShort(short v) { + throw new RuntimeException("type error"); + } + + public Object createFromInt(int v) { + throw new RuntimeException("type error"); + } + + public Object createFromLong(long v) { + throw new RuntimeException("type error"); + } + + public Object createFromFloat(float v) { + throw new RuntimeException("type error"); + } + + public Object createFromDouble(double v) { + throw new RuntimeException("type error"); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw new RuntimeException("type error"); + } + + /* FIXME + public Object createFromBoolean(boolean v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromByte(byte v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromShort(short v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromInt(int v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromLong(long v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromFloat(float v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromDouble(double v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw MessageTypeException.schemaMismatch(this); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 00000000..471605f5 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,82 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() { + super(); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public Object getData() { + return data; + } + + public boolean isFinished() { + return finished; + } + + public void reset() { + super.reset(); + this.offset = 0; + } + + int getOffset() { + return offset; + } + + void setOffset(int offset) { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackException.java b/java-plan3/src/org/msgpack/UnpackException.java new file mode 100644 index 00000000..db08d959 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackException.java @@ -0,0 +1,29 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class UnpackException extends IOException { + public UnpackException() { } + + public UnpackException(String s) { + super(s); + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackIterator.java b/java-plan3/src/org/msgpack/UnpackIterator.java new file mode 100644 index 00000000..9975b680 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,66 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() { + if(!have && !hasNext()) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan3/src/org/msgpack/Unpacker.java b/java-plan3/src/org/msgpack/Unpacker.java new file mode 100644 index 00000000..af211c69 --- /dev/null +++ b/java-plan3/src/org/msgpack/Unpacker.java @@ -0,0 +1,245 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() { + this(DEFAULT_BUFFER_SIZE); + } + + public Unpacker(int bufferReserveSize) { + this(null, bufferReserveSize); + } + + public Unpacker(InputStream stream) { + this(stream, DEFAULT_BUFFER_SIZE); + } + + public Unpacker(InputStream stream, int bufferReserveSize) { + super(); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public void reserveBuffer(int size) { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() { + return buffer; + } + + public int getBufferOffset() { + return used; + } + + public int getBufferCapacity() { + return buffer.length - used; + } + + public void bufferConsumed(int size) { + used += size; + } + + public void feed(byte[] buffer) { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() { + return super.getData(); + } + + public void reset() { + super.reset(); + parsed = 0; + } + + public int getMessageSize() { + return parsed - offset + used; + } + + public int getParsedSize() { + return parsed; + } + + public int getNonParsedSize() { + return used - offset; + } + + public void skipNonparsedBuffer(int size) { + offset += size; + } + + public void removeNonparsedBuffer() { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 00000000..47a1800d --- /dev/null +++ b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,483 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.*; +import org.msgpack.schema.GenericSchema; +import org.msgpack.schema.IMapSchema; +import org.msgpack.schema.IArraySchema; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + private int cs; + private int trail; + private int top; + private int[] stack_ct = new int[MAX_STACK_SIZE]; + private int[] stack_count = new int[MAX_STACK_SIZE]; + private Object[] stack_obj = new Object[MAX_STACK_SIZE]; + private Schema[] stack_schema = new Schema[MAX_STACK_SIZE]; + private int top_ct; + private int top_count; + private Object top_obj; + private Schema top_schema; + private ByteBuffer castBuffer = ByteBuffer.allocate(8); + private boolean finished = false; + private Object data = null; + + private static final Schema GENERIC_SCHEMA = new GenericSchema(); + private Schema rootSchema; + + protected UnpackerImpl() + { + setSchema(GENERIC_SCHEMA); + } + + protected void setSchema(Schema schema) + { + this.rootSchema = schema; + reset(); + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + cs = CS_HEADER; + top = -1; + finished = false; + data = null; + top_ct = 0; + top_count = 0; + top_obj = null; + top_schema = rootSchema; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + //System.out.println("fixarray count:"+count); + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = top_schema.createFromNil(); + break _push; + case 0xc2: // false + obj = top_schema.createFromBoolean(false); + break _push; + case 0xc3: // true + obj = top_schema.createFromBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = top_schema.createFromShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported"); + } else { + obj = top_schema.createFromLong( o ); + } + } + break _push; + case CS_INT_8: + obj = top_schema.createFromByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = top_schema.createFromRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + ++top; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + default: + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(top_ct) { + case CT_ARRAY_ITEM: { + //System.out.println("array item "+obj); + Object[] ar = (Object[])top_obj; + ar[ar.length - top_count] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IArraySchema)top_schema).createFromArray(ar); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } else { + top_schema = ((IArraySchema)stack_schema[top]).getElementSchema(ar.length - top_count); + } + break _header_again; + } + case CT_MAP_KEY: { + //System.out.println("map key:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2] = obj; + top_ct = CT_MAP_VALUE; + top_schema = ((IMapSchema)stack_schema[top]).getValueSchema(); + break _header_again; + } + case CT_MAP_VALUE: { + //System.out.println("map value:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2 + 1] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IMapSchema)top_schema).createFromMap(mp); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } + top_ct = CT_MAP_KEY; + break _header_again; + } + default: + throw new UnpackException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ArraySchema.java b/java-plan3/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 00000000..24fa7580 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,125 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; +import java.util.RandomAccess; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema implements IArraySchema { + private Schema elementSchema; + + public ArraySchema(Schema elementSchema) + { + super("array"); + this.elementSchema = elementSchema; + } + + @Override + public String getFullName() + { + return "List<"+elementSchema.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj instanceof List) { + ArrayList d = (ArrayList)obj; + pk.packArray(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + elementSchema.pack(pk, d.get(i)); + } + } else { + for(Object e : d) { + elementSchema.pack(pk, e); + } + } + + } else if(obj instanceof Set) { + Set d = (Set)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementSchema.pack(pk, e); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException + { + if(obj instanceof List) { + List d = (List)obj; + ArrayList ar = new ArrayList(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + ar.add( elementSchema.convert(d.get(i)) ); + } + } else { + for(Object e : d) { + ar.add( elementSchema.convert(e) ); + } + } + return ar; + + } else if(obj instanceof Collection) { + Collection d = (Collection)obj; + ArrayList ar = new ArrayList(d.size()); + for(Object e : (Collection)obj) { + ar.add( elementSchema.convert(e) ); + } + return ar; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getElementSchema(int index) + { + return elementSchema; + } + + @Override + public Object createFromArray(Object[] obj) + { + return Arrays.asList(obj); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ByteSchema.java b/java-plan3/src/org/msgpack/schema/ByteSchema.java new file mode 100644 index 00000000..3bc70456 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ByteSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ByteSchema extends Schema { + public ByteSchema() { + super("Byte"); + } + + @Override + public String getExpression() { + return "byte"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packByte( ((Number)obj).byteValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Byte) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).byteValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (byte)v; + } + + @Override + public Object createFromShort(short v) { + return (byte)v; + } + + @Override + public Object createFromInt(int v) { + return (byte)v; + } + + @Override + public Object createFromLong(long v) { + return (byte)v; + } + + @Override + public Object createFromFloat(float v) { + return (byte)v; + } + + @Override + public Object createFromDouble(double v) { + return (byte)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassGenerator.java b/java-plan3/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 00000000..65213aad --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,241 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.Writer; +import org.msgpack.*; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getSchema()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getSchema()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementSchema(0)); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeySchema()); + findSubclassSchema(dst, as.getValueSchema()); + } + } + + private void writeHeader() throws IOException { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.ClassSchema;"); + line("import org.msgpack.schema.FieldSchema;"); + } + + private void writeFooter() throws IOException { + line(); + } + + private void writeClass() throws IOException { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getSchema().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeMergeFunction(); + writeFactoryFunction(); + } + + private void writeConstructors() throws IOException { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException { + line(); + line("@Override"); + line("public void messagePack(Packer _pk) throws IOException"); + line("{"); + pushIndent(); + line("_pk.packArray("+schema.getFields().length+");"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_fields["+i+"].getSchema().pack(_pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeMergeFunction() throws IOException { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageMerge(Object obj) throws MessageTypeException"); + line("{"); + pushIndent(); + line("Object[] _source = ((List)obj).toArray();"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getFullName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeFactoryFunction() throws IOException { + line(); + line("@SuppressWarnings(\"unchecked\")"); + line("public static "+schema.getName()+" createFromMessage(Object[] _message)"); + line("{"); + pushIndent(); + line(schema.getName()+" _self = new "+schema.getName()+"();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getFullName()+")_message["+i+"];"); + ++i; + } + line("return _self;"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException { + writer.write("\n"); + } + + private void pushIndent() { + indent += 1; + } + + private void popIndent() { + indent -= 1; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassSchema.java b/java-plan3/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 00000000..75315e70 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,95 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import org.msgpack.*; + +public abstract class ClassSchema extends Schema implements IArraySchema { + protected FieldSchema[] fields; + protected List imports; + protected String namespace; + protected String fqdn; + + public ClassSchema( + String name, String namespace, + List imports, List fields) { + super(name); + this.namespace = namespace; + this.imports = imports; // FIXME clone? + this.fields = new FieldSchema[fields.size()]; + System.arraycopy(fields.toArray(), 0, this.fields, 0, fields.size()); + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + public final FieldSchema[] getFields() { + return fields; + } + + String getNamespace() { + return namespace; + } + + List getImports() { + return imports; + } + + void setNamespace(String namespace) { + this.namespace = namespace; + } + + void setImports(List imports) { + this.imports = imports; // FIXME clone? + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + @Override + public String getExpression() { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(FieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + public boolean equals(SpecificClassSchema o) { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/DoubleSchema.java b/java-plan3/src/org/msgpack/schema/DoubleSchema.java new file mode 100644 index 00000000..feffbb98 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/DoubleSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class DoubleSchema extends Schema { + public DoubleSchema() { + super("Double"); + } + + @Override + public String getExpression() { + return "double"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packDouble( ((Number)obj).doubleValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Double) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).doubleValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (double)v; + } + + @Override + public Object createFromShort(short v) { + return (double)v; + } + + @Override + public Object createFromInt(int v) { + return (double)v; + } + + @Override + public Object createFromFloat(float v) { + return (double)v; + } + + @Override + public Object createFromDouble(double v) { + return (double)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FieldSchema.java b/java-plan3/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 00000000..3391f2b8 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,43 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public class FieldSchema { + private String name; + private Schema schema; + + public FieldSchema(String name, Schema schema) { + this.name = name; + this.schema = schema; + } + + public final String getName() { + return name; + } + + public final Schema getSchema() { + return schema; + } + + public String getExpression() { + return "(field "+name+" "+schema.getExpression()+")"; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FloatSchema.java b/java-plan3/src/org/msgpack/schema/FloatSchema.java new file mode 100644 index 00000000..2f4240a8 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FloatSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class FloatSchema extends Schema { + public FloatSchema() { + super("Float"); + } + + @Override + public String getExpression() { + return "float"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packFloat( ((Number)obj).floatValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Float) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).floatValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (float)v; + } + + @Override + public Object createFromShort(short v) { + return (float)v; + } + + @Override + public Object createFromInt(int v) { + return (float)v; + } + + @Override + public Object createFromFloat(float v) { + return (float)v; + } + + @Override + public Object createFromDouble(double v) { + return (float)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 00000000..736bdfa3 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,91 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class GenericClassSchema extends ClassSchema { + public GenericClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packArray(fields.length); + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + f.getSchema().pack(pk, d.get(f.getName())); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + // FIXME optimize + return createFromArray( ((Collection)obj).toArray() ); + + } else if(obj instanceof Map) { + HashMap m = new HashMap(fields.length); + Map d = (Map)obj; + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + String fieldName = f.getName(); + m.put(fieldName, f.getSchema().convert(d.get(fieldName))); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + HashMap m = new HashMap(fields.length); + int i=0; + for(; i < obj.length; ++i) { + m.put(fields[i].getName(), obj[i]); + } + for(; i < fields.length; ++i) { + m.put(fields[i].getName(), null); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericSchema.java b/java-plan3/src/org/msgpack/schema/GenericSchema.java new file mode 100644 index 00000000..52e01616 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericSchema.java @@ -0,0 +1,192 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; +//import org.msgpack.generic.*; + +public class GenericSchema extends Schema implements IArraySchema, IMapSchema { + public GenericSchema() { + super("Object"); + } + + @Override + public String getExpression() { + return "object"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + pk.pack(obj); + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return obj; + } + + @Override + public Schema getElementSchema(int index) { + return this; + } + + @Override + public Schema getKeySchema() { + return this; + } + + @Override + public Schema getValueSchema() { + return this; + } + + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return v; + } + + @Override + public Object createFromByte(byte v) { + return v; + } + + @Override + public Object createFromShort(short v) { + return v; + } + + @Override + public Object createFromInt(int v) { + return v; + } + + @Override + public Object createFromLong(long v) { + return v; + } + + @Override + public Object createFromFloat(float v) { + return v; + } + + @Override + public Object createFromDouble(double v) { + return v; + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] bytes = new byte[length]; + System.arraycopy(b, offset, bytes, 0, length); + return bytes; + } + + @Override + public Object createFromArray(Object[] obj) { + return Arrays.asList(obj); + } + + @Override + @SuppressWarnings("unchecked") + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + + /* + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return new GenericBoolean(v); + } + + @Override + public Object createFromFromByte(byte v) { + return new GenericByte(v); + } + + @Override + public Object createFromShort(short v) { + return new GenericShort(v); + } + + @Override + public Object createFromInt(int v) { + return new GenericInt(v); + } + + @Override + public Object createFromLong(long v) { + return new GenericLong(v); + } + + @Override + public Object createFromFloat(float v) { + return new GenericFloat(v); + } + + @Override + public Object createFromDouble(double v) { + return new GenericDouble(v); + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + return new GenericRaw(b, offset, length); + } + + @Override + public Object createFromArray(Object[] obj) { + // FIXME GenericArray + return Arrays.asList(obj); + } + + @Override + public Object createFromMap(Object[] obj) { + GenericMap m = new GenericMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/IArraySchema.java b/java-plan3/src/org/msgpack/schema/IArraySchema.java new file mode 100644 index 00000000..ccc6d14c --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IArraySchema.java @@ -0,0 +1,26 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IArraySchema { + public Schema getElementSchema(int index); + public Object createFromArray(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IMapSchema.java b/java-plan3/src/org/msgpack/schema/IMapSchema.java new file mode 100644 index 00000000..60b3e8e8 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IMapSchema.java @@ -0,0 +1,27 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IMapSchema { + public Schema getKeySchema(); + public Schema getValueSchema(); + public Object createFromMap(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IntSchema.java b/java-plan3/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 00000000..c54c0aec --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() { + super("Integer"); + } + + @Override + public String getExpression() { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packInt( ((Number)obj).intValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Integer) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).intValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (int)v; + } + + @Override + public Object createFromShort(short v) { + return (int)v; + } + + @Override + public Object createFromInt(int v) { + return (int)v; + } + + @Override + public Object createFromLong(long v) { + return (int)v; + } + + @Override + public Object createFromFloat(float v) { + return (int)v; + } + + @Override + public Object createFromDouble(double v) { + return (int)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/LongSchema.java b/java-plan3/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 00000000..ccf30433 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() { + super("Long"); + } + + @Override + public String getExpression() { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packLong( ((Number)obj).longValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Long) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).longValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (long)v; + } + + @Override + public Object createFromShort(short v) { + return (long)v; + } + + @Override + public Object createFromInt(int v) { + return (long)v; + } + + @Override + public Object createFromLong(long v) { + return (long)v; + } + + @Override + public Object createFromFloat(float v) { + return (long)v; + } + + @Override + public Object createFromDouble(double v) { + return (long)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/MapSchema.java b/java-plan3/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 00000000..71629b02 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,102 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema implements IMapSchema { + private Schema keySchema; + private Schema valueSchema; + + public MapSchema(Schema keySchema, Schema valueSchema) { + super("map"); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + } + + @Override + public String getFullName() { + return "HashList<"+keySchema.getFullName()+", "+valueSchema.getFullName()+">"; + } + + @Override + public String getExpression() { + return "(map "+keySchema.getExpression()+" "+valueSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keySchema.pack(pk, e.getKey()); + valueSchema.pack(pk, e.getValue()); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Map) { + Map d = (Map)obj; + Map m = new HashMap(); + for(Map.Entry e : d.entrySet()) { + m.put(keySchema.convert(e.getKey()), valueSchema.convert(e.getValue())); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getKeySchema() { + return keySchema; + } + + @Override + public Schema getValueSchema() { + return valueSchema; + } + + @Override + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/RawSchema.java b/java-plan3/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 00000000..582f766a --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,105 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() { + super("raw"); + } + + public String getFullName() { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + // FIXME copy? + //byte[] d = (byte[])obj; + //byte[] v = new byte[d.length]; + //System.arraycopy(d, 0, v, 0, d.length); + //return v; + return obj; + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return v; + + } else if(obj instanceof String) { + try { + return ((String)obj).getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java new file mode 100644 index 00000000..fb94adfa --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java @@ -0,0 +1,64 @@ +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.lang.reflect.*; +import org.msgpack.*; + +// FIXME +public abstract class ReflectionClassSchema extends ClassSchema { + private Constructor constructorCache; + + public ReflectionClassSchema(String name, List fields, String namespace, List imports) { + super(name, namespace, imports, fields); + } + + /* + Schema getElementSchema(int index) + { + // FIXME check index < fields.length + fields[index].getSchema(); + } + + Object createFromArray(Object[] obj) + { + Object o = newInstance(); + ((MessageConvertable)o).messageConvert(obj); + return o; + } + + Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/SSchemaParser.java b/java-plan3/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 00000000..c6bbc773 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,254 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import org.msgpack.*; + +// FIXME exception class + +public class SSchemaParser { + public static Schema parse(String source) { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) { + if(exp.isAtom()) { + String type = exp.getAtom(); + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("byte")) { + return new ByteSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("float")) { + return new FloatSchema(); + } else if(type.equals("double")) { + return new DoubleSchema(); + } else if(type.equals("object")) { + return new GenericSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, namespace, imports, fields); + } else { + return new GenericClassSchema(name, namespace, imports, fields); + } + } + + private ArraySchema parseArray(SExp exp) { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + return new FieldSchema(name, type); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ShortSchema.java b/java-plan3/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 00000000..089a0249 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() { + super("Short"); + } + + @Override + public String getExpression() { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packShort( ((Number)obj).shortValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Short) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).shortValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (short)v; + } + + @Override + public Object createFromShort(short v) { + return (short)v; + } + + @Override + public Object createFromInt(int v) { + return (short)v; + } + + @Override + public Object createFromLong(long v) { + return (short)v; + } + + @Override + public Object createFromFloat(float v) { + return (short)v; + } + + @Override + public Object createFromDouble(double v) { + return (short)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 00000000..81e5e002 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,122 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.lang.reflect.*; +import java.io.IOException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private Class classCache; + private Method factoryCache; + private Constructor constructorCache; + + public SpecificClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj == null) { + pk.packNil(); + return; + } + if(classCache == null) { + cacheFactory(); + } + if(classCache.isInstance(obj)) { + ((MessagePackable)obj).messagePack(pk); + } else { + // FIXME Map + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + if(constructorCache == null) { + cacheConstructor(); + } + try { + MessageMergeable o = (MessageMergeable)constructorCache.newInstance((Object[])null); + o.messageMerge(obj); + return o; + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + if(factoryCache == null) { + cacheFactory(); + } + try { + return factoryCache.invoke(null, new Object[]{obj}); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getCause()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheFactory() { + try { + classCache = Class.forName(fqdn); + factoryCache = classCache.getDeclaredMethod("createFromMessage", new Class[]{Object[].class}); + factoryCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheConstructor() { + try { + classCache = Class.forName(fqdn); + constructorCache = classCache.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } +} + diff --git a/java-plan3/src/org/msgpack/schema/StringSchema.java b/java-plan3/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 00000000..f5e0bf12 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,111 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() { + super("string"); + } + + @Override + public String getFullName() { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + return obj; + + } else if(obj instanceof byte[]) { + try { + return new String((byte[])obj, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + try { + if(d.hasArray()) { + return new String(d.array(), d.position(), d.capacity(), "UTF-8"); + } else { + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return new String(v, "UTF-8"); + } + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + try { + return new String(b, offset, length, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan3/test/Generate.java b/java-plan3/test/Generate.java new file mode 100644 index 00000000..1b72e903 --- /dev/null +++ b/java-plan3/test/Generate.java @@ -0,0 +1,38 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + String source = + "(class MediaContent"+ + " (package serializers.msgpack)"+ + " (field image (array (class Image"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field size int))))"+ + " (field media (class Media"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field format string)"+ + " (field duration long)"+ + " (field size long)"+ + " (field bitrate int)"+ + " (field person (array string))"+ + " (field player int)"+ + " (field copyright string)))"+ + " )"; + + Schema schema = Schema.parse(source); + + Writer output = new OutputStreamWriter(System.out); + ClassGenerator.write(schema, output); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 00000000..5dfbc8d2 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,173 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.ClassSchema; +import org.msgpack.schema.FieldSchema; + +public final class MediaContent implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public List image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(2); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, image); + _fields[1].getSchema().pack(_pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.image = (List)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.media = (Media)_fields[1].getSchema().convert(_source[1]); + } + + @SuppressWarnings("unchecked") + public static MediaContent createFromMessage(Object[] _message) + { + MediaContent _self = new MediaContent(); + if(_message.length <= 0) { return _self; } _self.image = (List)_message[0]; + if(_message.length <= 1) { return _self; } _self.media = (Media)_message[1]; + return _self; + } +} + +final class Image implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(5); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.size = (Integer)_fields[4].getSchema().convert(_source[4]); + } + + @SuppressWarnings("unchecked") + public static Image createFromMessage(Object[] _message) + { + Image _self = new Image(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.size = (Integer)_message[4]; + return _self; + } +} + +final class Media implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public List person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(11); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, format); + _fields[5].getSchema().pack(_pk, duration); + _fields[6].getSchema().pack(_pk, size); + _fields[7].getSchema().pack(_pk, bitrate); + _fields[8].getSchema().pack(_pk, person); + _fields[9].getSchema().pack(_pk, player); + _fields[10].getSchema().pack(_pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.format = (String)_fields[4].getSchema().convert(_source[4]); + if(_source.length <= 5) { return; } this.duration = (Long)_fields[5].getSchema().convert(_source[5]); + if(_source.length <= 6) { return; } this.size = (Long)_fields[6].getSchema().convert(_source[6]); + if(_source.length <= 7) { return; } this.bitrate = (Integer)_fields[7].getSchema().convert(_source[7]); + if(_source.length <= 8) { return; } this.person = (List)_fields[8].getSchema().convert(_source[8]); + if(_source.length <= 9) { return; } this.player = (Integer)_fields[9].getSchema().convert(_source[9]); + if(_source.length <= 10) { return; } this.copyright = (String)_fields[10].getSchema().convert(_source[10]); + } + + @SuppressWarnings("unchecked") + public static Media createFromMessage(Object[] _message) + { + Media _self = new Media(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.format = (String)_message[4]; + if(_message.length <= 5) { return _self; } _self.duration = (Long)_message[5]; + if(_message.length <= 6) { return _self; } _self.size = (Long)_message[6]; + if(_message.length <= 7) { return _self; } _self.bitrate = (Integer)_message[7]; + if(_message.length <= 8) { return _self; } _self.person = (List)_message[8]; + if(_message.length <= 9) { return _self; } _self.player = (Integer)_message[9]; + if(_message.length <= 10) { return _self; } _self.copyright = (String)_message[10]; + return _self; + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 00000000..547ba487 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java new file mode 100644 index 00000000..c8a88acc --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java @@ -0,0 +1,68 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackDynamicSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-dynamic"; + } + + public Object create() throws Exception { + ArrayList media = new ArrayList(11); + media.add("http://javaone.com/keynote.mpg"); + media.add("video/mpg4"); + media.add("Javaone Keynote"); + media.add(1234567L); + media.add(0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.add(person); + media.add(0); + media.add(0); + media.add(0); + media.add(123L); + media.add(""); + + ArrayList image1 = new ArrayList(5); + image1.add("http://javaone.com/keynote_large.jpg"); + image1.add(0); + image1.add(0); + image1.add(2); + image1.add("Javaone Keynote"); + + ArrayList image2 = new ArrayList(5); + image2.add("http://javaone.com/keynote_thumbnail.jpg"); + image2.add(0); + image2.add(0); + image2.add(1); + image2.add("Javaone Keynote"); + + ArrayList content = new ArrayList(2); + content.add(media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.add(images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java new file mode 100644 index 00000000..49358996 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackGenericSerializer implements ObjectSerializer +{ + private static final Schema MEDIA_CONTENT_SCHEMA = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + + public String getName() { + return "msgpack-generic"; + } + + public Object create() throws Exception { + HashMap media = new HashMap(11); + media.put("uri", "http://javaone.com/keynote.mpg"); + media.put("format", "video/mpg4"); + media.put("title", "Javaone Keynote"); + media.put("duration", 1234567L); + media.put("bitrate", 0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.put("person", person); + media.put("player", 0); + media.put("height", 0); + media.put("width", 0); + media.put("size", 123L); + media.put("copyright", ""); + + HashMap image1 = new HashMap(5); + image1.put("uri", "http://javaone.com/keynote_large.jpg"); + image1.put("width", 0); + image1.put("height", 0); + image1.put("size", 2); + image1.put("title", "Javaone Keynote"); + + HashMap image2 = new HashMap(5); + image2.put("uri", "http://javaone.com/keynote_thumbnail.jpg"); + image2.put("width", 0); + image2.put("height", 0); + image2.put("size", 1); + image2.put("title", "Javaone Keynote"); + + HashMap content = new HashMap(2); + content.put("media", media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.put("image", images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.packWithSchema(content, MEDIA_CONTENT_SCHEMA); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MEDIA_CONTENT_SCHEMA); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java new file mode 100644 index 00000000..2767474c --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java @@ -0,0 +1,67 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackIndirectSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-indirect"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + Object obj = pac.getData(); + return (MediaContent)MediaContent.getSchema().convert(obj); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java new file mode 100644 index 00000000..91ded5cd --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java @@ -0,0 +1,66 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackSpecificSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } +} + From 0ae1965f6b492efb71f7457ba1dec48ae6110399 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 04:32:33 +0900 Subject: [PATCH 07/24] ruby: fixes MessagePack_Unpacker_mark marks uninitialized map_key --- ruby/unpack.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index 411a94d2..e9d6494e 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -127,6 +127,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha static VALUE cUnpacker; static VALUE eUnpackError; +// FIXME slow operation +static void init_stack(msgpack_unpack_t* mp) +{ + size_t i; + for(i=0; i < MSGPACK_MAX_STACK_SIZE; ++i) { + mp->stack[i].map_key = Qnil; /* GC */ + } +} + static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -137,7 +146,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) unsigned int i; for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].map_key); + rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ } } @@ -154,6 +163,7 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); template_init(mp); + init_stack(mp); unpack_user u = {0, Qnil}; mp->user = u; return self; @@ -281,6 +291,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) msgpack_unpack_t mp; template_init(&mp); + init_stack(&mp); unpack_user u = {0, Qnil}; mp.user = u; From 7ce866ad7c2884f0b01ee77d99d2f9e01217fcdf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:19:53 +0900 Subject: [PATCH 08/24] msgpack template: architecture specific endian conversion --- c/object.c | 10 ++ c/object.h | 4 +- c/pack.h | 4 +- c/unpack.c | 35 ++++-- c/unpack.h | 6 +- c/vrefbuffer.h | 9 ++ cpp/object.hpp | 1 - cpp/pack.hpp | 3 +- msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 253 +++++++++++++++----------------------- msgpack/unpack_define.h | 39 +----- msgpack/unpack_template.h | 6 +- 12 files changed, 154 insertions(+), 219 deletions(-) diff --git a/c/object.c b/c/object.c index bcb2537e..a22ce214 100644 --- a/c/object.c +++ b/c/object.c @@ -18,7 +18,17 @@ #include "msgpack/object.h" #include "msgpack/pack.h" #include + +#ifndef _MSC_VER #include +#else +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif +#ifndef PRIi64 +#define PRIi64 "I64d" +#endif +#endif int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) diff --git a/c/object.h b/c/object.h index 7c603b35..0aed0e45 100644 --- a/c/object.h +++ b/c/object.h @@ -19,9 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include -#include -#include +#include "msgpack/sys.h" #include #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index 1a57ea4e..1525e0f2 100644 --- a/c/pack.h +++ b/c/pack.h @@ -18,11 +18,9 @@ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ -#include -#include -#include #include "msgpack/pack_define.h" #include "msgpack/object.h" +#include #ifdef __cplusplus extern "C" { diff --git a/c/unpack.c b/c/unpack.c index 08fd6cbf..4d9af9ea 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -101,7 +101,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac { o->type = MSGPACK_OBJECT_ARRAY; o->via.array.size = 0; - o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + o->via.array.ptr = (msgpack_object*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); if(o->via.array.ptr == NULL) { return -1; } return 0; } @@ -142,30 +142,47 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced -static const size_t COUNTER_SIZE = sizeof(unsigned int); +#ifndef _MSC_VER +typedef unsigned int counter_t; +#else +typedef long counter_t; +#endif + +#define COUNTER_SIZE (sizeof(volatile counter_t)) + static inline void init_count(void* buffer) { - *(volatile unsigned int*)buffer = 1; + *(volatile counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } + if( +#ifndef _MSC_VER + __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 +#else + InterlockedDecrement((volatile counter_t*)&buffer) == 0 +#endif + ) { free(buffer); } } static inline void incr_count(void* buffer) { - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); + // atomic ++*(counter_t*)buffer; +#ifndef _MSC_VER + __sync_add_and_fetch((counter_t*)buffer, 1); +#else + InterlockedIncrement((volatile counter_t*)&buffer); +#endif } -static inline unsigned int get_count(void* buffer) +static inline counter_t get_count(void* buffer) { - return *(volatile unsigned int*)buffer; + return *(volatile counter_t*)buffer; } diff --git a/c/unpack.h b/c/unpack.h index ef637744..e17d0d85 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef msgpack_unpacker_H__ -#define msgpack_unpacker_H__ +#ifndef MSGPACK_UNPACKER_H__ +#define MSGPACK_UNPACKER_H__ #include "msgpack/zone.h" #include "msgpack/object.h" -#include -#include #ifdef __cplusplus extern "C" { diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index baa7c030..063075f8 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -19,7 +19,16 @@ #define MSGPACK_VREFBUFFER_H__ #include "msgpack/zone.h" + +#ifndef _WIN32 #include +#else +struct iovec { + void *iov_base; + size_t iov_len; +}; +#endif + #ifdef __cplusplus extern "C" { diff --git a/cpp/object.hpp b/cpp/object.hpp index 9ee575fc..ed2e2905 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -20,7 +20,6 @@ #include "msgpack/object.h" #include "msgpack/pack.hpp" -#include #include #include #include diff --git a/cpp/pack.hpp b/cpp/pack.hpp index c8e37eb8..9c291c1c 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -18,10 +18,9 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include // __BYTE_ORDER +#include "msgpack/pack_define.h" #include #include -#include "msgpack/pack_define.h" namespace msgpack { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 33408e58..aef92953 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index aa620f53..ffbbbba5 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63668c24..027d409b 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,39 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index d67fd1ef..212a47e0 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -126,9 +126,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) if(p == pe) { goto _out; } do { From ba3ba0367cefdb2e87425f257711865e2e540adc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:40:29 +0900 Subject: [PATCH 09/24] msgpack template: macros for compilers that doesn't not support Case Ranges --- msgpack/unpack_template.h | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 212a47e0..6f99d549 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -130,16 +135,28 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) #define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif + if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -182,16 +199,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From 35929b46ae1ed8cd001d8b0964ec3a1bb1c053e1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 07:22:39 +0900 Subject: [PATCH 10/24] add msgpack/sysdep.h --- Makefile.am | 3 +- c/object.h | 2 +- c/unpack.c | 33 ++++---------- configure.in | 2 +- msgpack/pack_define.h | 2 +- msgpack/pack_template.h | 64 +++++++++++++------------- msgpack/sysdep.h | 94 +++++++++++++++++++++++++++++++++++++++ msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 6 +-- 9 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 msgpack/sysdep.h diff --git a/Makefile.am b/Makefile.am index be3d75f7..42fb2335 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,5 +8,6 @@ nobase_include_HEADERS = \ msgpack/pack_define.h \ msgpack/pack_template.h \ msgpack/unpack_define.h \ - msgpack/unpack_template.h + msgpack/unpack_template.h \ + msgpack/sysdep.h diff --git a/c/object.h b/c/object.h index 0aed0e45..27b593ef 100644 --- a/c/object.h +++ b/c/object.h @@ -19,7 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #ifdef __cplusplus diff --git a/c/unpack.c b/c/unpack.c index 4d9af9ea..6a435ba1 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -141,48 +141,31 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - -#ifndef _MSC_VER -typedef unsigned int counter_t; -#else -typedef long counter_t; -#endif - -#define COUNTER_SIZE (sizeof(volatile counter_t)) +#define COUNTER_SIZE (sizeof(_msgpack_atomic_counter_t)) static inline void init_count(void* buffer) { - *(volatile counter_t*)buffer = 1; + *(volatile _msgpack_atomic_counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } - if( -#ifndef _MSC_VER - __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 -#else - InterlockedDecrement((volatile counter_t*)&buffer) == 0 -#endif - ) { + // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { free(buffer); } } static inline void incr_count(void* buffer) { - // atomic ++*(counter_t*)buffer; -#ifndef _MSC_VER - __sync_add_and_fetch((counter_t*)buffer, 1); -#else - InterlockedIncrement((volatile counter_t*)&buffer); -#endif + // atomic ++*(_msgpack_atomic_counter_t*)buffer; + _msgpack_sync_incr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer); } -static inline counter_t get_count(void* buffer) +static inline _msgpack_atomic_counter_t get_count(void* buffer) { - return *(volatile counter_t*)buffer; + return *(volatile _msgpack_atomic_counter_t*)buffer; } diff --git a/configure.in b/configure.in index c2ca872b..76bd7e4f 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.8) +AM_INIT_AUTOMAKE(msgpack, 0.3.9) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index aef92953..692ef7d5 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index ffbbbba5..de148bf6 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -115,17 +115,17 @@ do { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -283,21 +283,21 @@ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -310,21 +310,21 @@ msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -557,7 +557,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -566,7 +566,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +610,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +631,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -652,11 +652,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h new file mode 100644 index 00000000..106158ea --- /dev/null +++ b/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 027d409b..c36aa51e 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #include #include diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6f99d549..3328ea3e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -131,9 +131,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { From 0d44348c7dabdbfee4ac1ee17bfb141dfe8706f2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 11 Dec 2009 04:13:24 +0900 Subject: [PATCH 11/24] ruby: version 0.3.2 --- ruby/gem/Rakefile | 6 +++--- ruby/gengem.sh | 1 + ruby/msgpack.gemspec | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 5445906e..f2fd6846 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -15,9 +15,9 @@ AUTHOR = "FURUHASHI Sadayuki" EMAIL = "frsyuki _at_ users.sourceforge.jp" DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" -HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +HOMEPATH = "http://msgpack.sourceforge.jp/" BIN_FILES = %w( ) -VERS = "0.3.1" +VERS = "0.3.2" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil @@ -44,7 +44,7 @@ spec = Gem::Specification.new do |s| s.name = NAME s.version = VERS s.platform = Gem::Platform::RUBY - s.has_rdoc = true + s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] s.summary = DESCRIPTION diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 0afb8f52..359debf3 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -15,6 +15,7 @@ cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ +cp ../msgpack/sysdep.h gem/msgpack/ cd gem && rake --trace package diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index f0b5c44c..59186a4c 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.1" + s.version = "0.3.2" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 5aa47d667783476277409b06d5829322a801df05 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 03:52:14 +0900 Subject: [PATCH 12/24] cpp: zone::push_finalizer supports std::auto_ptr --- cpp/zone.hpp.erb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 8fd14a6f..f1e46247 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -21,6 +21,7 @@ #include "msgpack/object.hpp" #include "msgpack/zone.h" #include +#include #include <% GENERATION_LIMIT = 15 %> @@ -38,6 +39,9 @@ public: void push_finalizer(void (*func)(void*), void* data); + template + void push_finalizer(std::auto_ptr obj); + void clear(); <%0.upto(GENERATION_LIMIT) {|i|%> @@ -94,6 +98,15 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +template +inline void zone::push_finalizer(std::auto_ptr obj) +{ + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, obj.get())) { + throw std::bad_alloc(); + } + obj.release(); +} + inline void zone::clear() { msgpack_zone_clear(this); From 686e8ca0f004004f4b8e10438fe91a48a95e6ff9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 04:08:36 +0900 Subject: [PATCH 13/24] c,cpp: fix unpacker --- c/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index 6a435ba1..d5bcb2d9 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -152,7 +152,7 @@ static inline void init_count(void* buffer) static inline void decl_count(void* buffer) { // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } - if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer) == 0) { free(buffer); } } From dd18402737bdd12fbf53fe0543e29509f9609f3f Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:05:31 +0900 Subject: [PATCH 14/24] Fix stream unpacker broken. --- python/msgpack/_msgpack.pyx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cb951463..9ff0c571 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -233,7 +233,9 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=1024*1024): + def __init__(self, file_like=None, int read_size=0): + if read_size == 0: + read_size = 1024*1024 self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -309,6 +311,7 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + template_init(&self.ctx) return template_data(&self.ctx) elif ret == 0: if self.file_like is not None: @@ -319,3 +322,10 @@ cdef class Unpacker(object): def __iter__(self): return UnpackIterator(self) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head From 5ff2c6be74cb61363995d43772c5a52566b19000 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:14:13 +0900 Subject: [PATCH 15/24] Fix bug come from previous commit --- python/msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9ff0c571..dcabc0f9 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -311,8 +311,9 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + o = template_data(&self.ctx) template_init(&self.ctx) - return template_data(&self.ctx) + return o elif ret == 0: if self.file_like is not None: return self.unpack() From 1ed4236bcf1f96e997dc453b2f88a7c4e385688c Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:18:17 +0900 Subject: [PATCH 16/24] Make new Python release. --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index b26b9e8e..ee370983 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.1' +version = '0.1.2' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index e28ef25e..abed7cd1 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.1dev' +version = '0.1.2dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 3a5f7f53ff26c73396a211157d3d5c713bfc1b8a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:01 +0900 Subject: [PATCH 17/24] Start 0.2.0 --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index ee370983..c48be14e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.2' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index abed7cd1..4efc7696 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.2dev' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 4d33bd456cb31e0aeb3a88128e2cebe89d116ce1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:22 +0900 Subject: [PATCH 18/24] Update new headers. --- python/msgpack/pack.h | 1 + python/msgpack/pack_define.h | 3 +- python/msgpack/pack_template.h | 253 ++++++++++++------------------- python/msgpack/sysdep.h | 94 ++++++++++++ python/msgpack/unpack_define.h | 52 +------ python/msgpack/unpack_template.h | 41 +++-- 6 files changed, 225 insertions(+), 219 deletions(-) create mode 100644 python/msgpack/sysdep.h diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 58f021ef..2ae95d17 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,6 +18,7 @@ #include #include +#include "sysdep.h" #include "pack_define.h" #ifdef __cplusplus diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h index 33408e58..f72391b7 100644 --- a/python/msgpack/pack_define.h +++ b/python/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h index aa620f53..de148bf6 100644 --- a/python/msgpack/pack_template.h +++ b/python/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/python/msgpack/sysdep.h b/python/msgpack/sysdep.h new file mode 100644 index 00000000..106158ea --- /dev/null +++ b/python/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index d997569e..63d90a8e 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,52 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#ifdef __WIN32__ -static inline uint16_t msgpack_betoh16(uint16_t x) { - return ((x << 8) & 0xff00U) | - ((x >> 8) & 0x00ffU); -} -static inline uint32_t msgpack_betoh32(uint32_t x) { - return ((x << 24) & 0xff000000UL ) | - ((x << 8) & 0x00ff0000UL ) | - ((x >> 8) & 0x0000ff00UL ) | - ((x >> 24) & 0x000000ffUL ); -} -#else -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) -#endif - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index c960c3ab..ca6e1f32 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -131,20 +136,32 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) + +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -187,16 +204,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From f4c5b15cc6cf972695bb2281714f5b13b299fbd3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 11:13:47 +0900 Subject: [PATCH 19/24] Add `use_tuple` option that returns tuple for array object to Unpacker. --- python/msgpack/_msgpack.pyx | 10 +++++++++- python/msgpack/unpack.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index dcabc0f9..b20ad7b9 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -151,7 +151,11 @@ def packb(object o): packs = packb cdef extern from "unpack.h": + ctypedef struct msgpack_user: + int use_tuple + ctypedef struct template_context: + msgpack_user user PyObject* obj size_t count unsigned int ct @@ -170,6 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) + ctx.user.use_tuple = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -225,6 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes + cdef int use_tuple def __cinit__(self): self.buf = NULL @@ -233,9 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0): + def __init__(self, file_like=None, int read_size=0, use_tuple=0): if read_size == 0: read_size = 1024*1024 + self.use_tuple = use_tuple self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -244,6 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) + self.ctx.user.use_tuple = use_tuple def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 40058d08..0bf2568d 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,6 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { + int use_tuple; } unpack_user; @@ -135,7 +136,8 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyList_New(n); + PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + if (!p) return -1; *o = p; @@ -143,7 +145,15 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac } static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ PyList_SET_ITEM(*c, current, o); return 0; } +{ + if (u->use_tuple) { + PyTuple_SET_ITEM(*c, current, o); + } + else { + PyList_SET_ITEM(*c, current, o); + } + return 0; +} static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From b9f78821d423c2883ddff4d8471cc4c93d032388 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:19:18 +0900 Subject: [PATCH 20/24] Make tuple default. --- python/msgpack/_msgpack.pyx | 12 ++++++------ python/msgpack/unpack.h | 12 +++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b20ad7b9..eb83c854 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -152,7 +152,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_tuple + int use_list ctypedef struct template_context: msgpack_user user @@ -174,7 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) - ctx.user.use_tuple = 0 + ctx.user.use_list = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -230,7 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_tuple + cdef int use_list def __cinit__(self): self.buf = NULL @@ -239,10 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_tuple=0): + def __init__(self, file_like=None, int read_size=0, use_list=0): if read_size == 0: read_size = 1024*1024 - self.use_tuple = use_tuple + self.use_list = use_list self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -251,7 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) - self.ctx.user.use_tuple = use_tuple + self.ctx.user.use_list = use_list def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 0bf2568d..61a3786c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,7 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_tuple; + int use_list; } unpack_user; @@ -136,7 +136,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) return -1; @@ -146,12 +146,10 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { - if (u->use_tuple) { - PyTuple_SET_ITEM(*c, current, o); - } - else { + if (u->use_list) PyList_SET_ITEM(*c, current, o); - } + else + PyTuple_SET_ITEM(*c, current, o); return 0; } From 5cf85a82d3a39dd9135de473d0fa4ed21aea8270 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:20:20 +0900 Subject: [PATCH 21/24] Add .gitignore for Python. --- python/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 python/.gitignore diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 00000000..81df440b --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +MANIFEST +build/* +dist/* From 63b9a876b0b402709b7b8773b08a58f8fd0bc8b2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 17:58:41 +0900 Subject: [PATCH 22/24] Fix tests. --- python/test/test_case.py | 9 +++++---- python/test/test_format.py | 21 +++++++++++---------- python/test/test_pack.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/python/test/test_case.py b/python/test/test_case.py index d754fb0b..a08c6ce1 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -5,6 +5,7 @@ from nose import main from nose.tools import * from msgpack import packs, unpacks + def check(length, obj): v = packs(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) @@ -54,7 +55,7 @@ def test_raw32(): def check_array(overhead, num): - check(num + overhead, [None] * num) + check(num + overhead, (None,) * num) def test_fixarray(): check_array(1, 0) @@ -86,9 +87,9 @@ def test_match(): (-129, '\xd1\xff\x7f'), ({1:1}, '\x81\x01\x01'), (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ([], '\x90'), - (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ((), '\x90'), + (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), diff --git a/python/test/test_format.py b/python/test/test_format.py index 009a7646..562ef54a 100644 --- a/python/test/test_format.py +++ b/python/test/test_format.py @@ -10,21 +10,21 @@ def check(src, should): def testSimpleValue(): check("\x93\xc0\xc2\xc3", - [None, False, True]) + (None, False, True,)) def testFixnum(): check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - [[0,64,127], [-32,-16,-1]] + ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): check("\x92\x90\x91\x91\xc0", - [[],[[None]]], + ((),((None,),),), ) def testFixRaw(): check("\x94\xa0\xa1a\xa2bc\xa3def", - ["", "a", "bc", "def"], + ("", "a", "bc", "def",), ) def testFixMap(): @@ -38,25 +38,26 @@ def testUnsignedInt(): "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" "\xd2\xff\xff\xff\xff", - [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ["", "a", "ab", "", "a", "ab"]) + ("", "a", "ab", "", "a", "ab")) def testArray(): check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" "\xc2\xc3", - [[], [None], [False,True], [], [None], [False,True]]) + ((), (None,), (False,True), (), (None,), (False,True)) + ) def testMap(): check( @@ -67,8 +68,8 @@ def testMap(): "\xdf\x00\x00\x00\x00" "\xdf\x00\x00\x00\x01\xc0\xc2" "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - [{}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False}]) + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) if __name__ == '__main__': main() diff --git a/python/test/test_pack.py b/python/test/test_pack.py index 86badb5d..5dec0680 100644 --- a/python/test/test_pack.py +++ b/python/test/test_pack.py @@ -17,7 +17,7 @@ def testPack(): 1.0, "", "a", "a"*31, "a"*32, None, True, False, - [], [[]], [[], None], + (), ((),), ((), None,), {None: 0}, (1<<23), ] From 232aced926635a7054ac4081d472529cdf96f749 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 18:03:33 +0900 Subject: [PATCH 23/24] Update gitignore. --- python/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/.gitignore b/python/.gitignore index 81df440b..430c633e 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,5 @@ MANIFEST build/* dist/* +*.pyc +*.pyo From c2dd22ec102d12374b97f4ae32dccdaaca8630cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 19 Dec 2009 22:09:29 +0900 Subject: [PATCH 24/24] c,cpp: add msgpack_vrefbuffer_migrate, msgpack::vrefbuffer::migrate --- c/vrefbuffer.c | 29 ++++++++++++++++++++++++++++- c/vrefbuffer.h | 1 + cpp/vrefbuffer.hpp | 7 +++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index bbaf61d7..2bf97afe 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -77,7 +77,7 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { if(vbuf->tail == vbuf->end) { - const size_t nused = vbuf->end - vbuf->array; + const size_t nused = vbuf->tail - vbuf->array; const size_t nnext = nused * 2; struct iovec* nvec = (struct iovec*)realloc( @@ -133,3 +133,30 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, } } +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to) +{ + const size_t tosize = to->tail - to->array; + if(vbuf->tail + tosize < vbuf->end) { + const size_t nused = vbuf->tail - vbuf->array; + const size_t nsize = vbuf->end - vbuf->array; + const size_t reqsize = nused + tosize; + size_t nnext = nsize * 2; + while(nnext < reqsize) { + nnext *= 2; + } + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + memcpy(vbuf->tail, vbuf->array, sizeof(struct iovec)*tosize); + return 0; +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index 063075f8..9f24f14f 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -76,6 +76,7 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len); +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to); int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) { diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp index 549d77fc..c8eca7b6 100644 --- a/cpp/vrefbuffer.hpp +++ b/cpp/vrefbuffer.hpp @@ -71,6 +71,13 @@ public: return msgpack_vrefbuffer_veclen(this); } + void migrate(vrefbuffer* to) + { + if(msgpack_vrefbuffer_migrate(this, to) < 0) { + throw std::bad_alloc(); + } + } + private: typedef msgpack_vrefbuffer base;