From 82a5dd6cf9a49f6ffa444d5fc0ba5d4e3f10eb3e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Jun 2010 14:02:58 -0700 Subject: [PATCH 001/259] java: update --- java/src/main/java/org/msgpack/Packer.java | 4 + java/src/main/java/org/msgpack/Schema.java | 79 ++---------- java/src/main/java/org/msgpack/Unpacker.java | 4 +- .../org/msgpack/schema/BooleanSchema.java | 64 ++++++++++ .../{RawSchema.java => ByteArraySchema.java} | 58 ++++----- .../java/org/msgpack/schema/ByteSchema.java | 55 +++++---- .../org/msgpack/schema/ClassGenerator.java | 23 ++-- .../java/org/msgpack/schema/ClassSchema.java | 58 ++++----- .../java/org/msgpack/schema/DoubleSchema.java | 50 +++----- .../java/org/msgpack/schema/FloatSchema.java | 50 +++----- .../msgpack/schema/GenericClassSchema.java | 4 - .../org/msgpack/schema/GenericSchema.java | 73 +---------- .../java/org/msgpack/schema/IntSchema.java | 55 +++++---- .../{ArraySchema.java => ListSchema.java} | 80 +++++------- .../java/org/msgpack/schema/LongSchema.java | 39 +++--- .../java/org/msgpack/schema/MapSchema.java | 43 ++++--- .../org/msgpack/schema/SSchemaParser.java | 22 +++- .../java/org/msgpack/schema/SetSchema.java | 115 ++++++++++++++++++ .../java/org/msgpack/schema/ShortSchema.java | 52 ++++---- .../java/org/msgpack/schema/StringSchema.java | 65 +++++----- 20 files changed, 512 insertions(+), 481 deletions(-) create mode 100644 java/src/main/java/org/msgpack/schema/BooleanSchema.java rename java/src/main/java/org/msgpack/schema/{RawSchema.java => ByteArraySchema.java} (70%) rename java/src/main/java/org/msgpack/schema/{ArraySchema.java => ListSchema.java} (60%) create mode 100644 java/src/main/java/org/msgpack/schema/SetSchema.java diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index a92d2b68..dd510f34 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -223,6 +223,10 @@ public class Packer { return this; } + public Packer packBoolean(boolean d) throws IOException { + return d ? packTrue() : packFalse(); + } + public Packer packArray(int n) throws IOException { if(n < 16) { final int d = 0x90 | n; diff --git a/java/src/main/java/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java index f191f7a9..25e10f90 100644 --- a/java/src/main/java/org/msgpack/Schema.java +++ b/java/src/main/java/org/msgpack/Schema.java @@ -20,28 +20,13 @@ package org.msgpack; import java.io.Writer; import java.io.IOException; import org.msgpack.schema.SSchemaParser; -import org.msgpack.schema.ClassGenerator; +//import org.msgpack.schema.ClassGenerator; public abstract class Schema { - private String expression; - private String name; + public Schema() { } - public Schema(String name) { - this.expression = expression; - this.name = name; - } - - public String getName() { - return name; - } - - public String getFullName() { - return name; - } - - public String getExpression() { - return name; - } + public abstract String getClassName(); + public abstract String getExpression(); public static Schema parse(String source) { return SSchemaParser.parse(source); @@ -51,83 +36,43 @@ public abstract class Schema { return SSchemaParser.load(source); } - public void write(Writer output) throws IOException { - ClassGenerator.write(this, output); - } - public abstract void pack(Packer pk, Object obj) throws IOException; - public abstract Object convert(Object obj) throws MessageTypeException; - public Object createFromNil() { return null; } public Object createFromBoolean(boolean v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromByte(byte v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromShort(short v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromInt(int v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromLong(long v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromFloat(float v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromDouble(double v) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } public Object createFromRaw(byte[] b, int offset, int length) { - throw new RuntimeException("type error"); + throw new MessageTypeException("type error"); } - - /* FIXME - public Object createFromBoolean(boolean v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromByte(byte v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromShort(short v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromInt(int v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromLong(long v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromFloat(float v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromDouble(double v) { - throw MessageTypeException.schemaMismatch(this); - } - - public Object createFromRaw(byte[] b, int offset, int length) { - throw MessageTypeException.schemaMismatch(this); - } - */ } diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index f22c58b6..32bab646 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -561,11 +561,11 @@ public class Unpacker implements Iterable { return impl.unpackObject(); } - final void unpack(MessageUnpackable obj) throws IOException, MessageTypeException { + final public void unpack(MessageUnpackable obj) throws IOException, MessageTypeException { obj.messageUnpack(this); } - final boolean tryUnpackNull() throws IOException { + final public boolean tryUnpackNull() throws IOException { return impl.tryUnpackNull(); } } diff --git a/java/src/main/java/org/msgpack/schema/BooleanSchema.java b/java/src/main/java/org/msgpack/schema/BooleanSchema.java new file mode 100644 index 00000000..2c325f1f --- /dev/null +++ b/java/src/main/java/org/msgpack/schema/BooleanSchema.java @@ -0,0 +1,64 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class BooleanSchema extends Schema { + public BooleanSchema() { } + + @Override + public String getClassName() { + return "Boolean"; + } + + @Override + public String getExpression() { + return "boolean"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Boolean) { + pk.packBoolean((Boolean)obj); + } else if(obj == null) { + pk.packNil(); + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public static final boolean convertBoolean(Object obj) throws MessageTypeException { + if(obj instanceof Boolean) { + return (Boolean)obj; + } + throw new MessageTypeException(); + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return convertBoolean(obj); + } + + @Override + public Object createFromBoolean(boolean v) { + return v; + } +} + diff --git a/java/src/main/java/org/msgpack/schema/RawSchema.java b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java similarity index 70% rename from java/src/main/java/org/msgpack/schema/RawSchema.java rename to java/src/main/java/org/msgpack/schema/ByteArraySchema.java index f621e4c4..af9c0edf 100644 --- a/java/src/main/java/org/msgpack/schema/RawSchema.java +++ b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java @@ -22,59 +22,48 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import org.msgpack.*; -public class RawSchema extends Schema { - public RawSchema() { - super("raw"); - } +public class ByteArraySchema extends Schema { + public ByteArraySchema() { } - public String getFullName() { + @Override + public String getClassName() { return "byte[]"; } + @Override + public String getExpression() { + return "raw"; + } + @Override public void pack(Packer pk, Object obj) throws IOException { - // FIXME instanceof GenericObject if(obj instanceof byte[]) { - byte[] d = (byte[])obj; - pk.packRaw(d.length); - pk.packRawBody(d); - - } else if(obj instanceof ByteBuffer) { - ByteBuffer d = (ByteBuffer)obj; - if(!d.hasArray()) { - throw MessageTypeException.invalidConvert(obj, this); - } - pk.packRaw(d.capacity()); - pk.packRawBody(d.array(), d.position(), d.capacity()); - + byte[] b = (byte[])obj; + pk.packRaw(b.length); + pk.packRawBody(b); } else if(obj instanceof String) { try { - byte[] d = ((String)obj).getBytes("UTF-8"); - pk.packRaw(d.length); - pk.packRawBody(d); + byte[] b = ((String)obj).getBytes("UTF-8"); + pk.packRaw(b.length); + pk.packRawBody(b); } catch (UnsupportedEncodingException e) { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } - } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } - @Override - public Object convert(Object obj) throws MessageTypeException { - // FIXME instanceof GenericObject + public static final byte[] convertByteArray(Object obj) throws MessageTypeException { if(obj instanceof byte[]) { // FIXME copy? //byte[] d = (byte[])obj; //byte[] v = new byte[d.length]; //System.arraycopy(d, 0, v, 0, d.length); //return v; - return obj; - + return (byte[])obj; } else if(obj instanceof ByteBuffer) { ByteBuffer d = (ByteBuffer)obj; byte[] v = new byte[d.capacity()]; @@ -82,19 +71,22 @@ public class RawSchema extends Schema { d.get(v); d.position(pos); return v; - } else if(obj instanceof String) { try { return ((String)obj).getBytes("UTF-8"); } catch (UnsupportedEncodingException e) { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } - } else { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } } + @Override + public Object convert(Object obj) throws MessageTypeException { + return convertByteArray(obj); + } + @Override public Object createFromRaw(byte[] b, int offset, int length) { byte[] d = new byte[length]; diff --git a/java/src/main/java/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java index 9ee6a82b..6003a0f5 100644 --- a/java/src/main/java/org/msgpack/schema/ByteSchema.java +++ b/java/src/main/java/org/msgpack/schema/ByteSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class ByteSchema extends Schema { - public ByteSchema() { - super("Byte"); + public ByteSchema() { } + + @Override + public String getClassName() { + return "Byte"; } @Override @@ -33,27 +36,32 @@ public class ByteSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { if(obj instanceof Number) { - pk.packByte( ((Number)obj).byteValue() ); - + short value = ((Number)obj).shortValue(); + if(value > Byte.MAX_VALUE) { + throw new MessageTypeException(); + } + pk.packByte((byte)value); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final byte convertByte(Object obj) throws MessageTypeException { + if(obj instanceof Number) { + short value = ((Number)obj).shortValue(); + if(value > Byte.MAX_VALUE) { + throw new MessageTypeException(); + } + return (byte)value; + } + throw new MessageTypeException(); + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Byte) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).byteValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } + return convertByte(obj); } @Override @@ -63,26 +71,25 @@ public class ByteSchema extends Schema { @Override public Object createFromShort(short v) { + if(v > Byte.MAX_VALUE) { + throw new MessageTypeException(); + } return (byte)v; } @Override public Object createFromInt(int v) { + if(v > Byte.MAX_VALUE) { + throw new MessageTypeException(); + } return (byte)v; } @Override public Object createFromLong(long v) { - return (byte)v; - } - - @Override - public Object createFromFloat(float v) { - return (byte)v; - } - - @Override - public Object createFromDouble(double v) { + if(v > Byte.MAX_VALUE) { + throw new MessageTypeException(); + } return (byte)v; } } diff --git a/java/src/main/java/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java index f8a13fad..a515996f 100644 --- a/java/src/main/java/org/msgpack/schema/ClassGenerator.java +++ b/java/src/main/java/org/msgpack/schema/ClassGenerator.java @@ -77,8 +77,11 @@ public class ClassGenerator { for(FieldSchema f : cs.getFields()) { findSubclassSchema(dst, f.getSchema()); } - } else if(s instanceof ArraySchema) { - ArraySchema as = (ArraySchema)s; + } else if(s instanceof ListSchema) { + ListSchema as = (ListSchema)s; + findSubclassSchema(dst, as.getElementSchema(0)); + } else if(s instanceof SetSchema) { + SetSchema as = (SetSchema)s; findSubclassSchema(dst, as.getElementSchema(0)); } else if(s instanceof MapSchema) { MapSchema as = (MapSchema)s; @@ -105,7 +108,7 @@ public class ClassGenerator { private void writeClass() throws IOException { line(); - line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("public final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable"); line("{"); pushIndent(); writeSchema(); @@ -117,7 +120,7 @@ public class ClassGenerator { private void writeSubclass() throws IOException { line(); - line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable"); line("{"); pushIndent(); writeSchema(); @@ -135,7 +138,7 @@ public class ClassGenerator { private void writeMemberVariables() throws IOException { line(); for(FieldSchema f : schema.getFields()) { - line("public "+f.getSchema().getFullName()+" "+f.getName()+";"); + line("public "+f.getSchema().getClassName()+" "+f.getName()+";"); } } @@ -156,7 +159,7 @@ public class ClassGenerator { private void writeConstructors() throws IOException { line(); - line("public "+schema.getName()+"() { }"); + line("public "+schema.getClassName()+"() { }"); } private void writeAccessors() throws IOException { @@ -195,7 +198,7 @@ public class ClassGenerator { line("FieldSchema[] _fields = _SCHEMA.getFields();"); int i = 0; for(FieldSchema f : schema.getFields()) { - line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getFullName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); + line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getClassName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); ++i; } popIndent(); @@ -205,13 +208,13 @@ public class ClassGenerator { private void writeFactoryFunction() throws IOException { line(); line("@SuppressWarnings(\"unchecked\")"); - line("public static "+schema.getName()+" createFromMessage(Object[] _message)"); + line("public static "+schema.getClassName()+" createFromMessage(Object[] _message)"); line("{"); pushIndent(); - line(schema.getName()+" _self = new "+schema.getName()+"();"); + line(schema.getClassName()+" _self = new "+schema.getClassName()+"();"); int i = 0; for(FieldSchema f : schema.getFields()) { - line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getFullName()+")_message["+i+"];"); + line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getClassName()+")_message["+i+"];"); ++i; } line("return _self;"); diff --git a/java/src/main/java/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java index cd5c008e..cd59755a 100644 --- a/java/src/main/java/org/msgpack/schema/ClassSchema.java +++ b/java/src/main/java/org/msgpack/schema/ClassSchema.java @@ -22,6 +22,7 @@ import java.util.List; import org.msgpack.*; public abstract class ClassSchema extends Schema implements IArraySchema { + protected String name; protected FieldSchema[] fields; protected List imports; protected String namespace; @@ -30,7 +31,7 @@ public abstract class ClassSchema extends Schema implements IArraySchema { public ClassSchema( String name, String namespace, List imports, List fields) { - super(name); + this.name = name; this.namespace = namespace; this.imports = imports; // FIXME clone? this.fields = new FieldSchema[fields.size()]; @@ -42,6 +43,31 @@ public abstract class ClassSchema extends Schema implements IArraySchema { } } + @Override + public String getClassName() { + return name; + } + + @Override + public String getExpression() { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(name); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(FieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + public boolean equals(ClassSchema o) { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + name.equals(o.name); + } + public final FieldSchema[] getFields() { return fields; } @@ -61,35 +87,5 @@ public abstract class ClassSchema extends Schema implements IArraySchema { void setImports(List imports) { this.imports = imports; // FIXME clone? } - - //@Override - //public String getFullName() - //{ - // if(namespace == null) { - // return getName(); - // } else { - // return namespace+"."+getName(); - // } - //} - - @Override - public String getExpression() { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(getName()); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(FieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - public boolean equals(SpecificClassSchema o) { - return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && - getName().equals(o.getName()); - } } diff --git a/java/src/main/java/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java index d53e47d5..cb857c38 100644 --- a/java/src/main/java/org/msgpack/schema/DoubleSchema.java +++ b/java/src/main/java/org/msgpack/schema/DoubleSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class DoubleSchema extends Schema { - public DoubleSchema() { - super("Double"); + public DoubleSchema() { } + + @Override + public String getClassName() { + return "Double"; } @Override @@ -32,43 +35,30 @@ public class DoubleSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - pk.packDouble( ((Number)obj).doubleValue() ); - + if(obj instanceof Double) { + pk.packDouble((Double)obj); + } else if(obj instanceof Float) { + pk.packFloat((Float)obj); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final double convertDouble(Object obj) throws MessageTypeException { + if(obj instanceof Double) { + return (Double)obj; + } else if(obj instanceof Float) { + return ((Float)obj).doubleValue(); + } else { + throw new MessageTypeException(); + } + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Double) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).doubleValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @Override - public Object createFromByte(byte v) { - return (double)v; - } - - @Override - public Object createFromShort(short v) { - return (double)v; - } - - @Override - public Object createFromInt(int v) { - return (double)v; + return convertDouble(obj); } @Override diff --git a/java/src/main/java/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java index 27775218..cd732014 100644 --- a/java/src/main/java/org/msgpack/schema/FloatSchema.java +++ b/java/src/main/java/org/msgpack/schema/FloatSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class FloatSchema extends Schema { - public FloatSchema() { - super("Float"); + public FloatSchema() { } + + @Override + public String getClassName() { + return "Float"; } @Override @@ -32,43 +35,30 @@ public class FloatSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - pk.packFloat( ((Number)obj).floatValue() ); - + if(obj instanceof Double) { + pk.packDouble((Double)obj); + } else if(obj instanceof Float) { + pk.packFloat((Float)obj); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final float convertFloat(Object obj) throws MessageTypeException { + if(obj instanceof Double) { + return ((Double)obj).floatValue(); + } else if(obj instanceof Float) { + return (Float)obj; + } else { + throw new MessageTypeException(); + } + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Float) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).floatValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @Override - public Object createFromByte(byte v) { - return (float)v; - } - - @Override - public Object createFromShort(short v) { - return (float)v; - } - - @Override - public Object createFromInt(int v) { - return (float)v; + return convertFloat(obj); } @Override diff --git a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java index ffdd4aba..1ab4c334 100644 --- a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java +++ b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java @@ -41,10 +41,8 @@ public class GenericClassSchema extends ClassSchema { FieldSchema f = fields[i]; f.getSchema().pack(pk, d.get(f.getName())); } - } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } @@ -55,7 +53,6 @@ public class GenericClassSchema extends ClassSchema { if(obj instanceof Collection) { // FIXME optimize return createFromArray( ((Collection)obj).toArray() ); - } else if(obj instanceof Map) { HashMap m = new HashMap(fields.length); Map d = (Map)obj; @@ -65,7 +62,6 @@ public class GenericClassSchema extends ClassSchema { m.put(fieldName, f.getSchema().convert(d.get(fieldName))); } return m; - } else { throw MessageTypeException.invalidConvert(obj, this); } diff --git a/java/src/main/java/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java index 0adf8980..f9098edd 100644 --- a/java/src/main/java/org/msgpack/schema/GenericSchema.java +++ b/java/src/main/java/org/msgpack/schema/GenericSchema.java @@ -22,11 +22,13 @@ import java.util.List; import java.util.HashMap; import java.io.IOException; import org.msgpack.*; -//import org.msgpack.generic.*; public class GenericSchema extends Schema implements IArraySchema, IMapSchema { - public GenericSchema() { - super("Object"); + public GenericSchema() { } + + @Override + public String getClassName() { + return "Object"; } @Override @@ -123,70 +125,5 @@ public class GenericSchema extends Schema implements IArraySchema, IMapSchema { } return m; } - - /* - @Override - public Object createFromNil() { - return null; - } - - @Override - public Object createFromBoolean(boolean v) { - return new GenericBoolean(v); - } - - @Override - public Object createFromFromByte(byte v) { - return new GenericByte(v); - } - - @Override - public Object createFromShort(short v) { - return new GenericShort(v); - } - - @Override - public Object createFromInt(int v) { - return new GenericInt(v); - } - - @Override - public Object createFromLong(long v) { - return new GenericLong(v); - } - - @Override - public Object createFromFloat(float v) { - return new GenericFloat(v); - } - - @Override - public Object createFromDouble(double v) { - return new GenericDouble(v); - } - - @Override - public Object createFromRaw(byte[] b, int offset, int length) { - return new GenericRaw(b, offset, length); - } - - @Override - public Object createFromArray(Object[] obj) { - // FIXME GenericArray - return Arrays.asList(obj); - } - - @Override - public Object createFromMap(Object[] obj) { - GenericMap m = new GenericMap(obj.length / 2); - int i = 0; - while(i < obj.length) { - Object k = obj[i++]; - Object v = obj[i++]; - m.put(k, v); - } - return m; - } - */ } diff --git a/java/src/main/java/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java index 5a7e2815..269f4fb7 100644 --- a/java/src/main/java/org/msgpack/schema/IntSchema.java +++ b/java/src/main/java/org/msgpack/schema/IntSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class IntSchema extends Schema { - public IntSchema() { - super("Integer"); + public IntSchema() { } + + @Override + public String getClassName() { + return "Integer"; } @Override @@ -33,27 +36,38 @@ public class IntSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { if(obj instanceof Number) { - pk.packInt( ((Number)obj).intValue() ); - + int value = ((Number)obj).intValue(); + if(value >= Short.MAX_VALUE) { + long lvalue = ((Number)obj).longValue(); + if(lvalue > Integer.MAX_VALUE) { + throw new MessageTypeException(); + } + } + pk.packInt(value); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final int convertInt(Object obj) throws MessageTypeException { + if(obj instanceof Number) { + int value = ((Number)obj).intValue(); + if(value >= Integer.MAX_VALUE) { + long lvalue = ((Number)obj).longValue(); + if(lvalue > Integer.MAX_VALUE) { + throw new MessageTypeException(); + } + } + return value; + } + throw new MessageTypeException(); + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Integer) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).intValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } + return convertInt(obj); } @Override @@ -73,16 +87,9 @@ public class IntSchema extends Schema { @Override public Object createFromLong(long v) { - return (int)v; - } - - @Override - public Object createFromFloat(float v) { - return (int)v; - } - - @Override - public Object createFromDouble(double v) { + if(v > Integer.MAX_VALUE) { + throw new MessageTypeException(); + } return (int)v; } } diff --git a/java/src/main/java/org/msgpack/schema/ArraySchema.java b/java/src/main/java/org/msgpack/schema/ListSchema.java similarity index 60% rename from java/src/main/java/org/msgpack/schema/ArraySchema.java rename to java/src/main/java/org/msgpack/schema/ListSchema.java index fd471439..bef8cc41 100644 --- a/java/src/main/java/org/msgpack/schema/ArraySchema.java +++ b/java/src/main/java/org/msgpack/schema/ListSchema.java @@ -22,37 +22,32 @@ import java.util.Collection; import java.util.Set; import java.util.List; import java.util.ArrayList; +import java.util.HashSet; import java.util.RandomAccess; import java.io.IOException; import org.msgpack.*; -public class ArraySchema extends Schema implements IArraySchema { +public class ListSchema extends Schema implements IArraySchema { private Schema elementSchema; - public ArraySchema(Schema elementSchema) - { - super("array"); + public ListSchema(Schema elementSchema) { this.elementSchema = elementSchema; } @Override - public String getFullName() - { - return "List<"+elementSchema.getFullName()+">"; + public String getClassName() { + return "List<"+elementSchema.getClassName()+">"; } @Override - public String getExpression() - { + public String getExpression() { return "(array "+elementSchema.getExpression()+")"; } @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException - { + public void pack(Packer pk, Object obj) throws IOException { if(obj instanceof List) { - ArrayList d = (ArrayList)obj; + List d = (List)obj; pk.packArray(d.size()); if(obj instanceof RandomAccess) { for(int i=0; i < d.size(); ++i) { @@ -63,62 +58,53 @@ public class ArraySchema extends Schema implements IArraySchema { elementSchema.pack(pk, e); } } - } else if(obj instanceof Set) { Set d = (Set)obj; pk.packArray(d.size()); for(Object e : d) { elementSchema.pack(pk, e); } - } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } - @Override @SuppressWarnings("unchecked") - public Object convert(Object obj) throws MessageTypeException - { - if(obj instanceof List) { - List d = (List)obj; - ArrayList ar = new ArrayList(d.size()); - if(obj instanceof RandomAccess) { - for(int i=0; i < d.size(); ++i) { - ar.add( elementSchema.convert(d.get(i)) ); - } - } else { - for(Object e : d) { - ar.add( elementSchema.convert(e) ); - } - } - return ar; - - } else if(obj instanceof Collection) { - Collection d = (Collection)obj; - ArrayList ar = new ArrayList(d.size()); - for(Object e : (Collection)obj) { - ar.add( elementSchema.convert(e) ); - } - return ar; - - } else { - throw MessageTypeException.invalidConvert(obj, this); + public static final List convertList(Object obj, + Schema elementSchema, List dest) throws MessageTypeException { + if(!(obj instanceof List)) { + throw new MessageTypeException(); } + List d = (List)obj; + if(dest == null) { + dest = new ArrayList(d.size()); + } + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + dest.add( (T)elementSchema.convert(d.get(i)) ); + } + } else { + for(Object e : d) { + dest.add( (T)elementSchema.convert(e) ); + } + } + return dest; } @Override - public Schema getElementSchema(int index) - { + public Object convert(Object obj) throws MessageTypeException { + return convertList(obj, elementSchema, null); + } + + @Override + public Schema getElementSchema(int index) { return elementSchema; } @Override - public Object createFromArray(Object[] obj) - { + public Object createFromArray(Object[] obj) { return Arrays.asList(obj); } } diff --git a/java/src/main/java/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java index 83a30e37..728fa21e 100644 --- a/java/src/main/java/org/msgpack/schema/LongSchema.java +++ b/java/src/main/java/org/msgpack/schema/LongSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class LongSchema extends Schema { - public LongSchema() { - super("Long"); + public LongSchema() { } + + @Override + public String getClassName() { + return "Long"; } @Override @@ -33,27 +36,25 @@ public class LongSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { if(obj instanceof Number) { - pk.packLong( ((Number)obj).longValue() ); - + long value = ((Number)obj).longValue(); + pk.packLong(value); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final long convertLong(Object obj) throws MessageTypeException { + if(obj instanceof Number) { + return ((Number)obj).longValue(); + } + throw new MessageTypeException(); + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Long) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).longValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } + return convertLong(obj); } @Override @@ -75,15 +76,5 @@ public class LongSchema extends Schema { public Object createFromLong(long v) { return (long)v; } - - @Override - public Object createFromFloat(float v) { - return (long)v; - } - - @Override - public Object createFromDouble(double v) { - return (long)v; - } } diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java index ba759939..339a5c29 100644 --- a/java/src/main/java/org/msgpack/schema/MapSchema.java +++ b/java/src/main/java/org/msgpack/schema/MapSchema.java @@ -27,14 +27,13 @@ public class MapSchema extends Schema implements IMapSchema { private Schema valueSchema; public MapSchema(Schema keySchema, Schema valueSchema) { - super("map"); this.keySchema = keySchema; this.valueSchema = valueSchema; } @Override - public String getFullName() { - return "HashList<"+keySchema.getFullName()+", "+valueSchema.getFullName()+">"; + public String getClassName() { + return "Map<"+keySchema.getClassName()+", "+valueSchema.getClassName()+">"; } @Override @@ -52,29 +51,33 @@ public class MapSchema extends Schema implements IMapSchema { keySchema.pack(pk, e.getKey()); valueSchema.pack(pk, e.getValue()); } - } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } - @Override @SuppressWarnings("unchecked") - public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Map) { - Map d = (Map)obj; - Map m = new HashMap(); - for(Map.Entry e : d.entrySet()) { - m.put(keySchema.convert(e.getKey()), valueSchema.convert(e.getValue())); - } - return m; - - } else { - throw MessageTypeException.invalidConvert(obj, this); + public static final Map convertMap(Object obj, + Schema keySchema, Schema valueSchema, Map dest) throws MessageTypeException { + if(!(obj instanceof Map)) { + throw new MessageTypeException(); } + Map d = (Map)obj; + if(dest == null) { + dest = new HashMap(d.size()); + } + for(Map.Entry e : d.entrySet()) { + dest.put((K)keySchema.convert(e.getKey()), + (V)valueSchema.convert(e.getValue())); + } + return (Map)d; + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return convertMap(obj, keySchema, valueSchema, null); } @Override @@ -90,14 +93,14 @@ public class MapSchema extends Schema implements IMapSchema { @Override @SuppressWarnings("unchecked") public Object createFromMap(Object[] obj) { - HashMap m = new HashMap(obj.length / 2); + HashMap dest = new HashMap(obj.length / 2); int i = 0; while(i < obj.length) { Object k = obj[i++]; Object v = obj[i++]; - m.put(k, v); + dest.put(k, v); } - return m; + return dest; } } diff --git a/java/src/main/java/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java index 4ae8a4b5..4345e924 100644 --- a/java/src/main/java/org/msgpack/schema/SSchemaParser.java +++ b/java/src/main/java/org/msgpack/schema/SSchemaParser.java @@ -140,7 +140,7 @@ public class SSchemaParser { if(type.equals("string")) { return new StringSchema(); } else if(type.equals("raw")) { - return new RawSchema(); + return new ByteArraySchema(); } else if(type.equals("byte")) { return new ByteSchema(); } else if(type.equals("short")) { @@ -163,11 +163,13 @@ public class SSchemaParser { if(type.equals("class")) { return parseClass(exp); } else if(type.equals("array")) { - return parseArray(exp); + return parseList(exp); + } else if(type.equals("set")) { + return parseSet(exp); } else if(type.equals("map")) { return parseMap(exp); } else { - throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + throw new RuntimeException("class, list, set or map is expected but got '"+type+"': "+exp); } } } @@ -209,12 +211,20 @@ public class SSchemaParser { } } - private ArraySchema parseArray(SExp exp) { + private ListSchema parseList(SExp exp) { if(exp.size() != 2) { - throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp); } Schema elementType = readType(exp.getTuple(1)); - return new ArraySchema(elementType); + return new ListSchema(elementType); + } + + private SetSchema parseSet(SExp exp) { + if(exp.size() != 2) { + throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new SetSchema(elementType); } private MapSchema parseMap(SExp exp) { diff --git a/java/src/main/java/org/msgpack/schema/SetSchema.java b/java/src/main/java/org/msgpack/schema/SetSchema.java new file mode 100644 index 00000000..a3e19741 --- /dev/null +++ b/java/src/main/java/org/msgpack/schema/SetSchema.java @@ -0,0 +1,115 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.RandomAccess; +import java.io.IOException; +import org.msgpack.*; + +public class SetSchema extends Schema implements IArraySchema { + private Schema elementSchema; + + public SetSchema(Schema elementSchema) { + this.elementSchema = elementSchema; + } + + @Override + public String getClassName() { + return "Set<"+elementSchema.getClassName()+">"; + } + + @Override + public String getExpression() { + return "(set "+elementSchema.getExpression()+")"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof List) { + List d = (List)obj; + pk.packArray(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + elementSchema.pack(pk, d.get(i)); + } + } else { + for(Object e : d) { + elementSchema.pack(pk, e); + } + } + } else if(obj instanceof Set) { + Set d = (Set)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementSchema.pack(pk, e); + } + } else if(obj == null) { + pk.packNil(); + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @SuppressWarnings("unchecked") + public static final Set convertSet(Object obj, + Schema elementSchema, Set dest) throws MessageTypeException { + if(!(obj instanceof List)) { + throw new MessageTypeException(); + } + List d = (List)obj; + if(dest == null) { + dest = new HashSet(d.size()); + } + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + dest.add( (T)elementSchema.convert(d.get(i)) ); + } + } else { + for(Object e : d) { + dest.add( (T)elementSchema.convert(e) ); + } + } + return dest; + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return convertSet(obj, elementSchema, null); + } + + @Override + public Schema getElementSchema(int index) { + return elementSchema; + } + + @Override + public Object createFromArray(Object[] obj) { + Set m = new HashSet(obj.length); + for(int i=0; i < obj.length; i++) { + m.add(obj[i]); + } + return m; + } +} + diff --git a/java/src/main/java/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java index f32ab411..21b93279 100644 --- a/java/src/main/java/org/msgpack/schema/ShortSchema.java +++ b/java/src/main/java/org/msgpack/schema/ShortSchema.java @@ -21,8 +21,11 @@ import java.io.IOException; import org.msgpack.*; public class ShortSchema extends Schema { - public ShortSchema() { - super("Short"); + public ShortSchema() { } + + @Override + public String getClassName() { + return "Short"; } @Override @@ -33,27 +36,32 @@ public class ShortSchema extends Schema { @Override public void pack(Packer pk, Object obj) throws IOException { if(obj instanceof Number) { - pk.packShort( ((Number)obj).shortValue() ); - + int value = ((Number)obj).intValue(); + if(value > Short.MAX_VALUE) { + throw new MessageTypeException(); + } + pk.packShort((short)value); } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } + public static final short convertShort(Object obj) throws MessageTypeException { + if(obj instanceof Number) { + int value = ((Number)obj).intValue(); + if(value > Short.MAX_VALUE) { + throw new MessageTypeException(); + } + return (short)value; + } + throw new MessageTypeException(); + } + @Override public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Short) { - return obj; - - } else if(obj instanceof Number) { - return ((Number)obj).shortValue(); - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } + return convertShort(obj); } @Override @@ -68,21 +76,17 @@ public class ShortSchema extends Schema { @Override public Object createFromInt(int v) { + if(v > Short.MAX_VALUE) { + throw new MessageTypeException(); + } return (short)v; } @Override public Object createFromLong(long v) { - return (short)v; - } - - @Override - public Object createFromFloat(float v) { - return (short)v; - } - - @Override - public Object createFromDouble(double v) { + if(v > Short.MAX_VALUE) { + throw new MessageTypeException(); + } return (short)v; } } diff --git a/java/src/main/java/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java index 46d515b4..23e4e64e 100644 --- a/java/src/main/java/org/msgpack/schema/StringSchema.java +++ b/java/src/main/java/org/msgpack/schema/StringSchema.java @@ -23,61 +23,48 @@ import java.io.UnsupportedEncodingException; import org.msgpack.*; public class StringSchema extends Schema { - public StringSchema() { - super("string"); - } + public StringSchema() { } @Override - public String getFullName() { + public String getClassName() { return "String"; } + @Override + public String getExpression() { + return "string"; + } + @Override public void pack(Packer pk, Object obj) throws IOException { - // FIXME instanceof GenericObject - if(obj instanceof String) { + if(obj instanceof byte[]) { + byte[] b = (byte[])obj; + pk.packRaw(b.length); + pk.packRawBody(b); + } else if(obj instanceof String) { try { - byte[] d = ((String)obj).getBytes("UTF-8"); - pk.packRaw(d.length); - pk.packRawBody(d); + byte[] b = ((String)obj).getBytes("UTF-8"); + pk.packRaw(b.length); + pk.packRawBody(b); } catch (UnsupportedEncodingException e) { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } - - } else if(obj instanceof byte[]) { - byte[] d = (byte[])obj; - pk.packRaw(d.length); - pk.packRawBody(d); - - } else if(obj instanceof ByteBuffer) { - ByteBuffer d = (ByteBuffer)obj; - if(!d.hasArray()) { - throw MessageTypeException.invalidConvert(obj, this); - } - pk.packRaw(d.capacity()); - pk.packRawBody(d.array(), d.position(), d.capacity()); - } else if(obj == null) { pk.packNil(); - } else { throw MessageTypeException.invalidConvert(obj, this); } } - @Override - public Object convert(Object obj) throws MessageTypeException { - // FIXME instanceof GenericObject - if(obj instanceof String) { - return obj; - - } else if(obj instanceof byte[]) { + public static final String convertString(Object obj) throws MessageTypeException { + if(obj instanceof byte[]) { try { return new String((byte[])obj, "UTF-8"); } catch (UnsupportedEncodingException e) { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } - + } else if(obj instanceof String) { + return (String)obj; } else if(obj instanceof ByteBuffer) { ByteBuffer d = (ByteBuffer)obj; try { @@ -91,14 +78,18 @@ public class StringSchema extends Schema { return new String(v, "UTF-8"); } } catch (UnsupportedEncodingException e) { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } - } else { - throw MessageTypeException.invalidConvert(obj, this); + throw new MessageTypeException(); } } + @Override + public Object convert(Object obj) throws MessageTypeException { + return convertString(obj); + } + @Override public Object createFromRaw(byte[] b, int offset, int length) { try { From 59603b902adf2abe0d274f41520569fde387a841 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 15 Jun 2010 17:51:24 +0900 Subject: [PATCH 002/259] Python: add "load(s)/dump(s)" alias for compatibility to simplejson/marshal/pickle. --- python/msgpack/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py index 797b29c2..26bd2dd4 100644 --- a/python/msgpack/__init__.py +++ b/python/msgpack/__init__.py @@ -1,3 +1,10 @@ # coding: utf-8 from _msgpack import * +# alias for compatibility to simplejson/marshal/pickle. +load = unpack +loads = unpackb + +dump = pack +dumps = packb + From f222f5ed9b49e9b9f4d31693969c4260ad97cef0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 15 Jun 2010 18:06:58 +0900 Subject: [PATCH 003/259] Python: 0.1.4 --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 8d8a6f45..64e71ede 100755 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.1.3' +version = '0.1.4' # take care of extension modules. if have_cython: From fd8069342052142449d4703f8a12014cf283fedf Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 22 Jun 2010 11:15:18 +0900 Subject: [PATCH 004/259] erlang: tests improved and code refined. --- erlang/OMakefile | 2 +- erlang/msgpack.erl | 116 ++++++++++++++++++----------------- erlang/testcase_generator.rb | 10 +-- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/erlang/OMakefile b/erlang/OMakefile index ee72f784..34c590f0 100644 --- a/erlang/OMakefile +++ b/erlang/OMakefile @@ -36,7 +36,7 @@ msgpack.beam: msgpack.erl erlc $< test: msgpack.beam - erl -s msgpack test -s init stop + erl -noshell -s msgpack test -s init stop clean: -rm *.beam diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index ca9769e6..5a468a10 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -31,10 +31,52 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). - - -type reason() :: enomem. +% ===== external APIs ===== % +pack(O) when is_integer(O) andalso O < 0 -> + pack_int_(O); +pack(O) when is_integer(O) -> + pack_uint_(O); +pack(O) when is_float(O)-> + pack_double(O); +pack(nil) -> + pack_nil(); +pack(Bool) when is_atom(Bool) -> + pack_bool(Bool); +pack(Bin) when is_binary(Bin)-> + pack_raw(Bin); +pack(List) when is_list(List)-> + pack_array(List); +pack({dict, Map})-> + pack_map({dict, Map}); +pack(_) -> + undefined. + +% unpacking. +% if failed in decoding and not end, get more data +% and feed more Bin into this function. +% TODO: error case for imcomplete format when short for any type formats. +-spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +unpack(Bin) when not is_binary(Bin)-> + {error, badard}; +unpack(Bin) when bit_size(Bin) >= 8 -> + << Flag:8/unsigned-integer, Payload/binary >> = Bin, + unpack_(Flag, Payload); +unpack(_)-> % when bit_size(Bin) < 8 -> + {more, 8}. + +unpack_all(Data)-> + case unpack(Data) of + { Term, Binary } when bit_size(Binary) =:= 0 -> + [Term]; + { Term, Binary } when is_binary(Binary) -> + [Term|unpack_all(Binary)] + end. + + +% ===== internal APIs ===== % + % positive fixnum pack_uint_(N) when is_integer( N ) , N < 128 -> << 2#0:1, N:7 >>; @@ -113,7 +155,7 @@ pack_array(L) when is_list(L)-> end. pack_array_([])-> <<>>; pack_array_([Head|Tail])-> - << (pack_object(Head))/binary, (pack_array_(Tail))/binary >>. + << (pack(Head))/binary, (pack_array_(Tail))/binary >>. unpack_array_(<<>>, 0)-> []; unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain]; @@ -134,7 +176,7 @@ pack_map({dict,M})-> pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> - << (pack_object(Key)),(pack_object(Value)),(pack_map_(Tail)) >>. + << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. unpack_map_(<<>>, 0)-> []; unpack_map_(Bin, 0) when is_binary(Bin)-> [Bin]; @@ -143,38 +185,7 @@ unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) -> { Value, Rest2 } = unpack(Rest), [{Key,Value}|unpack_map_(Rest2,Len-1)]. -pack_object(O) when is_integer(O) andalso O < 0 -> - pack_int_(O); -pack_object(O) when is_integer(O) -> - pack_uint_(O); -pack_object(O) when is_float(O)-> - pack_double(O); -pack_object(nil) -> - pack_nil(); -pack_object(Bool) when is_atom(Bool) -> - pack_bool(Bool); -pack_object(Bin) when is_binary(Bin)-> - pack_raw(Bin); -pack_object(List) when is_list(List)-> - pack_array(List); -pack_object({dict, Map})-> - pack_map({dict, Map}); -pack_object(_) -> - undefined. - -pack(Obj)-> - pack_object(Obj). - - -% unpacking. -% if failed in decoding and not end, get more data -% and feed more Bin into this function. -% TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. -unpack(Bin) when not is_binary(Bin)-> - {error, badard}; -unpack(Bin) when bit_size(Bin) >= 8 -> - << Flag:8/unsigned-integer, Payload/binary >> = Bin, +unpack_(Flag, Payload)-> case Flag of 16#C0 -> {nil, Payload}; @@ -305,20 +316,17 @@ unpack(Bin) when bit_size(Bin) >= 8 -> _Other -> erlang:display(_Other), {error, no_code_matches} - end; -unpack(_)-> % when bit_size(Bin) < 8 -> - {more, 8}. - -unpack_all(Data)-> - case unpack(Data) of - { Term, Binary } when bit_size(Binary) =:= 0 -> - [Term]; - { Term, Binary } when is_binary(Binary) -> - [Term|unpack_all(Binary)] end. -ifdef(EUNIT). +compare_all([], [])-> ok; +compare_all([], R)-> {toomuchrhs, R}; +compare_all(L, [])-> {toomuchlhs, L}; +compare_all([LH|LTL], [RH|RTL]) -> + LH=RH, + compare_all(LTL, RTL). + test_data()-> [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, @@ -339,7 +347,6 @@ port_test()-> {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])), Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), true = port_command(Port, msgpack:pack(Tests) ), - %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner receive {Port, {data, Data}}-> {Tests, <<>>}=msgpack:unpack(Data) after 1024-> ?assert(false) end, @@ -348,24 +355,23 @@ port_test()-> unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, -% "hogehoge", "243546rf7g68h798j", - 123.123 %-234.4355, 1.0e-34, 1.0e64, -% [23, 234, 0.23] -% [0,42,"sum", [1,2]], [1,42, nil, [3]] + <<"hogehoge">>, <<"243546rf7g68h798j">>, +% 123.123, %FIXME +% -234.4355, 1.0e-34, 1.0e64, % FIXME + [23, 234, 0.23], + [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], + 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), - %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner receive {Port, {data, Data}}-> - Tests=msgpack:unpack_all(Data) -% io:format("~p~n", [Tests]) + compare_all(Tests, msgpack:unpack_all(Data)) after 1024-> ?assert(false) end, port_close(Port). test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), -% io:format("testing: ~p => ~p~n", [S, Pack]), {S, <<>>} = msgpack:unpack( Pack ), 1+test_(Rest). diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index a173790a..a7c76c5f 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -39,10 +39,12 @@ end objs = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, -# "hogehoge", "243546rf7g68h798j", - 123.123, #-234.4355, 1.0e-34, 1.0e64, -# [23, 234, 0.23] -# [0,42,"sum", [1,2]], [1,42, nil, [3]] + "hogehoge", "243546rf7g68h798j", +# 123.123 , #FIXME +# -234.4355, 1.0e-34, 1.0e64, + [23, 234, 0.23], + [0,42,"sum", [1,2]], [1,42, nil, [3]], + 42 ] begin objs.each do |obj| From b1e66256ce35ea4a953d255a2a98e9c8ddbe6401 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 22 Jun 2010 11:28:36 +0900 Subject: [PATCH 005/259] erlang: external APIs' type/specs. --- erlang/msgpack.erl | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 5a468a10..f2380fcd 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -18,37 +18,38 @@ -module(msgpack). -author('kuenishi+msgpack@gmail.com'). -%% tuples, atoms are not supported. lists, integers, double, and so on. +%% tuples, atoms are not supported. lists, integers, double, and so on. %% see http://msgpack.sourceforge.jp/spec for %% supported formats. APIs are almost compatible %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). --export([pack/1, unpack/1, unpack_all/1, test/0]). - --include_lib("eunit/include/eunit.hrl"). +-export([pack/1, unpack/1, unpack_all/1]). % compile: % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem. +-type reason() :: enomem | badarg. +-type map() :: any(). % there's no 'dict' type... +-type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. % ===== external APIs ===== % +-spec pack(Term::msgpack_term()) -> binary(). pack(O) when is_integer(O) andalso O < 0 -> pack_int_(O); pack(O) when is_integer(O) -> pack_uint_(O); -pack(O) when is_float(O)-> +pack(O) when is_float(O) -> pack_double(O); pack(nil) -> pack_nil(); pack(Bool) when is_atom(Bool) -> pack_bool(Bool); -pack(Bin) when is_binary(Bin)-> +pack(Bin) when is_binary(Bin) -> pack_raw(Bin); -pack(List) when is_list(List)-> +pack(List) when is_list(List) -> pack_array(List); -pack({dict, Map})-> +pack({dict, Map}) -> pack_map({dict, Map}); pack(_) -> undefined. @@ -57,15 +58,16 @@ pack(_) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +-spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> - {error, badard}; + {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> << Flag:8/unsigned-integer, Payload/binary >> = Bin, unpack_(Flag, Payload); unpack(_)-> % when bit_size(Bin) < 8 -> {more, 8}. +-spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> case unpack(Data) of { Term, Binary } when bit_size(Binary) =:= 0 -> @@ -318,6 +320,8 @@ unpack_(Flag, Payload)-> {error, no_code_matches} end. +% ===== test codes ===== % +-include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). compare_all([], [])-> ok; From 230ee3a03b5148d11c614fc18c7c7603c9a8af14 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 23 Jun 2010 01:26:10 +0900 Subject: [PATCH 006/259] erlang: too short binary to decode causes error {more, Int}. --- erlang/msgpack.erl | 211 +++++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 86 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index f2380fcd..d2f7068c 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -29,7 +29,7 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem | badarg. +-type reason() :: enomem | badarg | no_code_matches. -type map() :: any(). % there's no 'dict' type... -type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. @@ -58,14 +58,15 @@ pack(_) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +-spec unpack( binary() )-> + {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> << Flag:8/unsigned-integer, Payload/binary >> = Bin, unpack_(Flag, Payload); -unpack(_)-> % when bit_size(Bin) < 8 -> - {more, 8}. +unpack(<<>>)-> % when bit_size(Bin) < 8 -> + {more, 1}. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -129,12 +130,9 @@ pack_bool(false)-> << 16#C2:8 >>. pack_double(F) when is_float(F)-> << 16#CB:8, F:64/big-float-unit:1 >>. -power(N,0) when is_integer(N) -> 1; -power(N,D) when is_integer(N) and is_integer(D) -> N * power(N, D-1). - % raw bytes pack_raw(Bin) when is_binary(Bin)-> - MaxLen = power(2,16), + MaxLen = 16#10000, % 65536 case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; @@ -146,7 +144,7 @@ pack_raw(Bin) when is_binary(Bin)-> % list / tuple pack_array(L) when is_list(L)-> - MaxLen = power(2,16), + MaxLen = 16#10000, %65536 case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; @@ -159,14 +157,19 @@ pack_array_([])-> <<>>; pack_array_([Head|Tail])-> << (pack(Head))/binary, (pack_array_(Tail))/binary >>. -unpack_array_(<<>>, 0)-> []; -unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain]; -unpack_array_(Bin, RestLen) when is_binary(Bin)-> - {Term, Rest} = unpack(Bin), - [Term|unpack_array_(Rest, RestLen-1)]. +% FIXME! this should be tail-recursive and without lists:reverse/1 +unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; +unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; +unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; +unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> + case unpack(Bin) of + {more, Len} -> {more, Len+RestLen-1}; + {Term, Rest}-> + unpack_array_(Rest, RestLen-1, [Term|RetList]) + end. pack_map({dict,M})-> - MaxLen = power(2,16), + MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; @@ -180,14 +183,25 @@ pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. -unpack_map_(<<>>, 0)-> []; -unpack_map_(Bin, 0) when is_binary(Bin)-> [Bin]; -unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) -> - { Key, Rest } = unpack(Bin), - { Value, Rest2 } = unpack(Rest), - [{Key,Value}|unpack_map_(Rest2,Len-1)]. +-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> + {more, non_neg_integer()} | { any(), binary()}. +unpack_map_(Bin, 0, TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin}; +unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) -> + case unpack(Bin) of + { more, MoreLen } -> { more, MoreLen+Len-1 }; + { Key, Rest } -> + case unpack(Rest) of + {more, MoreLen} -> { more, MoreLen+Len-1 }; + { Value, Rest2 }-> + unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap]) + end + end. +% {more, +-spec unpack_(Flag::integer(), Payload::binary())-> + {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. unpack_(Flag, Payload)-> + PayloadLen = byte_size(Payload), case Flag of 16#C0 -> {nil, Payload}; @@ -196,86 +210,101 @@ unpack_(Flag, Payload)-> 16#C3 -> {true, Payload}; - 16#CA -> % 32bit float + 16#CA when PayloadLen >= 4 -> % 32bit float << Return:32/float-unit:1, Rest/binary >> = Payload, {Return, Rest}; - 16#CB -> % 64bit float + 16#CA -> + {more, 4-PayloadLen}; % at least more + + 16#CB when PayloadLen >= 8 -> % 64bit float << Return:64/float-unit:1, Rest/binary >> = Payload, {Return, Rest}; + 16#CB -> + {more, 8-PayloadLen}; - 16#CC -> % uint 8 + 16#CC when PayloadLen >= 1 -> % uint 8 << Int:8/unsigned-integer, Rest/binary >> = Payload, {Int, Rest}; - 16#CD -> % uint 16 + 16#CC -> + {more, 1}; + + 16#CD when PayloadLen >= 2 -> % uint 16 << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#CE -> + 16#CD -> + {more, 2-PayloadLen}; + + 16#CE when PayloadLen >= 4 -> << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#CF -> + 16#CE -> + {more, 4-PayloadLen}; % at least more + + 16#CF when PayloadLen >= 8 -> << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; + 16#CF -> + {more, 8-PayloadLen}; - 16#D0 -> % int 8 + 16#D0 when PayloadLen >= 1 -> % int 8 << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D1 -> % int 16 + 16#D0 -> + {more, 1}; + + 16#D1 when PayloadLen >= 2 -> % int 16 << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D2 -> % int 32 + 16#D1 -> + {more, 2-PayloadLen}; + + 16#D2 when PayloadLen >= 4 -> % int 32 << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D3 -> % int 64 + 16#D2 -> + {more, 4-PayloadLen}; + + 16#D3 when PayloadLen >= 8 -> % int 64 << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#DA -> % raw 16 + 16#D3 -> + {more, 8-PayloadLen}; + + 16#DA when PayloadLen >= 2 -> % raw 16 << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload, << Return:Len/binary, Remain/binary >> = Rest, {Return, Remain}; - 16#DB -> % raw 32 + 16#DA -> + {more, 16-PayloadLen}; + + 16#DB when PayloadLen >= 4 -> % raw 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, << Return:Len/binary, Remain/binary >> = Rest, {Return, Remain}; - 16#DC -> % array 16 + 16#DB -> + {more, 4-PayloadLen}; + + 16#DC when PayloadLen >= 2 -> % array 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_array_(Rest, Len), - case length(Array) of - Len -> {Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; - 16#DD -> % array 32 + unpack_array_(Rest, Len, []); + 16#DC -> + {more, 2-PayloadLen}; + + 16#DD when PayloadLen >= 4 -> % array 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_array_(Rest, Len), - case length(Array) of - Len -> {Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; - 16#DE -> % map 16 + unpack_array_(Rest, Len, []); + 16#DD -> + {more, 4-PayloadLen}; + + 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_map_(Rest, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; - 16#DF -> % map 32 + unpack_map_(Rest, Len, []); + 16#DE -> + {more, 2-PayloadLen}; + + 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_map_(Rest, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; + unpack_map_(Rest, Len, []); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -294,29 +323,15 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10010000 , Code < 2#10100000 -> % 1001XXXX for FixArray Len = Code rem 2#10010000, - Array=unpack_array_(Payload, Len), - case length(Array) of - Len -> { Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; + unpack_array_(Payload, Len, []); Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - Array=unpack_map_(Payload, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; + unpack_map_(Payload, Len, []); _Other -> - erlang:display(_Other), +% erlang:display(_Other), {error, no_code_matches} end. @@ -356,6 +371,26 @@ port_test()-> after 1024-> ?assert(false) end, port_close(Port). +test_p(Len,Term,OrigBin,Len) -> + {Term, <<>>}=msgpack:unpack(OrigBin); +test_p(I,_,OrigBin,Len) -> + <> = OrigBin, + {more, N}=msgpack:unpack(Bin), + ?assert(0 < N), + ?assert(N < Len). + +partial_test()-> % error handling test. + Term = lists:seq(0, 45), + Bin=msgpack:pack(Term), + BinLen = byte_size(Bin), + [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. + +long_test()-> + Longer = lists:seq(0, 655), %55), +%% Longest = lists:seq(0,12345), + {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)). +%% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, @@ -377,6 +412,10 @@ test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), {S, <<>>} = msgpack:unpack( Pack ), +% ?debugVal( hoge ), 1+test_(Rest). +other_test()-> + {more,1}=msgpack:unpack(<<>>). + -endif. From bc0c5f0cdca4d5244b3237b63305cfd01d1e9a9d Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 23 Jun 2010 09:02:53 +0900 Subject: [PATCH 007/259] erlang: (un)pack_map improved, incremental unpacking --- erlang/msgpack.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d2f7068c..764de114 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -167,7 +167,8 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. - + +% FIXME: write test for pack_map/1 pack_map({dict,M})-> MaxLen = 16#10000, %65536 case dict:size(M) of @@ -183,17 +184,18 @@ pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. +% FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin}; -unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) -> +unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> { {dict, Dict}, Bin}; +unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap]) + unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict)) end end. @@ -298,13 +300,13 @@ unpack_(Flag, Payload)-> 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); + unpack_map_(Rest, Len, dict:new()); 16#DE -> {more, 2-PayloadLen}; 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); + unpack_map_(Rest, Len, dict:new()); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -328,7 +330,7 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - unpack_map_(Payload, Len, []); + unpack_map_(Payload, Len, dict:new()); _Other -> % erlang:display(_Other), @@ -373,7 +375,7 @@ port_test()-> test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); -test_p(I,_,OrigBin,Len) -> +test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, {more, N}=msgpack:unpack(Bin), ?assert(0 < N), From 2cdfbd8970756742af7b378612756c59d184d86b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 24 Jun 2010 07:26:34 +0900 Subject: [PATCH 008/259] erlang: testing pack_map/unpack_map with a silly bug --- erlang/msgpack.erl | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 764de114..dca5888a 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -49,10 +49,10 @@ pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> pack_array(List); -pack({dict, Map}) -> - pack_map({dict, Map}); -pack(_) -> - undefined. +pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> + pack_map(Map); +pack(_O) -> + {error, undefined}. % unpacking. % if failed in decoding and not end, get more data @@ -169,25 +169,25 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> end. % FIXME: write test for pack_map/1 -pack_map({dict,M})-> +pack_map(M)-> MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; Len when Len < MaxLen -> - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> end. pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> - << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. + << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>. % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> { {dict, Dict}, Bin}; +unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> {Dict, Bin}; unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; @@ -333,7 +333,6 @@ unpack_(Flag, Payload)-> unpack_map_(Payload, Len, dict:new()); _Other -> -% erlang:display(_Other), {error, no_code_matches} end. @@ -388,10 +387,25 @@ partial_test()-> % error handling test. [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. long_test()-> - Longer = lists:seq(0, 655), %55), -%% Longest = lists:seq(0,12345), - {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)). + Longer = lists:seq(0, 65), %55), +% Longest = lists:seq(0,12345), + {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)), +% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ok. + +map_test()-> + Ints = lists:seq(0, 65), %55), + Map = dict:from_list([ {X, X*2} || X <- Ints ]), + S=msgpack:pack(Map), +% ?debugVal(msgpack:unpack(S)), + {Map2, <<>>} = msgpack:unpack(S), + ?assertEqual(dict:size(Map), dict:size(Map2)), +% ?debugVal(dict:to_list(Map2)), + OrdMap = orddict:from_list( dict:to_list(Map) ), + OrdMap2 = orddict:from_list( dict:to_list(Map2) ), +% ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug. %% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ok. unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, From 92d192277e14be4f32135a85cbca233e7dca3182 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 00:22:53 +0900 Subject: [PATCH 009/259] erlang: unpack_map's silly bug fixed. use dict:store/3.... --- erlang/msgpack.erl | 22 +++++++++------------- erlang/testcase_generator.rb | 5 +++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index dca5888a..0d4151b5 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -195,7 +195,7 @@ unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict)) + unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict)) end end. @@ -352,7 +352,7 @@ test_data()-> -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], - "hogehoge", "243546rf7g68h798j", + <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, [0,42,"sum", [1,2]], [1,42, nil, [3]] ]. @@ -387,34 +387,31 @@ partial_test()-> % error handling test. [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. long_test()-> - Longer = lists:seq(0, 65), %55), + Longer = lists:seq(0, 655), % Longest = lists:seq(0,12345), {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)), % {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). ok. map_test()-> - Ints = lists:seq(0, 65), %55), + Ints = lists:seq(0, 65), Map = dict:from_list([ {X, X*2} || X <- Ints ]), - S=msgpack:pack(Map), -% ?debugVal(msgpack:unpack(S)), - {Map2, <<>>} = msgpack:unpack(S), + {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), ?assertEqual(dict:size(Map), dict:size(Map2)), -% ?debugVal(dict:to_list(Map2)), OrdMap = orddict:from_list( dict:to_list(Map) ), OrdMap2 = orddict:from_list( dict:to_list(Map2) ), -% ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug. -%% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ?assertEqual(OrdMap, OrdMap2), ok. unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, <<"hogehoge">>, <<"243546rf7g68h798j">>, -% 123.123, %FIXME -% -234.4355, 1.0e-34, 1.0e64, % FIXME + 123.123, + -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], + dict:from_list([{1,2},{<<"hoge">>,nil}]), 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), @@ -428,7 +425,6 @@ test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), {S, <<>>} = msgpack:unpack( Pack ), -% ?debugVal( hoge ), 1+test_(Rest). other_test()-> diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index a7c76c5f..cfc36f96 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -40,10 +40,11 @@ end objs = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, "hogehoge", "243546rf7g68h798j", -# 123.123 , #FIXME -# -234.4355, 1.0e-34, 1.0e64, + 123.123, + -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,"sum", [1,2]], [1,42, nil, [3]], + { 1 => 2, "hoge" => nil }, 42 ] begin From 57f0598373ce542f7499218792a15fa0eee161f7 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 00:44:14 +0900 Subject: [PATCH 010/259] erlang: code refined and tests added --- erlang/msgpack.erl | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 0d4151b5..789ed535 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -116,8 +116,7 @@ pack_int_( N ) when is_integer( N )-> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. % nil -pack_nil()-> - << 16#C0:8 >>. +pack_nil()-> << 16#C0:8 >>. % pack_true / pack_false pack_bool(true)-> << 16#C3:8 >>; pack_bool(false)-> << 16#C2:8 >>. @@ -132,11 +131,10 @@ pack_double(F) when is_float(F)-> % raw bytes pack_raw(Bin) when is_binary(Bin)-> - MaxLen = 16#10000, % 65536 case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DA:8, Len:16/big-unsigned-integer-unit:1, Bin/binary >>; Len -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> @@ -144,11 +142,10 @@ pack_raw(Bin) when is_binary(Bin)-> % list / tuple pack_array(L) when is_list(L)-> - MaxLen = 16#10000, %65536 case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>; Len -> << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >> @@ -164,17 +161,15 @@ unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> case unpack(Bin) of {more, Len} -> {more, Len+RestLen-1}; - {Term, Rest}-> - unpack_array_(Rest, RestLen-1, [Term|RetList]) + {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. % FIXME: write test for pack_map/1 pack_map(M)-> - MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> @@ -348,13 +343,15 @@ compare_all([LH|LTL], [RH|RTL]) -> compare_all(LTL, RTL). test_data()-> - [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, + [true, false, nil, + 0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, - [0,42,"sum", [1,2]], [1,42, nil, [3]] + [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]], + 42 ]. basic_test()-> @@ -395,7 +392,7 @@ long_test()-> map_test()-> Ints = lists:seq(0, 65), - Map = dict:from_list([ {X, X*2} || X <- Ints ]), + Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]), {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), ?assertEqual(dict:size(Map), dict:size(Map2)), OrdMap = orddict:from_list( dict:to_list(Map) ), From ad052cb510e3be659df0322cbf33e4840c010b9b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 01:26:57 +0900 Subject: [PATCH 011/259] updated readme --- erlang/README.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/erlang/README.md b/erlang/README.md index 50d446d6..8616d5ec 100644 --- a/erlang/README.md +++ b/erlang/README.md @@ -2,18 +2,8 @@ MessagePack for Erlang ====================== Binary-based efficient object serialization library. -## Status +see wiki ( http://redmine.msgpack.org/projects/msgpack/wiki/QuickStartErlang ) for details -still in development. +# Status -TODOs: - - decide string specification. - -## Installation - -## Example - -## License - - - - +0.1.0 released. From 0cca90c21deb2517de95c83a837bf90efc6b9fa0 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 25 Jun 2010 17:32:11 +0200 Subject: [PATCH 012/259] Fix encoding of fixmap type. The tag value was wrong, and a missing /binary flag caused an error. --- erlang/msgpack.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 789ed535..bbf9e64d 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -168,7 +168,7 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> pack_map(M)-> case dict:size(M) of Len when Len < 16 -> - << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len when Len < 16#10000 -> % 65536 << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> From a1b2b41cdcb4bbc98e21bcd334b729ec6e7b90d5 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sat, 26 Jun 2010 08:40:36 +0900 Subject: [PATCH 013/259] erlang: bugfix(serialization of -234 goes <<208,22>> while it should go int16 <<0xD1, ...>>) --- erlang/msgpack.erl | 20 ++++++++++---------- erlang/testcase_generator.rb | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 789ed535..63d648cd 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -86,15 +86,12 @@ pack_uint_(N) when is_integer( N ) , N < 128 -> % uint 8 pack_uint_( N ) when is_integer( N ) andalso N < 256 -> << 16#CC:8, N:8 >>; - % uint 16 pack_uint_( N ) when is_integer( N ) andalso N < 65536 -> << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>; - % uint 32 pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF-> << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>; - % uint 64 pack_uint_( N ) when is_integer( N )-> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. @@ -103,13 +100,13 @@ pack_uint_( N ) when is_integer( N )-> pack_int_( N ) when is_integer( N ) , N >= -32-> << 2#111:3, N:5 >>; % int 8 -pack_int_( N ) when is_integer( N ) , N >= -256 -> - << 16#D0:8, N:8 >>; +pack_int_( N ) when is_integer( N ) , N > -128 -> + << 16#D0:8, N:8/big-signed-integer-unit:1 >>; % int 16 -pack_int_( N ) when is_integer( N ), N >= -65536 -> +pack_int_( N ) when is_integer( N ), N > -32768 -> << 16#D1:8, N:16/big-signed-integer-unit:1 >>; % int 32 -pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF -> +pack_int_( N ) when is_integer( N ), N > -16#FFFFFFFF -> << 16#D2:8, N:32/big-signed-integer-unit:1 >>; % int 64 pack_int_( N ) when is_integer( N )-> @@ -351,6 +348,7 @@ test_data()-> <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]], + -234, -40000, -16#10000000, -16#100000000, 42 ]. @@ -409,6 +407,7 @@ unknown_test()-> [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], dict:from_list([{1,2},{<<"hoge">>,nil}]), + -234, -50000, 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), @@ -419,9 +418,10 @@ unknown_test()-> port_close(Port). test_([]) -> 0; -test_([S|Rest])-> - Pack = msgpack:pack(S), - {S, <<>>} = msgpack:unpack( Pack ), +test_([Before|Rest])-> + Pack = msgpack:pack(Before), + {After, <<>>} = msgpack:unpack( Pack ), + ?assertEqual(Before, After), 1+test_(Rest). other_test()-> diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index cfc36f96..8445bdd5 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -45,6 +45,7 @@ objs = [0, 1, 2, 123, 512, 1230, 678908, [23, 234, 0.23], [0,42,"sum", [1,2]], [1,42, nil, [3]], { 1 => 2, "hoge" => nil }, + -234, -50000, 42 ] begin From 279121f87f322f3c1a2430751eb3a1032030853d Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 11:56:12 +0200 Subject: [PATCH 014/259] erlang: Use a simple proplist instead of a dict. A dict is overkill (code, cpu, memory) in most cases, and proplist<->dict conversion can easily be done by the libray user if desired. This is in line with other erlang libraries I've seen for various encoding schemes. The map encoder had a bug until I looked at it (see previous commit), so I guess it wasn't used much yet and a change is ok at this stage. The chosen representation for maps is a tuple containing the proplist as the only element. --- erlang/msgpack.erl | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index bbf9e64d..bc636776 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -30,8 +30,7 @@ % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). -type reason() :: enomem | badarg | no_code_matches. --type map() :: any(). % there's no 'dict' type... --type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. +-type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float(). % ===== external APIs ===== % -spec pack(Term::msgpack_term()) -> binary(). @@ -49,8 +48,10 @@ pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> pack_array(List); -pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> +pack({Map}) when is_list(Map) -> pack_map(Map); +pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> + pack_map(dict:from_list(Map)); pack(_O) -> {error, undefined}. @@ -166,13 +167,13 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> % FIXME: write test for pack_map/1 pack_map(M)-> - case dict:size(M) of + case length(M) of Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >> end. pack_map_([])-> <<>>; @@ -182,15 +183,15 @@ pack_map_([{Key,Value}|Tail]) -> % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> {Dict, Bin}; -unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> +unpack_map_(Bin, 0, Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; - { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict)) + { Value, Rest2 } -> + unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]) end end. @@ -295,13 +296,13 @@ unpack_(Flag, Payload)-> 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, dict:new()); + unpack_map_(Rest, Len, []); 16#DE -> {more, 2-PayloadLen}; 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, dict:new()); + unpack_map_(Rest, Len, []); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -325,7 +326,7 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - unpack_map_(Payload, Len, dict:new()); + unpack_map_(Payload, Len, []); _Other -> {error, no_code_matches} @@ -392,12 +393,9 @@ long_test()-> map_test()-> Ints = lists:seq(0, 65), - Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]), + Map = {[ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]}, {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), - ?assertEqual(dict:size(Map), dict:size(Map2)), - OrdMap = orddict:from_list( dict:to_list(Map) ), - OrdMap2 = orddict:from_list( dict:to_list(Map2) ), - ?assertEqual(OrdMap, OrdMap2), + ?assertEqual(Map, Map2), ok. unknown_test()-> From 537322e3b59978d26fcf096433fc718c16cc8b84 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 14:17:44 +0200 Subject: [PATCH 015/259] Big speedup (around 40%) of maps and arrays encoding by using proper tail recursion. --- erlang/msgpack.erl | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index bc636776..255542b6 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -59,7 +59,7 @@ pack(_O) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> +-spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> {error, badarg}; @@ -82,7 +82,7 @@ unpack_all(Data)-> % ===== internal APIs ===== % % positive fixnum -pack_uint_(N) when is_integer( N ) , N < 128 -> +pack_uint_(N) when is_integer( N ) , N < 128 -> << 2#0:1, N:7 >>; % uint 8 pack_uint_( N ) when is_integer( N ) andalso N < 256 -> @@ -145,15 +145,15 @@ pack_raw(Bin) when is_binary(Bin)-> pack_array(L) when is_list(L)-> case length(L) of Len when Len < 16 -> - << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; + << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>; + << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>; Len -> - << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >> + << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >> end. -pack_array_([])-> <<>>; -pack_array_([Head|Tail])-> - << (pack(Head))/binary, (pack_array_(Tail))/binary >>. +pack_array_([], Acc) -> Acc; +pack_array_([Head|Tail], Acc) -> + pack_array_(Tail, <>). % FIXME! this should be tail-recursive and without lists:reverse/1 unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; @@ -169,16 +169,16 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> pack_map(M)-> case length(M) of Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> end. -pack_map_([])-> <<>>; -pack_map_([{Key,Value}|Tail]) -> - << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>. +pack_map_([], Acc) -> Acc; +pack_map_([{Key,Value}|Tail], Acc) -> + pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> @@ -200,7 +200,7 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. unpack_(Flag, Payload)-> PayloadLen = byte_size(Payload), - case Flag of + case Flag of 16#C0 -> {nil, Payload}; 16#C2 -> @@ -313,7 +313,7 @@ unpack_(Flag, Payload)-> {(Code - 16#100), Payload}; Code when Code >= 2#10100000 , Code < 2#11000000 -> -% 101XXXXX for FixRaw +% 101XXXXX for FixRaw Len = Code rem 2#10100000, << Return:Len/binary, Remain/binary >> = Payload, {Return, Remain}; @@ -344,7 +344,7 @@ compare_all([LH|LTL], [RH|RTL]) -> compare_all(LTL, RTL). test_data()-> - [true, false, nil, + [true, false, nil, 0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, From b471e52e281d189396a78cef3eb7ecacbab51d21 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 29 Jun 2010 00:21:47 +0900 Subject: [PATCH 016/259] erlang: explicit API for serializing proplists, so as not to make wrong call of pack({proplists()}). --- erlang/msgpack.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index c8070651..29e1728e 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -24,6 +24,7 @@ %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). +-export([pack_map/1]) % compile: % erl> c(msgpack). @@ -51,7 +52,7 @@ pack(List) when is_list(List) -> pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:from_list(Map)); + pack_map(dict:to_list(Map)); pack(_O) -> {error, undefined}. @@ -78,6 +79,15 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. +pack_map(M)-> + case length(M) of + Len when Len < 16 -> + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len when Len < 16#10000 -> % 65536 + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len -> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> + end. % ===== internal APIs ===== % @@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. -% FIXME: write test for pack_map/1 -pack_map(M)-> - case length(M) of - Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> - end. - pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). @@ -404,7 +403,7 @@ unknown_test()-> -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], - dict:from_list([{1,2},{<<"hoge">>,nil}]), + {[{1,2},{<<"hoge">>,nil}]}, -234, -50000, 42 ], From 90e305d789d231258e26a2dc546cfdb6f8c09ce8 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 29 Jun 2010 00:21:47 +0900 Subject: [PATCH 017/259] erlang: explicit API for serializing proplists, so as not to make wrong call of pack({proplists()}). --- erlang/msgpack.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index c8070651..a168b553 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -24,6 +24,7 @@ %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). +-export([pack_map/1]). % compile: % erl> c(msgpack). @@ -51,7 +52,7 @@ pack(List) when is_list(List) -> pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:from_list(Map)); + pack_map(dict:to_list(Map)); pack(_O) -> {error, undefined}. @@ -78,6 +79,15 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. +pack_map(M)-> + case length(M) of + Len when Len < 16 -> + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len when Len < 16#10000 -> % 65536 + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len -> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> + end. % ===== internal APIs ===== % @@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. -% FIXME: write test for pack_map/1 -pack_map(M)-> - case length(M) of - Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> - end. - pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). @@ -404,7 +403,7 @@ unknown_test()-> -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], - dict:from_list([{1,2},{<<"hoge">>,nil}]), + {[{1,2},{<<"hoge">>,nil}]}, -234, -50000, 42 ], From 8f7f23a0e5fcc595f6d6178e9e532b38b5cd1b46 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 18:11:52 +0200 Subject: [PATCH 018/259] Rewrite unpack_/1 using pattern matching to get a 30-40% speedup. Simplify pack_* and unpack_{array,map} function clauses to get more readability and a minor speedup. --- erlang/msgpack.erl | 290 +++++++++++++++++++-------------------------- 1 file changed, 123 insertions(+), 167 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 255542b6..e94262d1 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -42,8 +42,10 @@ pack(O) when is_float(O) -> pack_double(O); pack(nil) -> pack_nil(); -pack(Bool) when is_atom(Bool) -> - pack_bool(Bool); +pack(true) -> + pack_true(); +pack(false) -> + pack_false(); pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> @@ -61,13 +63,8 @@ pack(_O) -> % TODO: error case for imcomplete format when short for any type formats. -spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. -unpack(Bin) when not is_binary(Bin)-> - {error, badarg}; -unpack(Bin) when bit_size(Bin) >= 8 -> - << Flag:8/unsigned-integer, Payload/binary >> = Bin, - unpack_(Flag, Payload); -unpack(<<>>)-> % when bit_size(Bin) < 8 -> - {more, 1}. +unpack(Bin) -> + unpack_(Bin). -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -82,56 +79,52 @@ unpack_all(Data)-> % ===== internal APIs ===== % % positive fixnum -pack_uint_(N) when is_integer( N ) , N < 128 -> +pack_uint_(N) when N < 128 -> << 2#0:1, N:7 >>; % uint 8 -pack_uint_( N ) when is_integer( N ) andalso N < 256 -> +pack_uint_(N) when N < 256 -> << 16#CC:8, N:8 >>; - % uint 16 -pack_uint_( N ) when is_integer( N ) andalso N < 65536 -> +pack_uint_(N) when N < 65536 -> << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>; - % uint 32 -pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF-> +pack_uint_(N) when N < 16#FFFFFFFF-> << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>; - % uint 64 -pack_uint_( N ) when is_integer( N )-> +pack_uint_(N) -> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. % negative fixnum -pack_int_( N ) when is_integer( N ) , N >= -32-> +pack_int_(N) when N >= -32-> << 2#111:3, N:5 >>; % int 8 -pack_int_( N ) when is_integer( N ) , N >= -256 -> +pack_int_(N) when N >= -256 -> << 16#D0:8, N:8 >>; % int 16 -pack_int_( N ) when is_integer( N ), N >= -65536 -> +pack_int_(N) when N >= -65536 -> << 16#D1:8, N:16/big-signed-integer-unit:1 >>; % int 32 -pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF -> +pack_int_(N) when N >= -16#FFFFFFFF -> << 16#D2:8, N:32/big-signed-integer-unit:1 >>; % int 64 -pack_int_( N ) when is_integer( N )-> +pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. -% nil -pack_nil()-> << 16#C0:8 >>. -% pack_true / pack_false -pack_bool(true)-> << 16#C3:8 >>; -pack_bool(false)-> << 16#C2:8 >>. +% nil/true/false +pack_nil() -> << 16#C0:8 >>. +pack_true()-> << 16#C3:8 >>. +pack_false()-> << 16#C2:8 >>. % float : erlang's float is always IEEE 754 64bit format. %pack_float(F) when is_float(F)-> % << 16#CA:8, F:32/big-float-unit:1 >>. % pack_double(F). % double -pack_double(F) when is_float(F)-> +pack_double(F) -> << 16#CB:8, F:64/big-float-unit:1 >>. % raw bytes -pack_raw(Bin) when is_binary(Bin)-> +pack_raw(Bin) -> case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; @@ -142,24 +135,22 @@ pack_raw(Bin) when is_binary(Bin)-> end. % list / tuple -pack_array(L) when is_list(L)-> +pack_array(L) -> case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>; + << 16#DC:8, Len:16/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len -> - << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >> + << 16#DD:8, Len:32/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >> end. pack_array_([], Acc) -> Acc; pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). % FIXME! this should be tail-recursive and without lists:reverse/1 -unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; -unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; -unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; -unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> +unpack_array_(Remain, 0, RetList) -> {lists:reverse(RetList), Remain}; +unpack_array_(Bin, RestLen, RetList) -> case unpack(Bin) of {more, Len} -> {more, Len+RestLen-1}; {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) @@ -181,10 +172,10 @@ pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). % FIXME: write test for unpack_map/1 --spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> - {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin}; -unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> +-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}]) -> + {more, non_neg_integer()} | {any(), binary()}. +unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(Bin, Len, Acc) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> @@ -195,142 +186,107 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> end end. -% {more, --spec unpack_(Flag::integer(), Payload::binary())-> - {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. -unpack_(Flag, Payload)-> - PayloadLen = byte_size(Payload), - case Flag of - 16#C0 -> - {nil, Payload}; - 16#C2 -> - {false, Payload}; - 16#C3 -> - {true, Payload}; - 16#CA when PayloadLen >= 4 -> % 32bit float - << Return:32/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CA -> - {more, 4-PayloadLen}; % at least more +-spec unpack_(Payload::binary()) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. +unpack_(<<16#C0, Rest/binary>>) -> + {nil, Rest}; +unpack_(<<16#C2, Rest/binary>>) -> + {false, Rest}; +unpack_(<<16#C3, Rest/binary>>) -> + {true, Rest}; - 16#CB when PayloadLen >= 8 -> % 64bit float - << Return:64/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CB -> - {more, 8-PayloadLen}; +unpack_(<<16#CA, Return:32/float-unit:1, Rest/binary>>) -> % 32bit float + {Return, Rest}; +unpack_(<<16#CA, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#CB, Return:64/float-unit:1, Rest/binary>>) -> % 64bit float + {Return, Rest}; +unpack_(<<16#CB, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CC when PayloadLen >= 1 -> % uint 8 - << Int:8/unsigned-integer, Rest/binary >> = Payload, - {Int, Rest}; - 16#CC -> - {more, 1}; +unpack_(<<16#CC, Int:8/unsigned-integer, Rest/binary>>) -> % uint 8 + {Int, Rest}; +unpack_(<<16#CC>>) -> + {more, 1}; +unpack_(<<16#CD, Int:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 16 + {Int, Rest}; +unpack_(<<16#CD, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#CE, Int:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 32 + {Int, Rest}; +unpack_(<<16#CE, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#CF, Int:64/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 64 + {Int, Rest}; +unpack_(<<16#CF, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CD when PayloadLen >= 2 -> % uint 16 - << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CD -> - {more, 2-PayloadLen}; +unpack_(<<16#D0, Int:8/signed-integer, Rest/binary>>) -> % int 8 + {Int, Rest}; +unpack_(<<16#D0>>) -> + {more, 1}; +unpack_(<<16#D1, Int:16/big-signed-integer-unit:1, Rest/binary>>) -> % int 16 + {Int, Rest}; +unpack_(<<16#D1, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#D2, Int:32/big-signed-integer-unit:1, Rest/binary>>) -> % int 32 + {Int, Rest}; +unpack_(<<16#D2, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#D3, Int:64/big-signed-integer-unit:1, Rest/binary>>) -> % int 64 + {Int, Rest}; +unpack_(<<16#D3, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CE when PayloadLen >= 4 -> - << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CE -> - {more, 4-PayloadLen}; % at least more +unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 16 + {Val, Rest}; +unpack_(<<16#DA, Rest/binary>>) -> + {more, 16-byte_size(Rest)}; +unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 32 + {Val, Rest}; +unpack_(<<16#DB, Rest/binary>>) -> + {more, 32-byte_size(Rest)}; - 16#CF when PayloadLen >= 8 -> - << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CF -> - {more, 8-PayloadLen}; +unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 16 + unpack_array_(Rest, Len, []); +unpack_(<<16#DC, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 32 + unpack_array_(Rest, Len, []); +unpack_(<<16#DD, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; - 16#D0 when PayloadLen >= 1 -> % int 8 - << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D0 -> - {more, 1}; +unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 16 + unpack_map_(Rest, Len, []); +unpack_(<<16#DE, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 32 + unpack_map_(Rest, Len, []); +unpack_(<<16#DF, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; - 16#D1 when PayloadLen >= 2 -> % int 16 - << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D1 -> - {more, 2-PayloadLen}; +unpack_(<<0:1, Value:7, Rest/binary>>) -> % positive fixnum + {Value, Rest}; +unpack_(<<2#111:3, Value:5, Rest/binary>>) -> % negative fixnum + {Value - 2#100000, Rest}; +unpack_(<<2#101:3, Len:5, Value:Len/binary, Rest/binary>>) -> % fixraw + {Value, Rest}; +unpack_(<<2#101:3, Len:5, Rest/binary>>) -> + {more, Len-byte_size(Rest)}; +unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> % fixarray + unpack_array_(Rest, Len, []); +unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> % fixmap + unpack_map_(Rest, Len, []); - 16#D2 when PayloadLen >= 4 -> % int 32 - << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D2 -> - {more, 4-PayloadLen}; +%unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9-> +% {error, {badarg, <>}}. +%unpack_(Other) when is_binary(Bin) -> +% {more, 1}. +unpack_(<<>>) -> + {more, 1}. +unpack_(Other) -> + {error, {badarg, Other}}. - 16#D3 when PayloadLen >= 8 -> % int 64 - << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D3 -> - {more, 8-PayloadLen}; - - 16#DA when PayloadLen >= 2 -> % raw 16 - << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DA -> - {more, 16-PayloadLen}; - - 16#DB when PayloadLen >= 4 -> % raw 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DB -> - {more, 4-PayloadLen}; - - 16#DC when PayloadLen >= 2 -> % array 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DC -> - {more, 2-PayloadLen}; - - 16#DD when PayloadLen >= 4 -> % array 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DD -> - {more, 4-PayloadLen}; - - 16#DE when PayloadLen >= 2 -> % map 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - 16#DE -> - {more, 2-PayloadLen}; - - 16#DF when PayloadLen >= 4 -> % map 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - - % positive fixnum - Code when Code >= 2#00000000, Code < 2#10000000-> - {Code, Payload}; - - % negative fixnum - Code when Code >= 2#11100000 -> - {(Code - 16#100), Payload}; - - Code when Code >= 2#10100000 , Code < 2#11000000 -> -% 101XXXXX for FixRaw - Len = Code rem 2#10100000, - << Return:Len/binary, Remain/binary >> = Payload, - {Return, Remain}; - - Code when Code >= 2#10010000 , Code < 2#10100000 -> -% 1001XXXX for FixArray - Len = Code rem 2#10010000, - unpack_array_(Payload, Len, []); - - Code when Code >= 2#10000000 , Code < 2#10010000 -> -% 1000XXXX for FixMap - Len = Code rem 2#10000000, - unpack_map_(Payload, Len, []); - - _Other -> - {error, no_code_matches} - end. % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). @@ -419,7 +375,7 @@ unknown_test()-> test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), - {S, <<>>} = msgpack:unpack( Pack ), + ?assertEqual({S, <<>>}, msgpack:unpack(Pack)), 1+test_(Rest). other_test()-> From 9fffa9800ae4b09180d2b892f1f70215534a874b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 14:54:09 +0900 Subject: [PATCH 019/259] ruby: fixes RDoc of Unpacker#execute and Unpacker#execute_impl --- ruby/makegem.sh | 2 ++ ruby/unpack.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 827f4521..d21f06a1 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,6 +19,8 @@ cp ../test/cases.json test/ gem build msgpack.gemspec +rdoc rbinit.c pack.c unpack.c + if [ $? -eq 0 ]; then rm -rf ext msgpack test/msgpack_test.rb fi diff --git a/ruby/unpack.c b/ruby/unpack.c index 65ae476e..151dbf4a 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -638,7 +638,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, * Document-method: MessagePack::Unpacker#execute_limit * * call-seq: - * unpacker.unpack_limit(data, offset, limit) -> next offset + * unpacker.execute_limit(data, offset, limit) -> next offset * * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes. * @@ -660,7 +660,7 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, * Document-method: MessagePack::Unpacker#execute * * call-seq: - * unpacker.unpack(data, offset) -> next offset + * unpacker.execute(data, offset) -> next offset * * Deserializes one object over the specified buffer from _offset_ bytes. * From 34a29cd0a50eea4a0e008fe3947c86179d536540 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 14:54:40 +0900 Subject: [PATCH 020/259] ruby: fixes SEGV problem caused by GC bug at MessagePack_Unpacker_mark. --- ruby/test/test_helper.rb | 1 + ruby/unpack.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 19226ef0..bf9fee86 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,3 +5,4 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end +GC.stress = true diff --git a/ruby/unpack.c b/ruby/unpack.c index 151dbf4a..09481510 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -287,6 +287,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) unsigned int i; rb_gc_mark(mp->user.stream); rb_gc_mark(mp->user.streambuf); + rb_gc_mark_maybe(template_data(mp)); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); rb_gc_mark_maybe(mp->stack[i].map_key); @@ -297,6 +298,17 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); + + // rb_gc_mark (not _maybe) is used for following member objects. + mp->user.stream = Qnil; + mp->user.streambuf = Qnil; + + mp->user.finished = 0; + mp->user.offset = 0; + mp->user.buffer.size = 0; + mp->user.buffer.free = 0; + mp->user.buffer.ptr = NULL; + obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -343,14 +355,10 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) UNPACKER(self, mp); template_init(mp); - mp->user.finished = 0; - mp->user.offset = 0; - mp->user.buffer.size = 0; - mp->user.buffer.free = 0; - mp->user.buffer.ptr = NULL; mp->user.stream = stream; mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); + return self; } From 123ae024c6d5c217f18a9444c61b292145227278 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 15:12:52 +0900 Subject: [PATCH 021/259] ruby: MessagePack::VERSION constant --- ruby/extconf.rb | 3 ++- ruby/msgpack.gemspec | 3 ++- ruby/rbinit.c | 4 +++- ruby/test/test_helper.rb | 2 +- ruby/test/test_pack_unpack.rb | 4 ++++ ruby/version.rb | 3 +++ 6 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 ruby/version.rb diff --git a/ruby/extconf.rb b/ruby/extconf.rb index e6d4bd6d..eb6a389f 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,4 +1,5 @@ require 'mkmf' -$CFLAGS << " -I.. -Wall -O4" +require './version.rb' +$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"] create_makefile('msgpack') diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index fb6338a5..95a2bd0a 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,8 @@ +require './version.rb' Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.4.2" + s.version = MessagePack::VERSION s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/rbinit.c b/ruby/rbinit.c index ad51f6b4..28a8bfec 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -43,7 +43,9 @@ static VALUE mMessagePack; void Init_msgpack(void) { mMessagePack = rb_define_module("MessagePack"); + + rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); + Init_msgpack_unpack(mMessagePack); Init_msgpack_pack(mMessagePack); } - diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index bf9fee86..80d7806a 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,4 +5,4 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end -GC.stress = true +#GC.stress = true diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index 9dff44f1..25bde81e 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -276,6 +276,10 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase assert_equal(parsed, num) end + it "MessagePack::VERSION constant" do + p MessagePack::VERSION + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/version.rb b/ruby/version.rb new file mode 100644 index 00000000..b1566203 --- /dev/null +++ b/ruby/version.rb @@ -0,0 +1,3 @@ +module MessagePack + VERSION = "0.4.3" +end From 20de730541475516aa7a6361af1d1b5e4ea574b8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 15:39:47 +0900 Subject: [PATCH 022/259] ruby: 0.4.3 --- ruby/ChangeLog | 6 ++++++ ruby/makegem.sh | 1 + 2 files changed, 7 insertions(+) diff --git a/ruby/ChangeLog b/ruby/ChangeLog index e69de29b..d3a72829 100644 --- a/ruby/ChangeLog +++ b/ruby/ChangeLog @@ -0,0 +1,6 @@ + +2010-06-29 version 0.4.3: + + * Adds MessagePack::VERSION constant + * Fixes SEGV problem caused by GC bug at MessagePack_Unpacker_mark + diff --git a/ruby/makegem.sh b/ruby/makegem.sh index d21f06a1..bf30cd47 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -8,6 +8,7 @@ cp pack.h ext/ cp rbinit.c ext/ cp unpack.c ext/ cp unpack.h ext/ +cp version.rb ext/ cp ../msgpack/pack_define.h msgpack/ cp ../msgpack/pack_template.h msgpack/ cp ../msgpack/unpack_define.h msgpack/ From 33a7d56042539282e3b02d75d365dc3dfa57266c Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Tue, 29 Jun 2010 11:59:56 +0200 Subject: [PATCH 023/259] * Return {more,undefined} instead of {more,integer()}, as we can only know the "minimum bytes needed to continue" instead of the actually usefull "total packet size". * Merge all {more,...} clauses of unpack_/1 into one. * Reformat unpack_/1 for readability. * Fix some specs, error values, and documentation. --- erlang/msgpack.erl | 178 ++++++++++++++++----------------------------- 1 file changed, 62 insertions(+), 116 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index e94262d1..dc4907d8 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -29,11 +29,13 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem | badarg | no_code_matches. --type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float(). +-type reason() :: enomem | {badarg, term()}. +-type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary(). % ===== external APIs ===== % --spec pack(Term::msgpack_term()) -> binary(). + +% @doc Pack one erlang term into an msgpack message. +-spec pack(Term::msgpack_term()) -> binary() | reason(). pack(O) when is_integer(O) andalso O < 0 -> pack_int_(O); pack(O) when is_integer(O) -> @@ -54,17 +56,17 @@ pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> pack_map(dict:from_list(Map)); -pack(_O) -> - {error, undefined}. +pack(Other) -> + {error, {badarg, Other}}. -% unpacking. -% if failed in decoding and not end, get more data -% and feed more Bin into this function. -% TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> - {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. -unpack(Bin) -> - unpack_(Bin). +% @doc Unpack one (possibly deeply nested) msgpack message into an erlang term. +% If failed in decoding and not end, get more data +% and feed more Bin into this function. +-spec unpack( binary() ) -> {Decoded::msgpack_term(), Rest::binary()} | {more, undefined} | {error, reason()}. +unpack(Bin) when is_binary(Bin) -> + unpack_(Bin); +unpack(Other) -> + {error, {badarg, Other}}. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -134,7 +136,7 @@ pack_raw(Bin) -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> end. -% list / tuple +% list pack_array(L) -> case length(L) of Len when Len < 16 -> @@ -152,7 +154,7 @@ pack_array_([Head|Tail], Acc) -> unpack_array_(Remain, 0, RetList) -> {lists:reverse(RetList), Remain}; unpack_array_(Bin, RestLen, RetList) -> case unpack(Bin) of - {more, Len} -> {more, Len+RestLen-1}; + {more, undefined} -> {more, undefined}; {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. @@ -172,120 +174,66 @@ pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). % FIXME: write test for unpack_map/1 --spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}]) -> - {more, non_neg_integer()} | {any(), binary()}. +-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}]) -> {more, undefined} | {any(), binary()} | {error, reason()}. unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; unpack_map_(Bin, Len, Acc) -> case unpack(Bin) of - { more, MoreLen } -> { more, MoreLen+Len-1 }; - { Key, Rest } -> + {more, undefined} -> {more, undefined}; + {Key, Rest} -> case unpack(Rest) of - {more, MoreLen} -> { more, MoreLen+Len-1 }; - { Value, Rest2 } -> + {more, undefined} -> {more, undefined}; + {Value, Rest2} -> unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]) end end. --spec unpack_(Payload::binary()) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. -unpack_(<<16#C0, Rest/binary>>) -> - {nil, Rest}; -unpack_(<<16#C2, Rest/binary>>) -> - {false, Rest}; -unpack_(<<16#C3, Rest/binary>>) -> - {true, Rest}; +-spec unpack_(Payload::binary()) -> {more, undefined} | {msgpack_term(), binary()} | {error, reason()}. +% Atoms +unpack_(<<16#C0, Rest/binary>>) -> {nil, Rest}; +unpack_(<<16#C2, Rest/binary>>) -> {false, Rest}; +unpack_(<<16#C3, Rest/binary>>) -> {true, Rest}; -unpack_(<<16#CA, Return:32/float-unit:1, Rest/binary>>) -> % 32bit float - {Return, Rest}; -unpack_(<<16#CA, Rest/binary>>) -> - {more, 4-byte_size(Rest)}; -unpack_(<<16#CB, Return:64/float-unit:1, Rest/binary>>) -> % 64bit float - {Return, Rest}; -unpack_(<<16#CB, Rest/binary>>) -> - {more, 8-byte_size(Rest)}; +% Floats +unpack_(<<16#CA, Val:32/float-unit:1, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#CB, Val:64/float-unit:1, Rest/binary>>) -> {Val, Rest}; -unpack_(<<16#CC, Int:8/unsigned-integer, Rest/binary>>) -> % uint 8 - {Int, Rest}; -unpack_(<<16#CC>>) -> - {more, 1}; -unpack_(<<16#CD, Int:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 16 - {Int, Rest}; -unpack_(<<16#CD, Rest/binary>>) -> - {more, 2-byte_size(Rest)}; -unpack_(<<16#CE, Int:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 32 - {Int, Rest}; -unpack_(<<16#CE, Rest/binary>>) -> - {more, 4-byte_size(Rest)}; -unpack_(<<16#CF, Int:64/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 64 - {Int, Rest}; -unpack_(<<16#CF, Rest/binary>>) -> - {more, 8-byte_size(Rest)}; +% Unsigned integers +unpack_(<<16#CC, Val:8/unsigned-integer, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest}; -unpack_(<<16#D0, Int:8/signed-integer, Rest/binary>>) -> % int 8 - {Int, Rest}; -unpack_(<<16#D0>>) -> - {more, 1}; -unpack_(<<16#D1, Int:16/big-signed-integer-unit:1, Rest/binary>>) -> % int 16 - {Int, Rest}; -unpack_(<<16#D1, Rest/binary>>) -> - {more, 2-byte_size(Rest)}; -unpack_(<<16#D2, Int:32/big-signed-integer-unit:1, Rest/binary>>) -> % int 32 - {Int, Rest}; -unpack_(<<16#D2, Rest/binary>>) -> - {more, 4-byte_size(Rest)}; -unpack_(<<16#D3, Int:64/big-signed-integer-unit:1, Rest/binary>>) -> % int 64 - {Int, Rest}; -unpack_(<<16#D3, Rest/binary>>) -> - {more, 8-byte_size(Rest)}; +% Signed integers +unpack_(<<16#D0, Val:8/signed-integer, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest}; -unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 16 - {Val, Rest}; -unpack_(<<16#DA, Rest/binary>>) -> - {more, 16-byte_size(Rest)}; -unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 32 - {Val, Rest}; -unpack_(<<16#DB, Rest/binary>>) -> - {more, 32-byte_size(Rest)}; +% Raw bytes +unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> {Val, Rest}; +unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> {Val, Rest}; -unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 16 - unpack_array_(Rest, Len, []); -unpack_(<<16#DC, Rest/binary>>) -> - {more, 2-byte_size(Rest)}; -unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 32 - unpack_array_(Rest, Len, []); -unpack_(<<16#DD, Rest/binary>>) -> - {more, 4-byte_size(Rest)}; +% Arrays +unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_array_(Rest, Len, []); +unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_array_(Rest, Len, []); -unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 16 - unpack_map_(Rest, Len, []); -unpack_(<<16#DE, Rest/binary>>) -> - {more, 2-byte_size(Rest)}; -unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 32 - unpack_map_(Rest, Len, []); -unpack_(<<16#DF, Rest/binary>>) -> - {more, 4-byte_size(Rest)}; +% Maps +unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_map_(Rest, Len, []); +unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_map_(Rest, Len, []); -unpack_(<<0:1, Value:7, Rest/binary>>) -> % positive fixnum - {Value, Rest}; -unpack_(<<2#111:3, Value:5, Rest/binary>>) -> % negative fixnum - {Value - 2#100000, Rest}; -unpack_(<<2#101:3, Len:5, Value:Len/binary, Rest/binary>>) -> % fixraw - {Value, Rest}; -unpack_(<<2#101:3, Len:5, Rest/binary>>) -> - {more, Len-byte_size(Rest)}; -unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> % fixarray - unpack_array_(Rest, Len, []); -unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> % fixmap - unpack_map_(Rest, Len, []); +% Tag-encoded lengths (kept last, for speed) +unpack_(<<0:1, Val:7, Rest/binary>>) -> {Val, Rest}; % pos fixnum +unpack_(<<2#111:3, Val:5, Rest/binary>>) -> {Val - 2#100000, Rest}; % neg fixnum +unpack_(<<2#101:3, Len:5, Val:Len/binary, Rest/binary>>) -> {Val, Rest}; % fixraw +unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> unpack_array_(Rest, Len, []); % fixarray +unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> unpack_map_(Rest, Len, []); % fixmap -%unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9-> -% {error, {badarg, <>}}. -%unpack_(Other) when is_binary(Bin) -> -% {more, 1}. -unpack_(<<>>) -> - {more, 1}. -unpack_(Other) -> - {error, {badarg, Other}}. +% Incomplete / invalid data +unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9-> + {error, {badarg, <>}}; +unpack_(_Bin) -> + {more, undefined}. % ===== test codes ===== % @@ -330,9 +278,7 @@ test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, - {more, N}=msgpack:unpack(Bin), - ?assert(0 < N), - ?assert(N < Len). + ?assertEqual({more, undefined}, msgpack:unpack(Bin)). partial_test()-> % error handling test. Term = lists:seq(0, 45), @@ -379,6 +325,6 @@ test_([S|Rest])-> 1+test_(Rest). other_test()-> - {more,1}=msgpack:unpack(<<>>). + ?assertEqual({more,undefined}, msgpack:unpack(<<>>)). -endif. From 83b4b7d83d0d32b2c30d26e51439b4dfec3127f9 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 00:58:48 +0900 Subject: [PATCH 024/259] erlang: more suitable variable name and removing unnecessary guards. --- erlang/msgpack.erl | 75 +++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index a168b553..c99002c3 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -31,20 +31,24 @@ % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). -type reason() :: enomem | badarg | no_code_matches. --type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float(). +-type msgpack_term() :: [msgpack_term()] + | {[{msgpack_term(),msgpack_term()}]} + | integer() | float() | binary(). % ===== external APIs ===== % -spec pack(Term::msgpack_term()) -> binary(). -pack(O) when is_integer(O) andalso O < 0 -> - pack_int_(O); -pack(O) when is_integer(O) -> - pack_uint_(O); -pack(O) when is_float(O) -> - pack_double(O); +pack(I) when is_integer(I) andalso I < 0 -> + pack_int_(I); +pack(I) when is_integer(I) -> + pack_uint_(I); +pack(F) when is_float(F) -> + pack_double(F); pack(nil) -> - pack_nil(); -pack(Bool) when is_atom(Bool) -> - pack_bool(Bool); + << 16#C0:8 >>; +pack(true) -> + << 16#C3:8 >>; +pack(false) -> + << 16#C2:8 >>; pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> @@ -53,7 +57,7 @@ pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> pack_map(dict:to_list(Map)); -pack(_O) -> +pack(_Other) -> {error, undefined}. % unpacking. @@ -92,53 +96,47 @@ pack_map(M)-> % ===== internal APIs ===== % % positive fixnum -pack_uint_(N) when is_integer( N ) , N < 128 -> +pack_uint_(N) when N < 128 -> << 2#0:1, N:7 >>; % uint 8 -pack_uint_( N ) when is_integer( N ) andalso N < 256 -> +pack_uint_(N) when N < 256 -> << 16#CC:8, N:8 >>; % uint 16 -pack_uint_( N ) when is_integer( N ) andalso N < 65536 -> +pack_uint_(N) when N < 65536 -> << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>; % uint 32 -pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF-> +pack_uint_(N) when N < 16#FFFFFFFF-> << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>; % uint 64 -pack_uint_( N ) when is_integer( N )-> +pack_uint_(N) -> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. % negative fixnum -pack_int_( N ) when is_integer( N ) , N >= -32-> +pack_int_(N) when is_integer(N) , N >= -32-> << 2#111:3, N:5 >>; % int 8 -pack_int_( N ) when is_integer( N ) , N > -128 -> +pack_int_(N) when N > -128 -> << 16#D0:8, N:8/big-signed-integer-unit:1 >>; % int 16 -pack_int_( N ) when is_integer( N ), N > -32768 -> +pack_int_(N) when N > -32768 -> << 16#D1:8, N:16/big-signed-integer-unit:1 >>; % int 32 -pack_int_( N ) when is_integer( N ), N > -16#FFFFFFFF -> +pack_int_(N) when N > -16#FFFFFFFF -> << 16#D2:8, N:32/big-signed-integer-unit:1 >>; % int 64 -pack_int_( N ) when is_integer( N )-> +pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. -% nil -pack_nil()-> << 16#C0:8 >>. -% pack_true / pack_false -pack_bool(true)-> << 16#C3:8 >>; -pack_bool(false)-> << 16#C2:8 >>. - % float : erlang's float is always IEEE 754 64bit format. %pack_float(F) when is_float(F)-> % << 16#CA:8, F:32/big-float-unit:1 >>. % pack_double(F). % double -pack_double(F) when is_float(F)-> +pack_double(F) -> << 16#CB:8, F:64/big-float-unit:1 >>. % raw bytes -pack_raw(Bin) when is_binary(Bin)-> +pack_raw(Bin) -> case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; @@ -149,7 +147,7 @@ pack_raw(Bin) when is_binary(Bin)-> end. % list / tuple -pack_array(L) when is_list(L)-> +pack_array(L) -> case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>; @@ -165,10 +163,10 @@ pack_array_([Head|Tail], Acc) -> % FIXME! this should be tail-recursive and without lists:reverse/1 unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; -unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; +unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, undefined}; unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> case unpack(Bin) of - {more, Len} -> {more, Len+RestLen-1}; + {more, Len} -> {more, undefined}; {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. @@ -179,8 +177,8 @@ pack_map_([{Key,Value}|Tail], Acc) -> % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin}; -unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> +unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(Bin, Len, Acc) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> @@ -371,9 +369,12 @@ test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, - {more, N}=msgpack:unpack(Bin), - ?assert(0 < N), - ?assert(N < Len). + case msgpack:unpack(Bin) of + {more, N} when not is_integer(N) -> + ?assertEqual(undefined, N); + {more, N} -> + ?assert( N < Len ) + end. partial_test()-> % error handling test. Term = lists:seq(0, 45), From acb8fa613e87081267ec9963e5770580208e3e1f Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 01:02:19 +0900 Subject: [PATCH 025/259] erlang: adding shorthand fix for {more, undefined} problem --- erlang/msgpack.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index c99002c3..d24220be 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -166,7 +166,7 @@ unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetLis unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, undefined}; unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> case unpack(Bin) of - {more, Len} -> {more, undefined}; + {more, _} -> {more, undefined}; {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. @@ -180,11 +180,11 @@ pack_map_([{Key,Value}|Tail], Acc) -> unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; unpack_map_(Bin, Len, Acc) -> case unpack(Bin) of - { more, MoreLen } -> { more, MoreLen+Len-1 }; - { Key, Rest } -> + {more, _} -> {more, undefined}; + {Key, Rest} -> case unpack(Rest) of - {more, MoreLen} -> { more, MoreLen+Len-1 }; - { Value, Rest2 } -> + {more, _} -> {more, undefined}; + {Value, Rest2} -> unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]) end end. From 2469768a85a067e5ba425d09ebd717cd0104449a Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 01:07:56 +0900 Subject: [PATCH 026/259] erlang: reducing unnecessary binary matching in unpack_/2 * more efficient unpack_/1 by Vincent de Phille's code. thanks. --- erlang/msgpack.erl | 207 +++++++++++++++------------------------------ 1 file changed, 70 insertions(+), 137 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d24220be..703dce25 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -69,10 +69,11 @@ pack(_Other) -> unpack(Bin) when not is_binary(Bin)-> {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> - << Flag:8/unsigned-integer, Payload/binary >> = Bin, - unpack_(Flag, Payload); -unpack(<<>>)-> % when bit_size(Bin) < 8 -> - {more, 1}. + unpack_(Bin); +unpack(<<>>)-> + {more, 1}; +unpack(_) -> + {more, undefined}. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -83,6 +84,7 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. +-spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary(). pack_map(M)-> case length(M) of Len when Len < 16 -> @@ -189,143 +191,74 @@ unpack_map_(Bin, Len, Acc) -> end end. -% {more, --spec unpack_(Flag::integer(), Payload::binary())-> - {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. -unpack_(Flag, Payload)-> - PayloadLen = byte_size(Payload), - case Flag of - 16#C0 -> - {nil, Payload}; - 16#C2 -> - {false, Payload}; - 16#C3 -> - {true, Payload}; +-spec unpack_(Payload::binary()) -> + {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. +unpack_(Binary)-> + case Binary of +% ATOMS + <<16#C0, Rest/binary>> -> {nil, Rest}; + <<16#C2, Rest/binary>> -> {false, Rest}; + <<16#C3, Rest/binary>> -> {true, Rest}; +% Floats + <<16#CA, Val:32/float-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#CB, Val:64/float-unit:1, Rest/binary>> -> {Val, Rest}; +% Unsigned integers + <<16#CC, Val:8/unsigned-integer, Rest/binary>> -> {Val, Rest}; + <<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; +% Signed integers + <<16#D0, Val:8/signed-integer, Rest/binary>> -> {Val, Rest}; + <<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; +% Raw bytes + <<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest}; + <<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest}; +% Arrays + <<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []); + <<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []); +% Maps + <<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []); + <<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []); - 16#CA when PayloadLen >= 4 -> % 32bit float - << Return:32/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CA -> - {more, 4-PayloadLen}; % at least more +% Tag-encoded lengths (kept last, for speed) + <<0:1, Val:7, Rest/binary>> -> {Val, Rest}; % positive int + <<2#111:3, Val:5, Rest/binary>> -> {Val - 2#100000, Rest}; % negative int + <<2#101:3, Len:5, Val:Len/binary, Rest/binary>> -> {Val, Rest}; % raw bytes + <<2#1001:4, Len:4, Rest/binary>> -> unpack_array_(Rest, Len, []); % array + <<2#1000:4, Len:4, Rest/binary>> -> unpack_map_(Rest, Len, []); % map + +% Incomplete / invalid data + <<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)}; + <<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)}; + <<16#CC>> -> {more, 1}; + <<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)}; + <<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)}; + <<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)}; + <<16#D0>> -> {more, 1}; + <<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)}; + <<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)}; + <<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)}; + <<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)}; + <<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)}; + <<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)}; + <<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)}; + <<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)}; + <<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)}; + <<2#101:3, Len:5, Rest/binary>> -> {more, Len-byte_size(Rest)}; - 16#CB when PayloadLen >= 8 -> % 64bit float - << Return:64/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CB -> - {more, 8-PayloadLen}; - - 16#CC when PayloadLen >= 1 -> % uint 8 - << Int:8/unsigned-integer, Rest/binary >> = Payload, - {Int, Rest}; - 16#CC -> - {more, 1}; - - 16#CD when PayloadLen >= 2 -> % uint 16 - << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CD -> - {more, 2-PayloadLen}; - - 16#CE when PayloadLen >= 4 -> - << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CE -> - {more, 4-PayloadLen}; % at least more - - 16#CF when PayloadLen >= 8 -> - << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CF -> - {more, 8-PayloadLen}; - - 16#D0 when PayloadLen >= 1 -> % int 8 - << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D0 -> - {more, 1}; - - 16#D1 when PayloadLen >= 2 -> % int 16 - << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D1 -> - {more, 2-PayloadLen}; - - 16#D2 when PayloadLen >= 4 -> % int 32 - << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D2 -> - {more, 4-PayloadLen}; - - 16#D3 when PayloadLen >= 8 -> % int 64 - << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D3 -> - {more, 8-PayloadLen}; - - 16#DA when PayloadLen >= 2 -> % raw 16 - << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DA -> - {more, 16-PayloadLen}; - - 16#DB when PayloadLen >= 4 -> % raw 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DB -> - {more, 4-PayloadLen}; - - 16#DC when PayloadLen >= 2 -> % array 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DC -> - {more, 2-PayloadLen}; - - 16#DD when PayloadLen >= 4 -> % array 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DD -> - {more, 4-PayloadLen}; - - 16#DE when PayloadLen >= 2 -> % map 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - 16#DE -> - {more, 2-PayloadLen}; - - 16#DF when PayloadLen >= 4 -> % map 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - - % positive fixnum - Code when Code >= 2#00000000, Code < 2#10000000-> - {Code, Payload}; - - % negative fixnum - Code when Code >= 2#11100000 -> - {(Code - 16#100), Payload}; - - Code when Code >= 2#10100000 , Code < 2#11000000 -> -% 101XXXXX for FixRaw - Len = Code rem 2#10100000, - << Return:Len/binary, Remain/binary >> = Payload, - {Return, Remain}; - - Code when Code >= 2#10010000 , Code < 2#10100000 -> -% 1001XXXX for FixArray - Len = Code rem 2#10010000, - unpack_array_(Payload, Len, []); - - Code when Code >= 2#10000000 , Code < 2#10010000 -> -% 1000XXXX for FixMap - Len = Code rem 2#10000000, - unpack_map_(Payload, Len, []); - - _Other -> - {error, no_code_matches} + <<>> -> {more, 1}; + <<2#101:3, _/binary>> -> {more, undefined}; + <> when F==16#C1; + F==16#C7; F==16#C8; F==16#C9; F==16#D5; + F==16#D6; F==16#D7; F==16#D8; F==16#D9-> + {error, {badarg, <>}}; + Other -> + {error, {badarg, Other}} end. + % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). From 370e92b1a6d79cd6d65840cae588ded11b167fce Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 01:14:20 +0900 Subject: [PATCH 027/259] erlang: just a golf. --- erlang/msgpack.erl | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 703dce25..6d94a230 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -200,34 +200,34 @@ unpack_(Binary)-> <<16#C2, Rest/binary>> -> {false, Rest}; <<16#C3, Rest/binary>> -> {true, Rest}; % Floats - <<16#CA, Val:32/float-unit:1, Rest/binary>> -> {Val, Rest}; - <<16#CB, Val:64/float-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#CA, V:32/float-unit:1, Rest/binary>> -> {V, Rest}; + <<16#CB, V:64/float-unit:1, Rest/binary>> -> {V, Rest}; % Unsigned integers - <<16#CC, Val:8/unsigned-integer, Rest/binary>> -> {Val, Rest}; - <<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; - <<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; - <<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#CC, V:8/unsigned-integer, Rest/binary>> -> {V, Rest}; + <<16#CD, V:16/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest}; + <<16#CE, V:32/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest}; + <<16#CF, V:64/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest}; % Signed integers - <<16#D0, Val:8/signed-integer, Rest/binary>> -> {Val, Rest}; - <<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; - <<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; - <<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest}; + <<16#D0, V:8/signed-integer, Rest/binary>> -> {V, Rest}; + <<16#D1, V:16/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest}; + <<16#D2, V:32/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest}; + <<16#D3, V:64/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest}; % Raw bytes - <<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest}; - <<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest}; + <<16#DA, L:16/unsigned-integer-unit:1, V:L/binary, Rest/binary>> -> {V, Rest}; + <<16#DB, L:32/unsigned-integer-unit:1, V:L/binary, Rest/binary>> -> {V, Rest}; % Arrays - <<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []); - <<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []); + <<16#DC, L:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, L, []); + <<16#DD, L:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, L, []); % Maps - <<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []); - <<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []); + <<16#DE, L:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, L, []); + <<16#DF, L:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, L, []); % Tag-encoded lengths (kept last, for speed) - <<0:1, Val:7, Rest/binary>> -> {Val, Rest}; % positive int - <<2#111:3, Val:5, Rest/binary>> -> {Val - 2#100000, Rest}; % negative int - <<2#101:3, Len:5, Val:Len/binary, Rest/binary>> -> {Val, Rest}; % raw bytes - <<2#1001:4, Len:4, Rest/binary>> -> unpack_array_(Rest, Len, []); % array - <<2#1000:4, Len:4, Rest/binary>> -> unpack_map_(Rest, Len, []); % map + <<0:1, V:7, Rest/binary>> -> {V, Rest}; % positive int + <<2#111:3, V:5, Rest/binary>> -> {V - 2#100000, Rest}; % negative int + <<2#101:3, L:5, V:L/binary, Rest/binary>> -> {V, Rest}; % raw bytes + <<2#1001:4, L:4, Rest/binary>> -> unpack_array_(Rest, L, []); % array + <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map % Incomplete / invalid data <<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)}; @@ -246,7 +246,7 @@ unpack_(Binary)-> <<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)}; <<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)}; <<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<2#101:3, Len:5, Rest/binary>> -> {more, Len-byte_size(Rest)}; + <<2#101:3, L:5, Rest/binary>> -> {more, L-byte_size(Rest)}; <<>> -> {more, 1}; <<2#101:3, _/binary>> -> {more, undefined}; From ff5d5d7cbcb7e136370f3450d67b06244d5c3bc7 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 01:14:38 +0900 Subject: [PATCH 028/259] erlang: updated the comments --- erlang/msgpack.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 6d94a230..8a542712 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -162,7 +162,7 @@ pack_array_([], Acc) -> Acc; pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). -% FIXME! this should be tail-recursive and without lists:reverse/1 +% FIXME! this should be without lists:reverse/1 unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, undefined}; @@ -176,7 +176,7 @@ pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). -% FIXME: write test for unpack_map/1 +% FIXME! this should be without lists:reverse/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; From 584462f9b9efefa25e93600024c3d9680923b3a4 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 1 Jul 2010 01:16:25 +0900 Subject: [PATCH 029/259] erlang: improved spec. --- erlang/msgpack.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 8a542712..aab07b72 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -64,8 +64,9 @@ pack(_Other) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> - {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +-spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | + {more, non_neg_integer()} | {more, undefined} | + {error, reason()}. unpack(Bin) when not is_binary(Bin)-> {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> From 71dd44f4308dbdda2d087130452182093262ee01 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 12:26:21 +0900 Subject: [PATCH 030/259] cpp: adds operator<<(std::ostream&, const tuple&) (experimental) --- cpp/src/msgpack/type/tuple.hpp.erb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cpp/src/msgpack/type/tuple.hpp.erb b/cpp/src/msgpack/type/tuple.hpp.erb index 0d9ae91c..ebef8163 100644 --- a/cpp/src/msgpack/type/tuple.hpp.erb +++ b/cpp/src/msgpack/type/tuple.hpp.erb @@ -187,5 +187,20 @@ inline void operator<< ( } // namespace msgpack + +//inline std::ostream& operator<< (std::ostream& o, const msgpack::type::tuple<>& v) { +// return o << "[]"; +//} +//<%0.upto(GENERATION_LIMIT) {|i|%> +//template , typename A<%=j%><%}%>> +//inline std::ostream& operator<< (std::ostream& o, +// const msgpack::type::tuple, A<%=j%><%}%>>& v) { +// return o << "[" +// <%0.upto(i) {|j|%> +// <<<%if j != 0 then%> ", " <<<%end%> v.template get<<%=j%>>()<%}%> +// << "]"; +//} +//<%}%> + #endif /* msgpack/type/tuple.hpp */ From 3af10a1d000882f338529360bf35ae0e3a21ffc4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 17:00:58 +0900 Subject: [PATCH 031/259] cpp: adds MSGPACK_VERSION{,_MAJOR,_MINOR} macros and msgpack{,_major,_minor} functions --- cpp/Makefile.am | 1 - cpp/configure.in | 5 +++++ cpp/src/Makefile.am | 15 ++++++++++---- cpp/src/msgpack.h | 2 ++ cpp/src/msgpack/version.h.in | 40 ++++++++++++++++++++++++++++++++++++ cpp/src/version.c | 17 +++++++++++++++ cpp/test/Makefile.am | 3 +++ 7 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 cpp/src/msgpack/version.h.in create mode 100644 cpp/src/version.c diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 871ff8c0..7dd48910 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -12,7 +12,6 @@ EXTRA_DIST = \ $(DOC_FILES) doxygen: - ./preprocess ./preprocess clean cd src && $(MAKE) doxygen ./preprocess diff --git a/cpp/configure.in b/cpp/configure.in index 0895be4b..ffe3671e 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -54,5 +54,10 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f ]) fi +major=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\1/'` +minor=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\2/'` +AC_SUBST(VERSION_MAJOR, $major) +AC_SUBST(VERSION_MINOR, $minor) + AC_OUTPUT([Makefile src/Makefile test/Makefile]) diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index 6b6457e6..78d14aa3 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -4,6 +4,7 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ unpack.c \ objectc.c \ + version.c \ vrefbuffer.c \ zone.c \ object.cpp @@ -18,15 +19,12 @@ lib_LTLIBRARIES += libmsgpackc.la libmsgpackc_la_SOURCES = \ unpack.c \ objectc.c \ + version.c \ vrefbuffer.c \ zone.c libmsgpackc_la_LDFLAGS = -version-info 2:0:0 -# work around for duplicated file name -kumo_manager_CFLAGS = $(AM_CFLAGS) -kumo_manager_CXXFLAGS = $(AM_CXXFLAGS) - nobase_include_HEADERS = \ msgpack/pack_define.h \ @@ -44,6 +42,7 @@ nobase_include_HEADERS = \ msgpack/zone.h \ msgpack.hpp \ msgpack/sbuffer.hpp \ + msgpack/version.h \ msgpack/vrefbuffer.hpp \ msgpack/zbuffer.hpp \ msgpack/pack.hpp \ @@ -69,11 +68,19 @@ nobase_include_HEADERS = \ msgpack/type/tr1/unordered_set.hpp EXTRA_DIST = \ + msgpack/version.h.in \ msgpack/zone.hpp.erb \ msgpack/type/define.hpp.erb \ msgpack/type/tuple.hpp.erb +msgpack/version.h: msgpack/version.h.in Makefile.in + sed -e s/VERSION_UNDEFINED/$(VERSION)/ \ + -e s/VERSION_MAJOR_UNDEFINED/$(VERSION_MAJOR)/ \ + -e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \ + $< > $@ + + doxygen_c: cat ../Doxyfile > Doxyfile_c echo "FILE_PATTERNS = *.h" >> Doxyfile_c diff --git a/cpp/src/msgpack.h b/cpp/src/msgpack.h index 0cd8a199..08f27688 100644 --- a/cpp/src/msgpack.h +++ b/cpp/src/msgpack.h @@ -26,3 +26,5 @@ #include "msgpack/unpack.h" #include "msgpack/sbuffer.h" #include "msgpack/vrefbuffer.h" +#include "msgpack/version.h" + diff --git a/cpp/src/msgpack/version.h.in b/cpp/src/msgpack/version.h.in new file mode 100644 index 00000000..af292d0d --- /dev/null +++ b/cpp/src/msgpack/version.h.in @@ -0,0 +1,40 @@ +/* + * MessagePack for C version information + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_VERSION_H__ +#define MSGPACK_VERSION_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + +const char* msgpack_version(void); +int msgpack_version_major(void); +int msgpack_version_minor(void); + +#define MSGPACK_VERSION "VERSION_UNDEFINED" +#define MSGPACK_VERSION_MAJOR VERSION_MAJOR_UNDEFINED +#define MSGPACK_VERSION_MINOR VERSION_MINOR_UNDEFINED + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/version.h */ + diff --git a/cpp/src/version.c b/cpp/src/version.c new file mode 100644 index 00000000..3d956f18 --- /dev/null +++ b/cpp/src/version.c @@ -0,0 +1,17 @@ +#include "msgpack.h" + +const char* msgpack_version(void) +{ + return MSGPACK_VERSION; +} + +int msgpack_version_major(void) +{ + return MSGPACK_VERSION_MAJOR; +} + +int msgpack_version_minor(void) +{ + return MSGPACK_VERSION_MINOR; +} + diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am index 1f522150..bb9439e3 100644 --- a/cpp/test/Makefile.am +++ b/cpp/test/Makefile.am @@ -13,6 +13,7 @@ check_PROGRAMS = \ convert \ buffer \ cases \ + version \ msgpackc_test \ msgpack_test @@ -37,6 +38,8 @@ buffer_LDADD = -lz cases_SOURCES = cases.cc +version_SOURCES = version.cc + msgpackc_test_SOURCES = msgpackc_test.cpp msgpack_test_SOURCES = msgpack_test.cpp From c57f6161415156328bd7039be44895b868ee6f76 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 17:10:25 +0900 Subject: [PATCH 032/259] cpp: adds MSGPACK_VERSION{,_MAJOR,_MINOR} macros and msgpack{,_major,_minor} functions --- cpp/src/Makefile.am | 2 ++ cpp/test/version.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 cpp/test/version.cc diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index 78d14aa3..79b692d4 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -80,6 +80,8 @@ msgpack/version.h: msgpack/version.h.in Makefile.in -e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \ $< > $@ +#version.c: msgpack/version.h + doxygen_c: cat ../Doxyfile > Doxyfile_c diff --git a/cpp/test/version.cc b/cpp/test/version.cc new file mode 100644 index 00000000..9357271e --- /dev/null +++ b/cpp/test/version.cc @@ -0,0 +1,13 @@ +#include +#include + +TEST(version, print) +{ + printf("MSGPACK_VERSION : %s\n", MSGPACK_VERSION); + printf("MSGPACK_VERSION_MAJOR : %d\n", MSGPACK_VERSION_MAJOR); + printf("MSGPACK_VERSION_MINOR : %d\n", MSGPACK_VERSION_MINOR); + printf("msgpack_version() : %s\n", msgpack_version()); + printf("msgpack_version_major() : %d\n", msgpack_version_major()); + printf("msgpack_version_minor() : %d\n", msgpack_version_minor()); +} + From a2bd5ae6386af95617635fec1503640cbadb627b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 17:45:15 +0900 Subject: [PATCH 033/259] cpp: ./configure supports --disable-cxx option not to build/install C++ API --- cpp/configure.in | 27 ++++++++++++++++++--------- cpp/src/Makefile.am | 14 ++++++++++---- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/cpp/configure.in b/cpp/configure.in index ffe3671e..04fa8e34 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -9,23 +9,31 @@ CFLAGS="-O4 -Wall $CFLAGS" AC_SUBST(CXXFLAGS) CXXFLAGS="-O4 -Wall $CXXFLAGS" + AC_PROG_CC -AC_PROG_CXX + + +AC_MSG_CHECKING([if C++ API is enabled]) +AC_ARG_ENABLE(cxx, + AS_HELP_STRING([--disable-cxx], + [don't build C++ API]) ) +AC_MSG_RESULT([$enable_cxx]) +if test "$enable_cxx" != "no"; then + AC_PROG_CXX + AM_PROG_CC_C_O +fi +AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") + AC_PROG_LIBTOOL AM_PROG_AS -AM_PROG_CC_C_O - -AC_LANG_PUSH([C++]) -AC_CHECK_HEADERS(tr1/unordered_map) -AC_CHECK_HEADERS(tr1/unordered_set) -AC_LANG_POP([C++]) AC_MSG_CHECKING([if debug option is enabled]) AC_ARG_ENABLE(debug, AS_HELP_STRING([--disable-debug], - [disable assert macros and omit -g option.]) ) + [disable assert macros and omit -g option]) ) +AC_MSG_RESULT([$enable_debug]) if test "$enable_debug" != "no"; then CXXFLAGS="$CXXFLAGS -g" CFLAGS="$CFLAGS -g" @@ -33,7 +41,6 @@ else CXXFLAGS="$CXXFLAGS -DNDEBUG" CFLAGS="$CFLAGS -DNDEBUG" fi -AC_MSG_RESULT($enable_debug) AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [ @@ -54,10 +61,12 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f ]) fi + major=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\1/'` minor=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\2/'` AC_SUBST(VERSION_MAJOR, $major) AC_SUBST(VERSION_MINOR, $minor) + AC_OUTPUT([Makefile src/Makefile test/Makefile]) diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index 79b692d4..fc673856 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -6,8 +6,12 @@ libmsgpack_la_SOURCES = \ objectc.c \ version.c \ vrefbuffer.c \ - zone.c \ + zone.c + +if ENABLE_CXX +libmsgpack_la_SOURCES += \ object.cpp +endif # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 3:0:0 @@ -39,7 +43,10 @@ nobase_include_HEADERS = \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ - msgpack/zone.h \ + msgpack/zone.h + +if ENABLE_CXX +nobase_include_HEADERS += \ msgpack.hpp \ msgpack/sbuffer.hpp \ msgpack/version.h \ @@ -66,6 +73,7 @@ nobase_include_HEADERS = \ msgpack/type/define.hpp \ msgpack/type/tr1/unordered_map.hpp \ msgpack/type/tr1/unordered_set.hpp +endif EXTRA_DIST = \ msgpack/version.h.in \ @@ -80,8 +88,6 @@ msgpack/version.h: msgpack/version.h.in Makefile.in -e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \ $< > $@ -#version.c: msgpack/version.h - doxygen_c: cat ../Doxyfile > Doxyfile_c From 39facd5dc660b90304e4d3d593244a5b9f7cd6f0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 17:59:07 +0900 Subject: [PATCH 034/259] cpp: version 0.5.1 --- cpp/ChangeLog | 10 ++++++++++ cpp/configure.in | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cpp/ChangeLog b/cpp/ChangeLog index 53a99910..3277c139 100644 --- a/cpp/ChangeLog +++ b/cpp/ChangeLog @@ -1,4 +1,14 @@ +2010-07-06 version 0.5.1: + + * Add msgpack_vrefbuffer_new and msgpack_vrefbuffer_free + * Add msgpack_sbuffer_new and msgpack_sbuffer_free + * Add msgpack_unpacker_next and msgpack_unpack_next + * msgpack::unpack returns void + * Add MSGPACK_VERSION{,_MAJOR,_MINOR} macros to check header version + * Add msgpack_version{,_major,_minor} functions to check library version + * ./configure supports --disable-cxx option not to build C++ API + 2010-04-29 version 0.5.0: * msgpack_object_type is changed. MSGPACK_OBJECT_NIL is now 0x00. diff --git a/cpp/configure.in b/cpp/configure.in index 04fa8e34..dd04ed47 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,6 +1,6 @@ AC_INIT(src/object.cpp) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.5.0) +AM_INIT_AUTOMAKE(msgpack, 0.5.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 0c331d288715b156f262cdb7841fb0dba4dc5d83 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 18:18:28 +0900 Subject: [PATCH 035/259] cpp: updates vcproj --- cpp/msgpack_vc8.postbuild.bat | 77 ++++++++++++++++++----------------- cpp/msgpack_vc8.vcproj | 40 +++++++++++++----- cpp/src/Makefile.am | 2 +- 3 files changed, 70 insertions(+), 49 deletions(-) diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat index bae13f37..20fabbb9 100644 --- a/cpp/msgpack_vc8.postbuild.bat +++ b/cpp/msgpack_vc8.postbuild.bat @@ -2,42 +2,43 @@ IF NOT EXIST include MKDIR include IF NOT EXIST include\msgpack MKDIR include\msgpack IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 -copy msgpack\pack_define.h include\msgpack\ -copy msgpack\pack_template.h include\msgpack\ -copy msgpack\unpack_define.h include\msgpack\ -copy msgpack\unpack_template.h include\msgpack\ -copy msgpack\sysdep.h include\msgpack\ -copy msgpack.h include\ -copy msgpack\sbuffer.h include\msgpack\ -copy msgpack\vrefbuffer.h include\msgpack\ -copy msgpack\zbuffer.h include\msgpack\ -copy msgpack\pack.h include\msgpack\ -copy msgpack\unpack.h include\msgpack\ -copy msgpack\object.h include\msgpack\ -copy msgpack\zone.h include\msgpack\ -copy msgpack.hpp include\ -copy msgpack\sbuffer.hpp include\msgpack\ -copy msgpack\vrefbuffer.hpp include\msgpack\ -copy msgpack\zbuffer.hpp include\msgpack\ -copy msgpack\pack.hpp include\msgpack\ -copy msgpack\unpack.hpp include\msgpack\ -copy msgpack\object.hpp include\msgpack\ -copy msgpack\zone.hpp include\msgpack\ -copy msgpack\type.hpp include\msgpack\type\ -copy msgpack\type\bool.hpp include\msgpack\type\ -copy msgpack\type\float.hpp include\msgpack\type\ -copy msgpack\type\int.hpp include\msgpack\type\ -copy msgpack\type\list.hpp include\msgpack\type\ -copy msgpack\type\deque.hpp include\msgpack\type\ -copy msgpack\type\map.hpp include\msgpack\type\ -copy msgpack\type\nil.hpp include\msgpack\type\ -copy msgpack\type\pair.hpp include\msgpack\type\ -copy msgpack\type\raw.hpp include\msgpack\type\ -copy msgpack\type\set.hpp include\msgpack\type\ -copy msgpack\type\string.hpp include\msgpack\type\ -copy msgpack\type\vector.hpp include\msgpack\type\ -copy msgpack\type\tuple.hpp include\msgpack\type\ -copy msgpack\type\define.hpp include\msgpack\type\ -copy msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ -copy msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ +copy src\msgpack\pack_define.h include\msgpack\ +copy src\msgpack\pack_template.h include\msgpack\ +copy src\msgpack\unpack_define.h include\msgpack\ +copy src\msgpack\unpack_template.h include\msgpack\ +copy src\msgpack\sysdep.h include\msgpack\ +copy src\msgpack.h include\ +copy src\msgpack\sbuffer.h include\msgpack\ +copy src\msgpack\version.h include\msgpack\ +copy src\msgpack\vrefbuffer.h include\msgpack\ +copy src\msgpack\zbuffer.h include\msgpack\ +copy src\msgpack\pack.h include\msgpack\ +copy src\msgpack\unpack.h include\msgpack\ +copy src\msgpack\object.h include\msgpack\ +copy src\msgpack\zone.h include\msgpack\ +copy src\msgpack.hpp include\ +copy src\msgpack\sbuffer.hpp include\msgpack\ +copy src\msgpack\vrefbuffer.hpp include\msgpack\ +copy src\msgpack\zbuffer.hpp include\msgpack\ +copy src\msgpack\pack.hpp include\msgpack\ +copy src\msgpack\unpack.hpp include\msgpack\ +copy src\msgpack\object.hpp include\msgpack\ +copy src\msgpack\zone.hpp include\msgpack\ +copy src\msgpack\type.hpp include\msgpack\type\ +copy src\msgpack\type\bool.hpp include\msgpack\type\ +copy src\msgpack\type\float.hpp include\msgpack\type\ +copy src\msgpack\type\int.hpp include\msgpack\type\ +copy src\msgpack\type\list.hpp include\msgpack\type\ +copy src\msgpack\type\deque.hpp include\msgpack\type\ +copy src\msgpack\type\map.hpp include\msgpack\type\ +copy src\msgpack\type\nil.hpp include\msgpack\type\ +copy src\msgpack\type\pair.hpp include\msgpack\type\ +copy src\msgpack\type\raw.hpp include\msgpack\type\ +copy src\msgpack\type\set.hpp include\msgpack\type\ +copy src\msgpack\type\string.hpp include\msgpack\type\ +copy src\msgpack\type\vector.hpp include\msgpack\type\ +copy src\msgpack\type\tuple.hpp include\msgpack\type\ +copy src\msgpack\type\define.hpp include\msgpack\type\ +copy src\msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ +copy src\msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index 58047909..ed0daa47 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -157,7 +157,7 @@ UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" > + + + + + + + + @@ -247,23 +267,23 @@ UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" > diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index fc673856..a6910b4a 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -38,6 +38,7 @@ nobase_include_HEADERS = \ msgpack/sysdep.h \ msgpack.h \ msgpack/sbuffer.h \ + msgpack/version.h \ msgpack/vrefbuffer.h \ msgpack/zbuffer.h \ msgpack/pack.h \ @@ -49,7 +50,6 @@ if ENABLE_CXX nobase_include_HEADERS += \ msgpack.hpp \ msgpack/sbuffer.hpp \ - msgpack/version.h \ msgpack/vrefbuffer.hpp \ msgpack/zbuffer.hpp \ msgpack/pack.hpp \ From fe77251242f34e08f49f41fbbb2561e9278d8635 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 19:16:49 +0900 Subject: [PATCH 036/259] cpp: fixes missing dependency to generate version.h --- cpp/src/Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index a6910b4a..e12eb245 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -88,6 +88,8 @@ msgpack/version.h: msgpack/version.h.in Makefile.in -e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \ $< > $@ +version.c: msgpack/version.h + doxygen_c: cat ../Doxyfile > Doxyfile_c From 167e2475d89cb867428b9e7b5aac7f269fd95ecb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Jul 2010 23:30:15 +0900 Subject: [PATCH 037/259] cpp: generate version.h using AC_OUTPUT macro in ./configure --- cpp/configure.in | 5 ++++- cpp/src/Makefile.am | 9 --------- cpp/src/msgpack/version.h.in | 6 +++--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cpp/configure.in b/cpp/configure.in index dd04ed47..ab29501f 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -68,5 +68,8 @@ AC_SUBST(VERSION_MAJOR, $major) AC_SUBST(VERSION_MINOR, $minor) -AC_OUTPUT([Makefile src/Makefile test/Makefile]) +AC_OUTPUT([Makefile + src/Makefile + src/msgpack/version.h + test/Makefile]) diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index e12eb245..31096f0c 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -82,15 +82,6 @@ EXTRA_DIST = \ msgpack/type/tuple.hpp.erb -msgpack/version.h: msgpack/version.h.in Makefile.in - sed -e s/VERSION_UNDEFINED/$(VERSION)/ \ - -e s/VERSION_MAJOR_UNDEFINED/$(VERSION_MAJOR)/ \ - -e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \ - $< > $@ - -version.c: msgpack/version.h - - doxygen_c: cat ../Doxyfile > Doxyfile_c echo "FILE_PATTERNS = *.h" >> Doxyfile_c diff --git a/cpp/src/msgpack/version.h.in b/cpp/src/msgpack/version.h.in index af292d0d..f1feb331 100644 --- a/cpp/src/msgpack/version.h.in +++ b/cpp/src/msgpack/version.h.in @@ -27,9 +27,9 @@ const char* msgpack_version(void); int msgpack_version_major(void); int msgpack_version_minor(void); -#define MSGPACK_VERSION "VERSION_UNDEFINED" -#define MSGPACK_VERSION_MAJOR VERSION_MAJOR_UNDEFINED -#define MSGPACK_VERSION_MINOR VERSION_MINOR_UNDEFINED +#define MSGPACK_VERSION "@VERSION@" +#define MSGPACK_VERSION_MAJOR @VERSION_MAJOR@ +#define MSGPACK_VERSION_MINOR @VERSION_MINOR@ #ifdef __cplusplus From 45fb482ab42c7f47bf65313ade6808d0cfe3bca5 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 8 Jul 2010 23:36:18 +0900 Subject: [PATCH 038/259] erlang: added simple performance test. --- erlang/msgpack.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index aab07b72..ee993115 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -359,4 +359,9 @@ test_([Before|Rest])-> other_test()-> {more,1}=msgpack:unpack(<<>>). +benchmark_test()-> + Data=[test_data() || _ <- lists:seq(0, 10000)], + S=?debugTime(" serialize", msgpack:pack(Data)), + {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)). + -endif. From 485915c27a3ddf12e4ba4c9c0e27769869bb945c Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 8 Jul 2010 23:39:47 +0900 Subject: [PATCH 039/259] erlang: added simple performance test description. --- erlang/msgpack.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index ee993115..94fed86f 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -362,6 +362,7 @@ other_test()-> benchmark_test()-> Data=[test_data() || _ <- lists:seq(0, 10000)], S=?debugTime(" serialize", msgpack:pack(Data)), - {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)). + {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)), + ?debugFmt("for ~p KB test data.", [byte_size(S) div 1024]). -endif. From eab66a022e5b5fd9c4731ae8ba970b2146e27599 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 9 Jul 2010 01:04:09 +0900 Subject: [PATCH 040/259] erlang: added try-catch clause for easy error handling --- erlang/msgpack.erl | 220 +++++++++++++++++++++++---------------------- 1 file changed, 112 insertions(+), 108 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 94fed86f..d4fd0ba2 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -19,9 +19,8 @@ -author('kuenishi+msgpack@gmail.com'). %% tuples, atoms are not supported. lists, integers, double, and so on. -%% see http://msgpack.sourceforge.jp/spec for -%% supported formats. APIs are almost compatible -%% for C API (http://msgpack.sourceforge.jp/c:doc) +%% see http://msgpack.sourceforge.jp/spec for supported formats. +%% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). -export([pack_map/1]). @@ -30,51 +29,38 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem | badarg | no_code_matches. +-type reason() :: enomem | badarg | no_code_matches | undefined. -type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary(). % ===== external APIs ===== % --spec pack(Term::msgpack_term()) -> binary(). -pack(I) when is_integer(I) andalso I < 0 -> - pack_int_(I); -pack(I) when is_integer(I) -> - pack_uint_(I); -pack(F) when is_float(F) -> - pack_double(F); -pack(nil) -> - << 16#C0:8 >>; -pack(true) -> - << 16#C3:8 >>; -pack(false) -> - << 16#C2:8 >>; -pack(Bin) when is_binary(Bin) -> - pack_raw(Bin); -pack(List) when is_list(List) -> - pack_array(List); -pack({Map}) when is_list(Map) -> - pack_map(Map); -pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:to_list(Map)); -pack(_Other) -> - {error, undefined}. +-spec pack(Term::msgpack_term()) -> binary() | {error, reason()}. +pack(Term)-> + try + pack_(Term) + catch + error:Error when is_tuple(Error), element(1, Error) =:= error -> + Error; + throw:Exception -> + erlang:display(Exception), + {error, Exception} + end. % unpacking. % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | - {more, non_neg_integer()} | {more, undefined} | - {error, reason()}. -unpack(Bin) when not is_binary(Bin)-> - {error, badarg}; -unpack(Bin) when bit_size(Bin) >= 8 -> - unpack_(Bin); -unpack(<<>>)-> - {more, 1}; -unpack(_) -> - {more, undefined}. +-spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}. +unpack(Bin)-> + try + unpack_(Bin) + catch + error:Error when is_tuple(Error), element(1, Error) =:= error -> + Error; + throw:Exception -> + {error, Exception} + end. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -85,7 +71,7 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. --spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary(). +-spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary() | {error, badarg}. pack_map(M)-> case length(M) of Len when Len < 16 -> @@ -98,6 +84,31 @@ pack_map(M)-> % ===== internal APIs ===== % +% pack them all +-spec pack_(msgpack_term()) -> binary() | no_return(). +pack_(I) when is_integer(I) andalso I < 0 -> + pack_int_(I); +pack_(I) when is_integer(I) -> + pack_uint_(I); +pack_(F) when is_float(F) -> + pack_double(F); +pack_(nil) -> + << 16#C0:8 >>; +pack_(true) -> + << 16#C3:8 >>; +pack_(false) -> + << 16#C2:8 >>; +pack_(Bin) when is_binary(Bin) -> + pack_raw(Bin); +pack_(List) when is_list(List) -> + pack_array(List); +pack_({Map}) when is_list(Map) -> + pack_map(Map); +pack_(Map) when is_tuple(Map), element(1,Map)=:=dict -> + pack_map(dict:to_list(Map)); +pack_(_Other) -> + throw({error, undefined}). + % positive fixnum pack_uint_(N) when N < 128 -> << 2#0:1, N:7 >>; @@ -149,7 +160,7 @@ pack_raw(Bin) -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> end. -% list / tuple +% list pack_array(L) -> case length(L) of Len when Len < 16 -> @@ -159,43 +170,40 @@ pack_array(L) -> Len -> << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >> end. + pack_array_([], Acc) -> Acc; pack_array_([Head|Tail], Acc) -> - pack_array_(Tail, <>). + pack_array_(Tail, <>). -% FIXME! this should be without lists:reverse/1 -unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; -unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; -unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, undefined}; -unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> - case unpack(Bin) of - {more, _} -> {more, undefined}; - {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) - end. +% Users SHOULD NOT send too long list: this uses lists:reverse/1 +unpack_array_(Remain, 0, Acc) when is_binary(Remain)-> {lists:reverse(Acc), Remain}; +unpack_array_(<<>>, RestLen, _) when RestLen > 0 -> throw(short); +unpack_array_(Bin, RestLen, Acc) when is_binary(Bin)-> + {Term, Rest}=unpack_(Bin), + unpack_array_(Rest, RestLen-1, [Term|Acc]). pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> - pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). + pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>). -% FIXME! this should be without lists:reverse/1 --spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> - {more, non_neg_integer()} | { any(), binary()}. +% Users SHOULD NOT send too long list: this uses lists:reverse/1 +-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])-> + {[{msgpack_term(), msgpack_term()}], binary()} | no_return(). unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(<<>>, _, _ ) -> throw(short); unpack_map_(Bin, Len, Acc) -> - case unpack(Bin) of - {more, _} -> {more, undefined}; - {Key, Rest} -> - case unpack(Rest) of - {more, _} -> {more, undefined}; - {Value, Rest2} -> - unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]) - end - end. + {Key, Rest} = unpack_(Bin), + {Value, Rest2} = unpack_(Rest), + unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]). --spec unpack_(Payload::binary()) -> - {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. -unpack_(Binary)-> - case Binary of +% unpack then all +-spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return(). +unpack_(Bin) when not is_binary(Bin)-> + throw(badarg); +unpack_(<<>>)-> + throw(short); +unpack_(Bin) when bit_size(Bin) >= 8 -> + case Bin of % ATOMS <<16#C0, Rest/binary>> -> {nil, Rest}; <<16#C2, Rest/binary>> -> {false, Rest}; @@ -231,35 +239,36 @@ unpack_(Binary)-> <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map % Incomplete / invalid data - <<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)}; - <<16#CC>> -> {more, 1}; - <<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)}; - <<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)}; - <<16#D0>> -> {more, 1}; - <<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)}; - <<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)}; - <<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)}; - <<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)}; - <<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)}; - <<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)}; - <<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)}; - <<2#101:3, L:5, Rest/binary>> -> {more, L-byte_size(Rest)}; +% <<_:16/integer, _/binary>> + _ -> throw(short) +%% <<16#CA, _/binary>> -> {more, 4-byte_size(Rest)}; +%% <<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)}; +%% <<16#CC>> -> {more, 1}; +%% <<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)}; +%% <<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)}; +%% <<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)}; +%% <<16#D0>> -> {more, 1}; +%% <<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)}; +%% <<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)}; +%% <<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)}; +%% <<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)}; +%% <<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)}; +%% <<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)}; +%% <<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)}; +%% <<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)}; +%% <<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)}; +%% <<2#101:3, L:5, Rest/binary>> -> throw(short); % {more, L-byte_size(Rest)}; - <<>> -> {more, 1}; - <<2#101:3, _/binary>> -> {more, undefined}; - <> when F==16#C1; - F==16#C7; F==16#C8; F==16#C9; F==16#D5; - F==16#D6; F==16#D7; F==16#D8; F==16#D9-> - {error, {badarg, <>}}; - Other -> - {error, {badarg, Other}} +%% <<>> -> throw(short); % {more, 1}; +%% <<2#101:3, _/binary>> -> {more, undefined}; +%% <> when F==16#C1; +%% F==16#C7; F==16#C8; F==16#C9; F==16#D5; +%% F==16#D6; F==16#D7; F==16#D8; F==16#D9-> +%% throw({badarg, <>}); +% Other -> +% throw({unknown, Other}) end. - % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). @@ -268,9 +277,15 @@ compare_all([], [])-> ok; compare_all([], R)-> {toomuchrhs, R}; compare_all(L, [])-> {toomuchlhs, L}; compare_all([LH|LTL], [RH|RTL]) -> - LH=RH, + ?assertEqual(LH, RH), compare_all(LTL, RTL). +test_([]) -> 0; +test_([Term|Rest])-> + Pack = msgpack:pack(Term), + ?assertEqual({Term, <<>>}, msgpack:unpack( Pack )), + 1+test_(Rest). + test_data()-> [true, false, nil, 0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, @@ -303,12 +318,7 @@ test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, - case msgpack:unpack(Bin) of - {more, N} when not is_integer(N) -> - ?assertEqual(undefined, N); - {more, N} -> - ?assert( N < Len ) - end. + ?assertEqual({error,short}, msgpack:unpack(Bin)). partial_test()-> % error handling test. Term = lists:seq(0, 45), @@ -343,21 +353,15 @@ unknown_test()-> 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), + timer:sleep(1), receive {Port, {data, Data}}-> compare_all(Tests, msgpack:unpack_all(Data)) after 1024-> ?assert(false) end, port_close(Port). -test_([]) -> 0; -test_([Before|Rest])-> - Pack = msgpack:pack(Before), - {After, <<>>} = msgpack:unpack( Pack ), - ?assertEqual(Before, After), - 1+test_(Rest). - other_test()-> - {more,1}=msgpack:unpack(<<>>). + ?assertEqual({error,short},msgpack:unpack(<<>>)). benchmark_test()-> Data=[test_data() || _ <- lists:seq(0, 10000)], From e799082e5c4d39094c666da0f1c52ab2d6eb088c Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 9 Jul 2010 01:21:35 +0900 Subject: [PATCH 041/259] erlang: better test cases, except 'Broken pipe' --- erlang/msgpack.erl | 74 +++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d4fd0ba2..96ea407c 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -142,7 +142,7 @@ pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. % float : erlang's float is always IEEE 754 64bit format. -%pack_float(F) when is_float(F)-> +% pack_float(F) when is_float(F)-> % << 16#CA:8, F:32/big-float-unit:1 >>. % pack_double(F). % double @@ -198,10 +198,7 @@ unpack_map_(Bin, Len, Acc) -> % unpack then all -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return(). -unpack_(Bin) when not is_binary(Bin)-> - throw(badarg); -unpack_(<<>>)-> - throw(short); +unpack_(Bin) when not is_binary(Bin)-> throw(badarg); unpack_(Bin) when bit_size(Bin) >= 8 -> case Bin of % ATOMS @@ -239,43 +236,31 @@ unpack_(Bin) when bit_size(Bin) >= 8 -> <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map % Incomplete / invalid data -% <<_:16/integer, _/binary>> - _ -> throw(short) -%% <<16#CA, _/binary>> -> {more, 4-byte_size(Rest)}; -%% <<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)}; -%% <<16#CC>> -> {more, 1}; -%% <<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)}; -%% <<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)}; -%% <<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)}; -%% <<16#D0>> -> {more, 1}; -%% <<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)}; -%% <<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)}; -%% <<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)}; -%% <<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)}; -%% <<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)}; -%% <<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)}; -%% <<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)}; -%% <<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)}; -%% <<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)}; -%% <<2#101:3, L:5, Rest/binary>> -> throw(short); % {more, L-byte_size(Rest)}; - -%% <<>> -> throw(short); % {more, 1}; -%% <<2#101:3, _/binary>> -> {more, undefined}; -%% <> when F==16#C1; -%% F==16#C7; F==16#C8; F==16#C9; F==16#D5; -%% F==16#D6; F==16#D7; F==16#D8; F==16#D9-> -%% throw({badarg, <>}); -% Other -> -% throw({unknown, Other}) - end. + <> when F==16#CA; F==16#CB; F==16#CC; + F==16#CD; F==16#CE; F==16#CF; + F==16#D0; F==16#D1; F==16#D2; + F==16#D3; F==16#DA; F==16#DB; + F==16#DC; F==16#DD; F==16#DE; + F==16#DF -> + throw(short); + <> when F==16#C1; + F==16#C7; F==16#C8; F==16#C9; + F==16#D5; F==16#D6; F==16#D7; + F==16#D8; F==16#D9 -> + throw(badarg); + _ -> + throw(short) % or unknown/badarg? + end; +unpack_(<<>>)-> throw(short); +unpack_(<<2#101:3, _/binary>>) -> throw(short). % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). compare_all([], [])-> ok; -compare_all([], R)-> {toomuchrhs, R}; -compare_all(L, [])-> {toomuchlhs, L}; +compare_all([], R)-> {toomuchrhs, R}; +compare_all(L, [])-> {toomuchlhs, L}; compare_all([LH|LTL], [RH|RTL]) -> ?assertEqual(LH, RH), compare_all(LTL, RTL). @@ -305,9 +290,9 @@ basic_test()-> Passed = length(Tests). port_test()-> + Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), Tests = test_data(), {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])), - Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), true = port_command(Port, msgpack:pack(Tests) ), receive {Port, {data, Data}}-> {Tests, <<>>}=msgpack:unpack(Data) @@ -328,10 +313,7 @@ partial_test()-> % error handling test. long_test()-> Longer = lists:seq(0, 655), -% Longest = lists:seq(0,12345), - {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)), -% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). - ok. + {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)). map_test()-> Ints = lists:seq(0, 65), @@ -341,6 +323,7 @@ map_test()-> ok. unknown_test()-> + Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, <<"hogehoge">>, <<"243546rf7g68h798j">>, @@ -348,16 +331,13 @@ unknown_test()-> -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], - {[{1,2},{<<"hoge">>,nil}]}, + {[{1,2},{<<"hoge">>,nil}]}, % map -234, -50000, 42 ], - Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), - timer:sleep(1), receive - {Port, {data, Data}}-> - compare_all(Tests, msgpack:unpack_all(Data)) - after 1024-> ?assert(false) end, + {Port, {data, Data}}-> compare_all(Tests, msgpack:unpack_all(Data)) + after 1024-> ?assert(false) end, port_close(Port). other_test()-> From 64c36b7a8faac55d8dc80342f27929b5538c4307 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 13:06:57 +0200 Subject: [PATCH 042/259] Remove a couple of superfluous 'when' clauses. The when clause for unpack_/1 has been moved to unpack/1 so that it is performed only once. --- erlang/msgpack.erl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 1291d542..d1ba9cd0 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -52,7 +52,9 @@ pack(Term)-> % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. -spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}. -unpack(Bin)-> +unpack(Bin) when not is_binary(Bin) -> + {error, badarg}; +unpack(Bin) -> try unpack_(Bin) catch @@ -126,7 +128,7 @@ pack_uint_(N) -> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. % negative fixnum -pack_int_(N) when is_integer(N) , N >= -32-> +pack_int_(N) when N >= -32-> << 2#111:3, N:5 >>; % int 8 pack_int_(N) when N > -128 -> @@ -176,9 +178,9 @@ pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). % Users SHOULD NOT send too long list: this uses lists:reverse/1 -unpack_array_(Remain, 0, Acc) when is_binary(Remain)-> {lists:reverse(Acc), Remain}; -unpack_array_(<<>>, RestLen, _) when RestLen > 0 -> throw(short); -unpack_array_(Bin, RestLen, Acc) when is_binary(Bin)-> +unpack_array_(Remain, 0, Acc) -> {lists:reverse(Acc), Remain}; +unpack_array_(<<>>, RestLen, _) -> throw(short); +unpack_array_(Bin, RestLen, Acc) -> {Term, Rest}=unpack_(Bin), unpack_array_(Rest, RestLen-1, [Term|Acc]). @@ -198,8 +200,7 @@ unpack_map_(Bin, Len, Acc) -> % unpack then all -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return(). -unpack_(Bin) when not is_binary(Bin)-> throw(badarg); -unpack_(Bin) when bit_size(Bin) >= 8 -> +unpack_(Bin) -> case Bin of % ATOMS <<16#C0, Rest/binary>> -> {nil, Rest}; @@ -234,7 +235,7 @@ unpack_(Bin) when bit_size(Bin) >= 8 -> <<2#101:3, L:5, V:L/binary, Rest/binary>> -> {V, Rest}; % raw bytes <<2#1001:4, L:4, Rest/binary>> -> unpack_array_(Rest, L, []); % array <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map - + % Incomplete / invalid data <> when F==16#CA; F==16#CB; F==16#CC; F==16#CD; F==16#CE; F==16#CF; From 6abc120279ced47d899cd8596e4d48fc2171e2cf Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 13:29:47 +0200 Subject: [PATCH 043/259] erlang: Fix incomplete/invalid cases of unpack_/1 * fix list of invalid bytes was missing 3 possibilities (see type chart section of msgpack format spec) * fix matching of invalid bytes to look at 1 byte instead of 2 * simplify 'incomplete' case : anything that's not complete or invalid is by definition incomplete --- erlang/msgpack.erl | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d1ba9cd0..1c1eae91 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -236,24 +236,15 @@ unpack_(Bin) -> <<2#1001:4, L:4, Rest/binary>> -> unpack_array_(Rest, L, []); % array <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map -% Incomplete / invalid data - <> when F==16#CA; F==16#CB; F==16#CC; - F==16#CD; F==16#CE; F==16#CF; - F==16#D0; F==16#D1; F==16#D2; - F==16#D3; F==16#DA; F==16#DB; - F==16#DC; F==16#DD; F==16#DE; - F==16#DF -> - throw(short); - <> when F==16#C1; - F==16#C7; F==16#C8; F==16#C9; - F==16#D5; F==16#D6; F==16#D7; - F==16#D8; F==16#D9 -> +% Invalid data + <> when F==16#C1; + F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; + F==16#D4; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9 -> throw(badarg); +% Incomplete data (we've covered every complete/invalid case; anything left is incomplete) _ -> - throw(short) % or unknown/badarg? - end; -unpack_(<<>>)-> throw(short); -unpack_(<<2#101:3, _/binary>>) -> throw(short). + throw(short) + end. % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). From ba4a971bfaabe7da2159a634cd07977e06c61e3a Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 13:44:02 +0200 Subject: [PATCH 044/259] erlang: Remove unecessary 'throw(short)' clause for unpack_{array,map}_/1 Unecessary because unpack_/1 will throw it anyway. This does mean that we go a tiny bit deeper to find that we don't have enough data, but that should be a rare code path. Keep the main code path fast and the code clean. While at it, rename vars to match its sibling function and to avoid thinking that RestLen is a byte count (it's an item count). --- erlang/msgpack.erl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 1c1eae91..ff3eac74 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -178,11 +178,10 @@ pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). % Users SHOULD NOT send too long list: this uses lists:reverse/1 -unpack_array_(Remain, 0, Acc) -> {lists:reverse(Acc), Remain}; -unpack_array_(<<>>, RestLen, _) -> throw(short); -unpack_array_(Bin, RestLen, Acc) -> - {Term, Rest}=unpack_(Bin), - unpack_array_(Rest, RestLen-1, [Term|Acc]). +unpack_array_(Bin, 0, Acc) -> {lists:reverse(Acc), Bin}; +unpack_array_(Bin, Len, Acc) -> + {Term, Rest} = unpack_(Bin), + unpack_array_(Rest, Len-1, [Term|Acc]). pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> @@ -191,14 +190,13 @@ pack_map_([{Key,Value}|Tail], Acc) -> % Users SHOULD NOT send too long list: this uses lists:reverse/1 -spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])-> {[{msgpack_term(), msgpack_term()}], binary()} | no_return(). -unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; -unpack_map_(<<>>, _, _ ) -> throw(short); +unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; unpack_map_(Bin, Len, Acc) -> {Key, Rest} = unpack_(Bin), {Value, Rest2} = unpack_(Rest), - unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]). + unpack_map_(Rest2, Len-1, [{Key,Value}|Acc]). -% unpack then all +% unpack them all -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return(). unpack_(Bin) -> case Bin of From a4258505a99087df2397cf8d6cfbf194311a995c Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 9 Jul 2010 23:23:00 +0900 Subject: [PATCH 045/259] erlang: modified wrong testcase. --- erlang/msgpack.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index ff3eac74..e862cad1 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -282,10 +282,11 @@ basic_test()-> port_test()-> Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), Tests = test_data(), - {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])), - true = port_command(Port, msgpack:pack(Tests) ), + S=msgpack:pack([Tests]), + true = port_command(Port, S), + {[Tests],<<>>} = msgpack:unpack(S), receive - {Port, {data, Data}}-> {Tests, <<>>}=msgpack:unpack(Data) + {Port, {data, Data}}-> {[Tests], <<>>}=msgpack:unpack(Data) after 1024-> ?assert(false) end, port_close(Port). From 8a3f090684f0835958e60e6a5e3e81b4d8ff541a Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 17:36:36 +0200 Subject: [PATCH 046/259] erlang: Fix some existing specs and add a few other. dialyzer still complains about dict() and ?assert(false), but I don't think they're real issues. --- erlang/msgpack.erl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index ff3eac74..fb9a3e12 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -29,7 +29,7 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem | badarg | no_code_matches | undefined. +-type reason() :: badarg | short. -type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary(). @@ -73,7 +73,7 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. --spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary() | {error, badarg}. +-spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return(). pack_map(M)-> case length(M) of Len when Len < 16 -> @@ -109,8 +109,9 @@ pack_({Map}) when is_list(Map) -> pack_(Map) when is_tuple(Map), element(1,Map)=:=dict -> pack_map(dict:to_list(Map)); pack_(_Other) -> - throw({error, undefined}). + throw({error, badarg}). +-spec pack_uint_(non_neg_integer()) -> binary(). % positive fixnum pack_uint_(N) when N < 128 -> << 2#0:1, N:7 >>; @@ -127,6 +128,7 @@ pack_uint_(N) when N < 16#FFFFFFFF-> pack_uint_(N) -> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. +-spec pack_int_(integer()) -> binary(). % negative fixnum pack_int_(N) when N >= -32-> << 2#111:3, N:5 >>; @@ -143,6 +145,7 @@ pack_int_(N) when N > -16#FFFFFFFF -> pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. +-spec pack_double(float()) -> binary(). % float : erlang's float is always IEEE 754 64bit format. % pack_float(F) when is_float(F)-> % << 16#CA:8, F:32/big-float-unit:1 >>. @@ -151,6 +154,7 @@ pack_int_(N) -> pack_double(F) -> << 16#CB:8, F:64/big-float-unit:1 >>. +-spec pack_raw(binary()) -> binary(). % raw bytes pack_raw(Bin) -> case byte_size(Bin) of @@ -162,6 +166,7 @@ pack_raw(Bin) -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> end. +-spec pack_array([msgpack_term()]) -> binary() | no_return(). % list pack_array(L) -> case length(L) of @@ -178,6 +183,7 @@ pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). % Users SHOULD NOT send too long list: this uses lists:reverse/1 +-spec unpack_array_(binary(), non_neg_integer(), [msgpack_term()]) -> {[msgpack_term()], binary()} | no_return(). unpack_array_(Bin, 0, Acc) -> {lists:reverse(Acc), Bin}; unpack_array_(Bin, Len, Acc) -> {Term, Rest} = unpack_(Bin), @@ -188,8 +194,8 @@ pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>). % Users SHOULD NOT send too long list: this uses lists:reverse/1 --spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])-> - {[{msgpack_term(), msgpack_term()}], binary()} | no_return(). +-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}]) -> + {{[{msgpack_term(), msgpack_term()}]}, binary()} | no_return(). unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; unpack_map_(Bin, Len, Acc) -> {Key, Rest} = unpack_(Bin), @@ -197,7 +203,7 @@ unpack_map_(Bin, Len, Acc) -> unpack_map_(Rest2, Len-1, [{Key,Value}|Acc]). % unpack them all --spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return(). +-spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | no_return(). unpack_(Bin) -> case Bin of % ATOMS From 21992f1b9e43f3e0d2c3da8f1f8d94d9ee70b6fb Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 18:53:24 +0200 Subject: [PATCH 047/259] erlang: fix receiving from port_command in unit tests Ports can send data bit by bit; make sure we read all the port has to offer in one go. This should fix the "broken pipe" error we sometime got during testing. We did not previously check the return of compare_all/2, which is why the bug was not noticed. Incidentally, this change fixes dialyzer warnings too. --- erlang/msgpack.erl | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index fb9a3e12..58ad4143 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -261,6 +261,15 @@ compare_all([LH|LTL], [RH|RTL]) -> ?assertEqual(LH, RH), compare_all(LTL, RTL). +port_receive(Port) -> + port_receive(Port, <<>>). +port_receive(Port, Acc) -> + receive + {Port, {data, Data}} -> port_receive(Port, <>); + {Port, eof} -> Acc + after 1000 -> Acc + end. + test_([]) -> 0; test_([Term|Rest])-> Pack = msgpack:pack(Term), @@ -286,13 +295,12 @@ basic_test()-> Passed = length(Tests). port_test()-> - Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), Tests = test_data(), - {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])), - true = port_command(Port, msgpack:pack(Tests) ), - receive - {Port, {data, Data}}-> {Tests, <<>>}=msgpack:unpack(Data) - after 1024-> ?assert(false) end, + ?assertEqual({[Tests],<<>>}, msgpack:unpack(msgpack:pack([Tests]))), + + Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary, eof]), + true = port_command(Port, msgpack:pack(Tests)), + ?assertEqual({Tests, <<>>}, msgpack:unpack(port_receive(Port))), port_close(Port). test_p(Len,Term,OrigBin,Len) -> @@ -319,7 +327,7 @@ map_test()-> ok. unknown_test()-> - Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), + Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary, eof]), Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, <<"hogehoge">>, <<"243546rf7g68h798j">>, @@ -331,9 +339,7 @@ unknown_test()-> -234, -50000, 42 ], - receive - {Port, {data, Data}}-> compare_all(Tests, msgpack:unpack_all(Data)) - after 1024-> ?assert(false) end, + ?assertEqual(ok, compare_all(Tests, msgpack:unpack_all(port_receive(Port)))), port_close(Port). other_test()-> From 2c29377abf1997d3a55178ef0562aa38d255f0fe Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 20:30:17 +0200 Subject: [PATCH 048/259] erlang: s/short/incomplete/ and s/badarg/{badarg,Term}/ Nicer error returns. --- erlang/msgpack.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 58ad4143..102efed4 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -29,7 +29,7 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: badarg | short. +-type reason() :: {badarg,term()} | incomplete. -type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary(). @@ -43,7 +43,6 @@ pack(Term)-> error:Error when is_tuple(Error), element(1, Error) =:= error -> Error; throw:Exception -> - erlang:display(Exception), {error, Exception} end. @@ -52,9 +51,7 @@ pack(Term)-> % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. -spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}. -unpack(Bin) when not is_binary(Bin) -> - {error, badarg}; -unpack(Bin) -> +unpack(Bin) when is_binary(Bin) -> try unpack_(Bin) catch @@ -62,7 +59,9 @@ unpack(Bin) -> Error; throw:Exception -> {error, Exception} - end. + end; +unpack(Other) -> + {error, {badarg, Other}}. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -108,8 +107,8 @@ pack_({Map}) when is_list(Map) -> pack_map(Map); pack_(Map) when is_tuple(Map), element(1,Map)=:=dict -> pack_map(dict:to_list(Map)); -pack_(_Other) -> - throw({error, badarg}). +pack_(Other) -> + throw({error, {badarg, Other}}). -spec pack_uint_(non_neg_integer()) -> binary(). % positive fixnum @@ -241,13 +240,13 @@ unpack_(Bin) -> <<2#1000:4, L:4, Rest/binary>> -> unpack_map_(Rest, L, []); % map % Invalid data - <> when F==16#C1; + <> when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D4; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9 -> - throw(badarg); + throw({badarg, <>}); % Incomplete data (we've covered every complete/invalid case; anything left is incomplete) _ -> - throw(short) + throw(incomplete) end. % ===== test codes ===== % @@ -307,7 +306,7 @@ test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, - ?assertEqual({error,short}, msgpack:unpack(Bin)). + ?assertEqual({error,incomplete}, msgpack:unpack(Bin)). partial_test()-> % error handling test. Term = lists:seq(0, 45), @@ -343,7 +342,7 @@ unknown_test()-> port_close(Port). other_test()-> - ?assertEqual({error,short},msgpack:unpack(<<>>)). + ?assertEqual({error,incomplete},msgpack:unpack(<<>>)). benchmark_test()-> Data=[test_data() || _ <- lists:seq(0, 10000)], From 02c882bda39f65bdbe7be1b149b74f0f9a8283c6 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 20:34:38 +0200 Subject: [PATCH 049/259] erlang: Make pack_map/1 api private --- erlang/msgpack.erl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 102efed4..0e7fb5d4 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -23,7 +23,6 @@ %% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). --export([pack_map/1]). % compile: % erl> c(msgpack). @@ -72,16 +71,6 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. --spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return(). -pack_map(M)-> - case length(M) of - Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> - end. % ===== internal APIs ===== % @@ -188,6 +177,17 @@ unpack_array_(Bin, Len, Acc) -> {Term, Rest} = unpack_(Bin), unpack_array_(Rest, Len-1, [Term|Acc]). +-spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return(). +pack_map(M)-> + case length(M) of + Len when Len < 16 -> + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len when Len < 16#10000 -> % 65536 + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len -> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> + end. + pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>). From e944c1ee93083a054f318e42a16eff699d9f066d Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 9 Jul 2010 20:37:06 +0200 Subject: [PATCH 050/259] erlang: Only handle throw() in pack/1 and unpack/1 Rationale: We only use throw/1 for error handling, never erlang:error/1. Caller bugs will get a nice {error,...} return while library bugs will bubble up in all their uglyness; that's the proper way to do things in erlang. --- erlang/msgpack.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 0e7fb5d4..f23ac878 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -39,8 +39,6 @@ pack(Term)-> try pack_(Term) catch - error:Error when is_tuple(Error), element(1, Error) =:= error -> - Error; throw:Exception -> {error, Exception} end. @@ -54,8 +52,6 @@ unpack(Bin) when is_binary(Bin) -> try unpack_(Bin) catch - error:Error when is_tuple(Error), element(1, Error) =:= error -> - Error; throw:Exception -> {error, Exception} end; From e629e8784ff242f7c5e62066b0670f5d82afe4e2 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 12 Jul 2010 14:08:22 +0200 Subject: [PATCH 051/259] erlang: Improve documentation The doc is in edoc format, generated from the source as an html file. The makefile's default action now also generates the documentation. I ignored unpack_all/1 and pack(dict()) for now because their future is still uncertain. --- erlang/OMakefile | 7 +++-- erlang/msgpack.erl | 70 +++++++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/erlang/OMakefile b/erlang/OMakefile index 34c590f0..89b1c63f 100644 --- a/erlang/OMakefile +++ b/erlang/OMakefile @@ -30,13 +30,16 @@ # If so, define the subdirectory targets and uncomment this section. # -.DEFAULT: msgpack.beam +.DEFAULT: msgpack.beam msgpack.html msgpack.beam: msgpack.erl erlc $< +msgpack.html: msgpack.erl + erl -noshell -run edoc_run file $< + test: msgpack.beam erl -noshell -s msgpack test -s init stop clean: - -rm *.beam + -rm -f *.beam *.html diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index f23ac878..aa9851da 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -15,26 +15,47 @@ %% See the License for the specific language governing permissions and %% limitations under the License. + +%% @doc MessagePack codec for Erlang. +%% +%% APIs are almost compatible with C API +%% except for buffering functions (both copying and zero-copying), which are unavailable. +%% +%% +%% +%% +%% +%% +%% +%% +%% +%% +%% +%%
Equivalence between Erlang and Msgpack type :
erlang msgpack
integer() pos_fixnum/neg_fixnum/uint8/uint16/uint32/uint64/int8/int16/int32/int64
float() float/double
nil nil
boolean() boolean
binary() fix_raw/raw16/raw32
list() fix_array/array16/array32
{proplist()} fix_map/map16/map32
+%% @end + -module(msgpack). -author('kuenishi+msgpack@gmail.com'). -%% tuples, atoms are not supported. lists, integers, double, and so on. -%% see http://msgpack.sourceforge.jp/spec for supported formats. -%% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc) -%% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). -% compile: -% erl> c(msgpack). -% erl> S = . -% erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: {badarg,term()} | incomplete. +% @type msgpack_term() = [msgpack_term()] +% | {[{msgpack_term(),msgpack_term()}]} +% | integer() | float() | binary(). +% Erlang representation of msgpack data. -type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary(). -% ===== external APIs ===== % --spec pack(Term::msgpack_term()) -> binary() | {error, reason()}. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% external APIs +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% @doc Encode an erlang term into an msgpack binary. +% Returns {error, {badarg, term()}} if the input is illegal. +% @spec pack(Term::msgpack_term()) -> binary() | {error, {badarg, term()}} +-spec pack(Term::msgpack_term()) -> binary() | {error, {badarg, term()}}. pack(Term)-> try pack_(Term) @@ -43,11 +64,12 @@ pack(Term)-> {error, Exception} end. -% unpacking. -% if failed in decoding and not end, get more data -% and feed more Bin into this function. -% TODO: error case for imcomplete format when short for any type formats. --spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}. +% @doc Decode an msgpack binary into an erlang term. +% It only decodes the first msgpack packet contained in the binary; the rest is returned as is. +% Returns {error, {badarg, term()}} if the input is corrupted. +% Returns {error, incomplete} if the input is not a full msgpack packet (caller should gather more data and try again). +% @spec unpack(Bin::binary()) -> {msgpack_term(), binary()} | {error, incomplete} | {error, {badarg, term()}} +-spec unpack(Bin::binary()) -> {msgpack_term(), binary()} | {error, incomplete} | {error, {badarg, term()}}. unpack(Bin) when is_binary(Bin) -> try unpack_(Bin) @@ -58,7 +80,7 @@ unpack(Bin) when is_binary(Bin) -> unpack(Other) -> {error, {badarg, Other}}. --spec unpack_all( binary() ) -> [msgpack_term()]. +-spec unpack_all(binary()) -> [msgpack_term()]. unpack_all(Data)-> case unpack(Data) of { Term, Binary } when bit_size(Binary) =:= 0 -> @@ -68,7 +90,9 @@ unpack_all(Data)-> end. -% ===== internal APIs ===== % +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% internal APIs +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % pack them all -spec pack_(msgpack_term()) -> binary() | no_return(). @@ -95,6 +119,7 @@ pack_(Map) when is_tuple(Map), element(1,Map)=:=dict -> pack_(Other) -> throw({error, {badarg, Other}}). + -spec pack_uint_(non_neg_integer()) -> binary(). % positive fixnum pack_uint_(N) when N < 128 -> @@ -129,6 +154,7 @@ pack_int_(N) when N > -16#FFFFFFFF -> pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. + -spec pack_double(float()) -> binary(). % float : erlang's float is always IEEE 754 64bit format. % pack_float(F) when is_float(F)-> @@ -138,6 +164,7 @@ pack_int_(N) -> pack_double(F) -> << 16#CB:8, F:64/big-float-unit:1 >>. + -spec pack_raw(binary()) -> binary(). % raw bytes pack_raw(Bin) -> @@ -150,6 +177,7 @@ pack_raw(Bin) -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> end. + -spec pack_array([msgpack_term()]) -> binary() | no_return(). % list pack_array(L) -> @@ -173,6 +201,7 @@ unpack_array_(Bin, Len, Acc) -> {Term, Rest} = unpack_(Bin), unpack_array_(Rest, Len-1, [Term|Acc]). + -spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return(). pack_map(M)-> case length(M) of @@ -245,7 +274,10 @@ unpack_(Bin) -> throw(incomplete) end. -% ===== test codes ===== % + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% unit tests +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). From ca0c844f32038329b21f92ce42c62618057ecc02 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 14 Jul 2010 09:58:05 +0900 Subject: [PATCH 052/259] clearly specified this distribution requires requires C99. --- perl/Makefile.PL | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 58ab7c79..e9f9618a 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,4 +1,6 @@ use inc::Module::Install; +use Config; + name 'Data-MessagePack'; all_from 'lib/Data/MessagePack.pm'; readme_from('lib/Data/MessagePack.pm'); @@ -11,6 +13,8 @@ tests 't/*.t'; recursive_author_tests('xt'); use_ppport 3.19; +requires_c99(); # msgpack C library requires C99. + clean_files qw{ *.stackdump *.gcov *.gcda *.gcno From 9ac69337e89305a36ae3a3d3a9d89272a7453452 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 14 Jul 2010 09:58:28 +0900 Subject: [PATCH 053/259] perl: bump up version to 0.13! --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index b03a9d0c..f32d5f49 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.13 + + - clearly specify requires_c99(), because msgpack C header requires C99. + 0.12 - PERL_NO_GET_CONTEXT makes horrible dTHXs. remove it. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index dcc713d5..ee07d35f 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.12'; +our $VERSION = '0.13'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 331bf0af21cecfec1025e0f69fb4b3ffad4129fe Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Jul 2010 17:02:04 +0900 Subject: [PATCH 054/259] cpp: type::raw_ref::str(), operator==, operator!=, operator< and operator> are now const --- cpp/src/msgpack/type/raw.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/msgpack/type/raw.hpp b/cpp/src/msgpack/type/raw.hpp index 21d9a0d9..87d188f6 100644 --- a/cpp/src/msgpack/type/raw.hpp +++ b/cpp/src/msgpack/type/raw.hpp @@ -33,25 +33,25 @@ struct raw_ref { uint32_t size; const char* ptr; - std::string str() { return std::string(ptr, size); } + std::string str() const { return std::string(ptr, size); } - bool operator== (const raw_ref& x) + bool operator== (const raw_ref& x) const { return size == x.size && memcmp(ptr, x.ptr, size) == 0; } - bool operator!= (const raw_ref& x) + bool operator!= (const raw_ref& x) const { return !(*this != x); } - bool operator< (const raw_ref& x) + bool operator< (const raw_ref& x) const { if(size == x.size) { return memcmp(ptr, x.ptr, size) < 0; } else { return size < x.size; } } - bool operator> (const raw_ref& x) + bool operator> (const raw_ref& x) const { if(size == x.size) { return memcmp(ptr, x.ptr, size) > 0; } else { return size > x.size; } From f5453d38ec96b55e95ac745472d9b55087fc1d2f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Jul 2010 17:06:16 +0900 Subject: [PATCH 055/259] cpp: version 0.5.2 --- cpp/ChangeLog | 5 +++++ cpp/configure.in | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/ChangeLog b/cpp/ChangeLog index 3277c139..504ac4ba 100644 --- a/cpp/ChangeLog +++ b/cpp/ChangeLog @@ -1,4 +1,9 @@ +2010-07-14 version 0.5.2: + + * type::raw::str(), operator==, operator!=, operator< and operator> are now const + * generates version.h using AC_OUTPUT macro in ./configure + 2010-07-06 version 0.5.1: * Add msgpack_vrefbuffer_new and msgpack_vrefbuffer_free diff --git a/cpp/configure.in b/cpp/configure.in index ab29501f..93174da2 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,6 +1,6 @@ AC_INIT(src/object.cpp) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.5.1) +AM_INIT_AUTOMAKE(msgpack, 0.5.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 7b152640d962434192496e4d96788622d4e90886 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sun, 18 Jul 2010 23:40:25 +0900 Subject: [PATCH 056/259] erlang: 'edoc' document generation --- erlang/.gitignore | 1 + erlang/OMakefile | 7 +++++-- erlang/edoc/.gitignore | 4 ++++ erlang/msgpack.erl | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 erlang/edoc/.gitignore diff --git a/erlang/.gitignore b/erlang/.gitignore index 7d6d3247..0f7faad4 100644 --- a/erlang/.gitignore +++ b/erlang/.gitignore @@ -2,3 +2,4 @@ MANIFEST *.beam .omakedb* *.omc +*~ \ No newline at end of file diff --git a/erlang/OMakefile b/erlang/OMakefile index 89b1c63f..d213444a 100644 --- a/erlang/OMakefile +++ b/erlang/OMakefile @@ -30,10 +30,10 @@ # If so, define the subdirectory targets and uncomment this section. # -.DEFAULT: msgpack.beam msgpack.html +.DEFAULT: msgpack.beam msgpack.beam: msgpack.erl - erlc $< + erlc -Wall +debug_info $< msgpack.html: msgpack.erl erl -noshell -run edoc_run file $< @@ -41,5 +41,8 @@ msgpack.html: msgpack.erl test: msgpack.beam erl -noshell -s msgpack test -s init stop +edoc: msgpack.erl + erl -noshell -eval 'ok=edoc:files(["msgpack.erl"], [{dir, "edoc"}]).' -s init stop + clean: -rm -f *.beam *.html diff --git a/erlang/edoc/.gitignore b/erlang/edoc/.gitignore new file mode 100644 index 00000000..97f4246c --- /dev/null +++ b/erlang/edoc/.gitignore @@ -0,0 +1,4 @@ +*.html +*.css +*.png +edoc-info diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index aa9851da..a6974834 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -18,7 +18,7 @@ %% @doc MessagePack codec for Erlang. %% -%% APIs are almost compatible with C API +%% APIs are almost compatible with C API %% except for buffering functions (both copying and zero-copying), which are unavailable. %% %% From dad7a03d1975c28574fb768e10a63355832a0474 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sun, 18 Jul 2010 23:42:23 +0900 Subject: [PATCH 057/259] erlang: stopped support for dict() type. --- erlang/msgpack.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index a6974834..13bb8e11 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -114,8 +114,6 @@ pack_(List) when is_list(List) -> pack_array(List); pack_({Map}) when is_list(Map) -> pack_map(Map); -pack_(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:to_list(Map)); pack_(Other) -> throw({error, {badarg, Other}}). From 6cabad19d5a576e3e44e069c0a2b107bd33d30ef Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sun, 18 Jul 2010 23:48:20 +0900 Subject: [PATCH 058/259] erlang: unpack_all/1 improve, error handling added. --- erlang/msgpack.erl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 13bb8e11..d4a0bc8a 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -80,16 +80,22 @@ unpack(Bin) when is_binary(Bin) -> unpack(Other) -> {error, {badarg, Other}}. --spec unpack_all(binary()) -> [msgpack_term()]. +-spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}}. unpack_all(Data)-> - case unpack(Data) of - { Term, Binary } when bit_size(Binary) =:= 0 -> + try + unpack_all_(Data) + catch + throw:Exception -> + {error, Exception} + end. +unpack_all_(Data)-> + case unpack_(Data) of + { Term, <<>> } -> [Term]; { Term, Binary } when is_binary(Binary) -> - [Term|unpack_all(Binary)] + [Term|unpack_all_(Binary)] end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % internal APIs %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% From 8a3ac6d9bd374acc6b134d330bf7aeb906fa1a80 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sun, 18 Jul 2010 23:50:29 +0900 Subject: [PATCH 059/259] erlang: omake menus added. --- erlang/OMakefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/erlang/OMakefile b/erlang/OMakefile index d213444a..21079400 100644 --- a/erlang/OMakefile +++ b/erlang/OMakefile @@ -22,7 +22,7 @@ # Phony targets are scoped, so you probably want to declare them first. # -.PHONY: all clean test #install +.PHONY: all clean test edoc dialyzer #install ######################################################################## # Subdirectories. @@ -44,5 +44,8 @@ test: msgpack.beam edoc: msgpack.erl erl -noshell -eval 'ok=edoc:files(["msgpack.erl"], [{dir, "edoc"}]).' -s init stop +dialyzer: msgpack.erl + dialyzer --src $< + clean: -rm -f *.beam *.html From dcbcf5842f58b7ead3f524ac027576ce0c55e9ab Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sun, 18 Jul 2010 23:55:07 +0900 Subject: [PATCH 060/259] erlang: msgpack:unpack_all/1 doc. --- erlang/msgpack.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d4a0bc8a..b54874d5 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -80,6 +80,11 @@ unpack(Bin) when is_binary(Bin) -> unpack(Other) -> {error, {badarg, Other}}. +% @doc Decode an msgpack binary into an erlang terms. +% It only decodes ALL msgpack packets contained in the binary. No packets should not remain. +% Returns {error, {badarg, term()}} if the input is corrupted. +% Returns {error, incomplete} if the input is not a full msgpack packet (caller should gather more data and try again). +% @spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}} -spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}}. unpack_all(Data)-> try From 227c168b65be72e5e3a843af38d9382b59fc858a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 24 Jul 2010 18:07:22 +0900 Subject: [PATCH 061/259] java: fixes fatal offset calculation bugs on BufferedUnpackerIMPL.unpackInt() --- java/src/main/java/org/msgpack/BufferedUnpackerImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index cc6604d4..f4ed35b7 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -103,7 +103,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { case 0xcc: // unsigned int 8 more(2); advance(2); - return (int)((short)buffer[offset+1] & 0xff); + return (int)((short)(buffer[offset-1]) & 0xff); case 0xcd: // unsigned int 16 more(3); castBuffer.rewind(); @@ -137,7 +137,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { case 0xd0: // signed int 8 more(2); advance(2); - return (int)buffer[offset+1]; + return (int)buffer[offset-1]; case 0xd1: // signed int 16 more(3); castBuffer.rewind(); @@ -178,7 +178,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { case 0xcc: // unsigned int 8 more(2); advance(2); - return (long)((short)buffer[offset+1] & 0xff); + return (long)((short)(buffer[offset-1]) & 0xff); case 0xcd: // unsigned int 16 more(3); castBuffer.rewind(); @@ -207,7 +207,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { case 0xd0: // signed int 8 more(2); advance(2); - return (long)buffer[offset+1]; + return (long)buffer[offset-1]; case 0xd1: // signed int 16 more(3); castBuffer.rewind(); From 2aef495d62d19b2f1721989225700942ea71e582 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 24 Jul 2010 18:08:00 +0900 Subject: [PATCH 062/259] java: adds MessagePackObject class --- .../java/org/msgpack/MessagePackObject.java | 130 ++++++++++++++++++ .../java/org/msgpack/object/ArrayType.java | 48 +++++++ .../msgpack/object/BigIntegerTypeIMPL.java | 92 +++++++++++++ .../java/org/msgpack/object/BooleanType.java | 39 ++++++ .../org/msgpack/object/DoubleTypeIMPL.java | 71 ++++++++++ .../java/org/msgpack/object/FloatType.java | 28 ++++ .../org/msgpack/object/FloatTypeIMPL.java | 70 ++++++++++ .../java/org/msgpack/object/IntegerType.java | 49 +++++++ .../msgpack/object/LongIntegerTypeIMPL.java | 89 ++++++++++++ .../main/java/org/msgpack/object/MapType.java | 48 +++++++ .../main/java/org/msgpack/object/NilType.java | 28 ++++ .../msgpack/object/ShortIntegerTypeIMPL.java | 91 ++++++++++++ 12 files changed, 783 insertions(+) create mode 100644 java/src/main/java/org/msgpack/MessagePackObject.java create mode 100644 java/src/main/java/org/msgpack/object/ArrayType.java create mode 100644 java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java create mode 100644 java/src/main/java/org/msgpack/object/BooleanType.java create mode 100644 java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java create mode 100644 java/src/main/java/org/msgpack/object/FloatType.java create mode 100644 java/src/main/java/org/msgpack/object/FloatTypeIMPL.java create mode 100644 java/src/main/java/org/msgpack/object/IntegerType.java create mode 100644 java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java create mode 100644 java/src/main/java/org/msgpack/object/MapType.java create mode 100644 java/src/main/java/org/msgpack/object/NilType.java create mode 100644 java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java new file mode 100644 index 00000000..aba27e48 --- /dev/null +++ b/java/src/main/java/org/msgpack/MessagePackObject.java @@ -0,0 +1,130 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.util.List; +import java.util.Set; +import java.util.Map; +import java.math.BigInteger; + +public abstract class MessagePackObject { + public boolean isNull() { + return false; + } + + public boolean isBooleanType() { + return false; + } + + public boolean isIntegerType() { + return false; + } + + public boolean isFloatType() { + return false; + } + + public boolean isArrayType() { + return false; + } + + public boolean isMapType() { + return false; + } + + public boolean isRawType() { + return false; + } + + public boolean asBoolean() { + throw new MessageTypeException("type error"); + } + + public byte asByte() { + throw new MessageTypeException("type error"); + } + + public short asShort() { + throw new MessageTypeException("type error"); + } + + public int asInt() { + throw new MessageTypeException("type error"); + } + + public long asLong() { + throw new MessageTypeException("type error"); + } + + public BigInteger asBigInteger() { + throw new MessageTypeException("type error"); + } + + public float asFloat() { + throw new MessageTypeException("type error"); + } + + public double asDouble() { + throw new MessageTypeException("type error"); + } + + public byte[] asByteArray() { + throw new MessageTypeException("type error"); + } + + public String asString() { + throw new MessageTypeException("type error"); + } + + public MessagePackObject[] asArray() { + throw new MessageTypeException("type error"); + } + + public List asList() { + throw new MessageTypeException("type error"); + } + + public Map asMap() { + throw new MessageTypeException("type error"); + } + + public byte byteValue() { + throw new MessageTypeException("type error"); + } + + public short shortValue() { + throw new MessageTypeException("type error"); + } + + public int intValue() { + throw new MessageTypeException("type error"); + } + + public long longValue() { + throw new MessageTypeException("type error"); + } + + public float floatValue() { + throw new MessageTypeException("type error"); + } + + public double doubleValue() { + throw new MessageTypeException("type error"); + } +} + diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java new file mode 100644 index 00000000..06e9b167 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/ArrayType.java @@ -0,0 +1,48 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.util.List; +import java.util.Set; +import java.util.Map; +import java.util.Arrays; +import org.msgpack.*; + +public class ArrayType extends MessagePackObject { + private MessagePackObject[] array; + + public ArrayType(MessagePackObject[] array) { + this.array = array; + } + + @Override + public boolean isArrayType() { + return true; + } + + @Override + public MessagePackObject[] asArray() { + return array; + } + + @Override + public List asList() { + return Arrays.asList(array); + } +} + diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java new file mode 100644 index 00000000..1d638c9d --- /dev/null +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -0,0 +1,92 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +class BigIntegerTypeIMPL extends IntegerType { + private BigInteger value; + + BigIntegerTypeIMPL(BigInteger vlaue) { + this.value = value; + } + + @Override + public byte asByte() { + if(value.compareTo(BigInteger.valueOf((long)Byte.MAX_VALUE)) > 0) { + throw new MessageTypeException("type error"); + } + return value.byteValue(); + } + + @Override + public short asShort() { + if(value.compareTo(BigInteger.valueOf((long)Short.MAX_VALUE)) > 0) { + throw new MessageTypeException("type error"); + } + return value.shortValue(); + } + + @Override + public int asInt() { + if(value.compareTo(BigInteger.valueOf((long)Integer.MAX_VALUE)) > 0) { + throw new MessageTypeException("type error"); + } + return value.intValue(); + } + + @Override + public long asLong() { + if(value.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + throw new MessageTypeException("type error"); + } + return value.longValue(); + } + + @Override + public byte byteValue() { + return value.byteValue(); + } + + @Override + public short shortValue() { + return value.shortValue(); + } + + @Override + public int intValue() { + return value.intValue(); + } + + @Override + public long longValue() { + return value.longValue(); + } + + @Override + public float floatValue() { + return value.floatValue(); + } + + @Override + public double doubleValue() { + return value.doubleValue(); + } +} + diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java new file mode 100644 index 00000000..c9e84b61 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/BooleanType.java @@ -0,0 +1,39 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import org.msgpack.*; + +public class BooleanType extends MessagePackObject { + private boolean value; + + public BooleanType(boolean value) { + this.value = value; + } + + @Override + public boolean isBooleanType() { + return false; + } + + @Override + public boolean asBoolean() { + return value; + } +} + diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java new file mode 100644 index 00000000..0e33d5b3 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java @@ -0,0 +1,71 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +class DoubleTypeIMPL extends FloatType { + private double value; + + public DoubleTypeIMPL(double vlaue) { + this.value = value; + } + + @Override + public float asFloat() { + // FIXME check overflow, underflow? + return (float)value; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public byte byteValue() { + return (byte)value; + } + + @Override + public short shortValue() { + return (short)value; + } + + @Override + public int intValue() { + return (int)value; + } + + @Override + public long longValue() { + return (long)value; + } + + @Override + public float floatValue() { + return (float)value; + } + + @Override + public double doubleValue() { + return (double)value; + } +} + diff --git a/java/src/main/java/org/msgpack/object/FloatType.java b/java/src/main/java/org/msgpack/object/FloatType.java new file mode 100644 index 00000000..2782ddaa --- /dev/null +++ b/java/src/main/java/org/msgpack/object/FloatType.java @@ -0,0 +1,28 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import org.msgpack.*; + +public abstract class FloatType extends MessagePackObject { + @Override + public boolean isFloatType() { + return true; + } +} + diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java new file mode 100644 index 00000000..75a50701 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java @@ -0,0 +1,70 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +class FloatTypeIMPL extends FloatType { + private float value; + + public FloatTypeIMPL(float vlaue) { + this.value = value; + } + + @Override + public float asFloat() { + return value; + } + + @Override + public double asDouble() { + return (double)value; + } + + @Override + public byte byteValue() { + return (byte)value; + } + + @Override + public short shortValue() { + return (short)value; + } + + @Override + public int intValue() { + return (int)value; + } + + @Override + public long longValue() { + return (long)value; + } + + @Override + public float floatValue() { + return (float)value; + } + + @Override + public double doubleValue() { + return (double)value; + } +} + diff --git a/java/src/main/java/org/msgpack/object/IntegerType.java b/java/src/main/java/org/msgpack/object/IntegerType.java new file mode 100644 index 00000000..d6a9b541 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/IntegerType.java @@ -0,0 +1,49 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +public abstract class IntegerType extends MessagePackObject { + public static IntegerType create(byte value) { + return new ShortIntegerTypeIMPL((int)value); + } + + public static IntegerType create(short value) { + return new ShortIntegerTypeIMPL((int)value); + } + + public static IntegerType create(int value) { + return new ShortIntegerTypeIMPL(value); + } + + public static IntegerType create(long value) { + return new LongIntegerTypeIMPL(value); + } + + public static IntegerType create(BigInteger value) { + return new BigIntegerTypeIMPL(value); + } + + @Override + public boolean isIntegerType() { + return true; + } +} + diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java new file mode 100644 index 00000000..c914e91d --- /dev/null +++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +class LongIntegerTypeIMPL extends IntegerType { + private long value; + + LongIntegerTypeIMPL(long value) { + this.value = value; + } + + @Override + public byte asByte() { + if(value > (long)Byte.MAX_VALUE) { + throw new MessageTypeException("type error"); + } + return (byte)value; + } + + @Override + public short asShort() { + if(value > (long)Short.MAX_VALUE) { + throw new MessageTypeException("type error"); + } + return (short)value; + } + + @Override + public int asInt() { + if(value > (long)Integer.MAX_VALUE) { + throw new MessageTypeException("type error"); + } + return (int)value; + } + + @Override + public long asLong() { + return value; + } + + @Override + public byte byteValue() { + return (byte)value; + } + + @Override + public short shortValue() { + return (short)value; + } + + @Override + public int intValue() { + return (int)value; + } + + @Override + public long longValue() { + return (long)value; + } + + @Override + public float floatValue() { + return (float)value; + } + + @Override + public double doubleValue() { + return (double)value; + } +} + diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java new file mode 100644 index 00000000..dbd145b8 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/MapType.java @@ -0,0 +1,48 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.util.HashMap; +import java.util.Map; +import org.msgpack.*; + +public class MapType extends MessagePackObject { + MessagePackObject[] map; + + public MapType(MessagePackObject[] map) { + this.map = map; + } + + @Override + public boolean isMapType() { + return false; + } + + @Override + public Map asMap() { + HashMap m = new HashMap(map.length / 2); + int i = 0; + while(i < map.length) { + MessagePackObject k = map[i++]; + MessagePackObject v = map[i++]; + m.put(k, v); + } + return m; + } +} + diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java new file mode 100644 index 00000000..c36ff05c --- /dev/null +++ b/java/src/main/java/org/msgpack/object/NilType.java @@ -0,0 +1,28 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import org.msgpack.*; + +public class NilType extends MessagePackObject { + @Override + public boolean isNull() { + return true; + } +} + diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java new file mode 100644 index 00000000..a725950d --- /dev/null +++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java @@ -0,0 +1,91 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import java.math.BigInteger; +import org.msgpack.*; + +class ShortIntegerTypeIMPL extends IntegerType { + private int value; + + ShortIntegerTypeIMPL(int value) { + this.value = value; + } + + @Override + public byte asByte() { + if(value > (int)Byte.MAX_VALUE) { + throw new MessageTypeException("type error"); + } + return (byte)value; + } + + @Override + public short asShort() { + if(value > (int)Short.MAX_VALUE) { + throw new MessageTypeException("type error"); + } + return (short)value; + } + + @Override + public int asInt() { + return value; + } + + @Override + public long asLong() { + return value; + } + + @Override + public BigInteger asBigInteger() { + return BigInteger.valueOf((long)value); + } + + @Override + public byte byteValue() { + return (byte)value; + } + + @Override + public short shortValue() { + return (short)value; + } + + @Override + public int intValue() { + return (int)value; + } + + @Override + public long longValue() { + return (long)value; + } + + @Override + public float floatValue() { + return (float)value; + } + + @Override + public double doubleValue() { + return (double)value; + } +} + From 02ae247536ec5570c3a150de8283ef399aaf82eb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 24 Jul 2010 18:20:00 +0900 Subject: [PATCH 063/259] java: adds MessagePackObject class 2 --- .../msgpack/object/BigIntegerTypeIMPL.java | 2 +- .../java/org/msgpack/object/BooleanType.java | 2 +- .../org/msgpack/object/DoubleTypeIMPL.java | 2 +- .../java/org/msgpack/object/FloatType.java | 8 ++++ .../org/msgpack/object/FloatTypeIMPL.java | 2 +- .../java/org/msgpack/object/IntegerType.java | 10 ++-- .../main/java/org/msgpack/object/MapType.java | 4 +- .../main/java/org/msgpack/object/RawType.java | 48 +++++++++++++++++++ 8 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 java/src/main/java/org/msgpack/object/RawType.java diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java index 1d638c9d..1ebb83df 100644 --- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -23,7 +23,7 @@ import org.msgpack.*; class BigIntegerTypeIMPL extends IntegerType { private BigInteger value; - BigIntegerTypeIMPL(BigInteger vlaue) { + BigIntegerTypeIMPL(BigInteger value) { this.value = value; } diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java index c9e84b61..d272b6f0 100644 --- a/java/src/main/java/org/msgpack/object/BooleanType.java +++ b/java/src/main/java/org/msgpack/object/BooleanType.java @@ -28,7 +28,7 @@ public class BooleanType extends MessagePackObject { @Override public boolean isBooleanType() { - return false; + return true; } @Override diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java index 0e33d5b3..8bbc52ac 100644 --- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java @@ -23,7 +23,7 @@ import org.msgpack.*; class DoubleTypeIMPL extends FloatType { private double value; - public DoubleTypeIMPL(double vlaue) { + public DoubleTypeIMPL(double value) { this.value = value; } diff --git a/java/src/main/java/org/msgpack/object/FloatType.java b/java/src/main/java/org/msgpack/object/FloatType.java index 2782ddaa..514efd5b 100644 --- a/java/src/main/java/org/msgpack/object/FloatType.java +++ b/java/src/main/java/org/msgpack/object/FloatType.java @@ -24,5 +24,13 @@ public abstract class FloatType extends MessagePackObject { public boolean isFloatType() { return true; } + + public static FloatType create(float value) { + return new FloatTypeIMPL(value); + } + + public static FloatType create(double value) { + return new DoubleTypeIMPL(value); + } } diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java index 75a50701..8821640e 100644 --- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java @@ -23,7 +23,7 @@ import org.msgpack.*; class FloatTypeIMPL extends FloatType { private float value; - public FloatTypeIMPL(float vlaue) { + public FloatTypeIMPL(float value) { this.value = value; } diff --git a/java/src/main/java/org/msgpack/object/IntegerType.java b/java/src/main/java/org/msgpack/object/IntegerType.java index d6a9b541..43357e8e 100644 --- a/java/src/main/java/org/msgpack/object/IntegerType.java +++ b/java/src/main/java/org/msgpack/object/IntegerType.java @@ -21,6 +21,11 @@ import java.math.BigInteger; import org.msgpack.*; public abstract class IntegerType extends MessagePackObject { + @Override + public boolean isIntegerType() { + return true; + } + public static IntegerType create(byte value) { return new ShortIntegerTypeIMPL((int)value); } @@ -40,10 +45,5 @@ public abstract class IntegerType extends MessagePackObject { public static IntegerType create(BigInteger value) { return new BigIntegerTypeIMPL(value); } - - @Override - public boolean isIntegerType() { - return true; - } } diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java index dbd145b8..359ebe6a 100644 --- a/java/src/main/java/org/msgpack/object/MapType.java +++ b/java/src/main/java/org/msgpack/object/MapType.java @@ -22,7 +22,7 @@ import java.util.Map; import org.msgpack.*; public class MapType extends MessagePackObject { - MessagePackObject[] map; + private MessagePackObject[] map; public MapType(MessagePackObject[] map) { this.map = map; @@ -30,7 +30,7 @@ public class MapType extends MessagePackObject { @Override public boolean isMapType() { - return false; + return true; } @Override diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java new file mode 100644 index 00000000..107ba278 --- /dev/null +++ b/java/src/main/java/org/msgpack/object/RawType.java @@ -0,0 +1,48 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.object; + +import org.msgpack.*; + +class RawType extends MessagePackObject { + private byte[] bytes; + + public RawType(byte[] bytes) { + this.bytes = bytes; + } + + @Override + public boolean isRawType() { + return true; + } + + @Override + public byte[] asByteArray() { + return bytes; + } + + @Override + public String asString() { + try { + return new String(bytes, "UTF-8"); + } catch (Exception e) { + throw new MessageTypeException("type error"); + } + } +} + From cd83388f8b035102f260f2ab45a551233dd593da Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 27 Jul 2010 08:59:09 +0900 Subject: [PATCH 064/259] java: fixed repository location. msgpack.sourceforge.net => msgpack.org --- java/pom.xml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 9b74b4cf..959d2df4 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -7,7 +7,7 @@ MessagePack for Java MessagePack for Java - http://msgpack.sourceforge.net/ + http://msgpack.org/ @@ -97,29 +97,26 @@ - msgpack.sourceforge.net + msgpack.org MessagePack Maven2 Repository - http://msgpack.sourceforge.net/maven2 - - - msgpack.sourceforge.net - MessagePack Maven2 Snapshot Repository - http://msgpack.sourceforge.net/maven2-snapshot + http://msgpack.org/maven2 false - shell.sourceforge.net - Repository at sourceforge.net - scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/ + msgpack.org + Repository at msgpack.org + + file:///Users/otakazuki/soft/website/maven2/ true - shell.sourceforge.net - Repository Name - scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2-snapshot/ + msgpack.org + Repository at msgpack.org + + file:///Users/otakazuki/soft/website/maven2/ From cba47b635a5a3bab2050fb4e0d099c23f8455867 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 27 Jul 2010 09:50:57 +0900 Subject: [PATCH 065/259] java: changed deploy path to ./target/website/maven2 directory. --- java/pom.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 959d2df4..44806d51 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -108,15 +108,13 @@ false msgpack.org Repository at msgpack.org - - file:///Users/otakazuki/soft/website/maven2/ + file://${project.build.directory}/website/maven2/ true msgpack.org Repository at msgpack.org - - file:///Users/otakazuki/soft/website/maven2/ + file://${project.build.directory}/website/maven2/ From 6c91b862c9da56ce0914d430fea2cb388e32d5d5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 28 Jul 2010 01:17:20 +0900 Subject: [PATCH 066/259] java: MessagePackObject implements Cloneable and MessagePackable interfaces --- .../java/org/msgpack/MessagePackObject.java | 4 ++- java/src/main/java/org/msgpack/Packer.java | 22 +++++++++++++++ .../java/org/msgpack/object/ArrayType.java | 28 +++++++++++++++++-- .../msgpack/object/BigIntegerTypeIMPL.java | 19 +++++++++++++ .../java/org/msgpack/object/BooleanType.java | 19 +++++++++++++ .../org/msgpack/object/DoubleTypeIMPL.java | 19 +++++++++++++ .../org/msgpack/object/FloatTypeIMPL.java | 19 +++++++++++++ .../msgpack/object/LongIntegerTypeIMPL.java | 19 +++++++++++++ .../main/java/org/msgpack/object/MapType.java | 27 ++++++++++++++++++ .../main/java/org/msgpack/object/NilType.java | 19 +++++++++++++ .../main/java/org/msgpack/object/RawType.java | 23 ++++++++++++++- .../msgpack/object/ShortIntegerTypeIMPL.java | 19 +++++++++++++ 12 files changed, 233 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java index aba27e48..b1a6faba 100644 --- a/java/src/main/java/org/msgpack/MessagePackObject.java +++ b/java/src/main/java/org/msgpack/MessagePackObject.java @@ -22,7 +22,7 @@ import java.util.Set; import java.util.Map; import java.math.BigInteger; -public abstract class MessagePackObject { +public abstract class MessagePackObject implements Cloneable, MessagePackable { public boolean isNull() { return false; } @@ -126,5 +126,7 @@ public abstract class MessagePackObject { public double doubleValue() { throw new MessageTypeException("type error"); } + + abstract public Object clone(); } diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index dd510f34..fbf7e35c 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.math.BigInteger; /** * Packer enables you to serialize objects into OutputStream. @@ -194,6 +195,27 @@ public class Packer { return this; } + public Packer packBigInteger(BigInteger d) throws IOException { + if(d.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { + return packLong(d.longValue()); + } else if(d.bitLength() <= 64) { + castBytes[0] = (byte)0xcf; + byte[] barray = d.toByteArray(); + castBytes[1] = barray[0]; + castBytes[2] = barray[1]; + castBytes[3] = barray[2]; + castBytes[4] = barray[3]; + castBytes[5] = barray[4]; + castBytes[6] = barray[5]; + castBytes[7] = barray[6]; + castBytes[8] = barray[7]; + out.write(castBytes); + return this; + } else { + throw new MessageTypeException("can't BigInteger larger than 0xffffffffffffffff"); + } + } + public Packer packFloat(float d) throws IOException { castBytes[0] = (byte)0xca; castBuffer.putFloat(1, d); diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java index 06e9b167..694f53f0 100644 --- a/java/src/main/java/org/msgpack/object/ArrayType.java +++ b/java/src/main/java/org/msgpack/object/ArrayType.java @@ -18,9 +18,8 @@ package org.msgpack.object; import java.util.List; -import java.util.Set; -import java.util.Map; import java.util.Arrays; +import java.io.IOException; import org.msgpack.*; public class ArrayType extends MessagePackObject { @@ -44,5 +43,30 @@ public class ArrayType extends MessagePackObject { public List asList() { return Arrays.asList(array); } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packArray(array.length); + for(int i=0; i < array.length; i++) { + array[i].messagePack(pk); + } + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return Arrays.equals(((ArrayType)obj).array, array); + } + + @Override + public Object clone() { + MessagePackObject[] copy = new MessagePackObject[array.length]; + for(int i=0; i < array.length; i++) { + copy[i] = (MessagePackObject)array[i].clone(); + } + return copy; + } } diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java index 1ebb83df..f7c73ae1 100644 --- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -18,6 +18,7 @@ package org.msgpack.object; import java.math.BigInteger; +import java.io.IOException; import org.msgpack.*; class BigIntegerTypeIMPL extends IntegerType { @@ -88,5 +89,23 @@ class BigIntegerTypeIMPL extends IntegerType { public double doubleValue() { return value.doubleValue(); } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packBigInteger(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((BigIntegerTypeIMPL)obj).value.equals(value); + } + + @Override + public Object clone() { + return new BigIntegerTypeIMPL(value); + } } diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java index d272b6f0..c6e4f300 100644 --- a/java/src/main/java/org/msgpack/object/BooleanType.java +++ b/java/src/main/java/org/msgpack/object/BooleanType.java @@ -17,6 +17,7 @@ // package org.msgpack.object; +import java.io.IOException; import org.msgpack.*; public class BooleanType extends MessagePackObject { @@ -35,5 +36,23 @@ public class BooleanType extends MessagePackObject { public boolean asBoolean() { return value; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packBoolean(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((BooleanType)obj).value == value; + } + + @Override + public Object clone() { + return new BooleanType(value); + } } diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java index 8bbc52ac..dafe5405 100644 --- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java @@ -18,6 +18,7 @@ package org.msgpack.object; import java.math.BigInteger; +import java.io.IOException; import org.msgpack.*; class DoubleTypeIMPL extends FloatType { @@ -67,5 +68,23 @@ class DoubleTypeIMPL extends FloatType { public double doubleValue() { return (double)value; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packDouble(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((DoubleTypeIMPL)obj).value == value; + } + + @Override + public Object clone() { + return new DoubleTypeIMPL(value); + } } diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java index 8821640e..234b2ad9 100644 --- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java @@ -18,6 +18,7 @@ package org.msgpack.object; import java.math.BigInteger; +import java.io.IOException; import org.msgpack.*; class FloatTypeIMPL extends FloatType { @@ -66,5 +67,23 @@ class FloatTypeIMPL extends FloatType { public double doubleValue() { return (double)value; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packFloat(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((FloatTypeIMPL)obj).value == value; + } + + @Override + public Object clone() { + return new FloatTypeIMPL(value); + } } diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java index c914e91d..0ce22a25 100644 --- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java @@ -18,6 +18,7 @@ package org.msgpack.object; import java.math.BigInteger; +import java.io.IOException; import org.msgpack.*; class LongIntegerTypeIMPL extends IntegerType { @@ -85,5 +86,23 @@ class LongIntegerTypeIMPL extends IntegerType { public double doubleValue() { return (double)value; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packLong(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((LongIntegerTypeIMPL)obj).value == value; + } + + @Override + public Object clone() { + return new LongIntegerTypeIMPL(value); + } } diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java index 359ebe6a..d456e78f 100644 --- a/java/src/main/java/org/msgpack/object/MapType.java +++ b/java/src/main/java/org/msgpack/object/MapType.java @@ -19,6 +19,8 @@ package org.msgpack.object; import java.util.HashMap; import java.util.Map; +import java.util.Arrays; +import java.io.IOException; import org.msgpack.*; public class MapType extends MessagePackObject { @@ -44,5 +46,30 @@ public class MapType extends MessagePackObject { } return m; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packMap(map.length / 2); + for(int i=0; i < map.length; i++) { + map[i].messagePack(pk); + } + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return Arrays.equals(((MapType)obj).map, map); + } + + @Override + public Object clone() { + MessagePackObject[] copy = new MessagePackObject[map.length]; + for(int i=0; i < map.length; i++) { + copy[i] = (MessagePackObject)map[i].clone(); + } + return copy; + } } diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java index c36ff05c..7a463764 100644 --- a/java/src/main/java/org/msgpack/object/NilType.java +++ b/java/src/main/java/org/msgpack/object/NilType.java @@ -17,6 +17,7 @@ // package org.msgpack.object; +import java.io.IOException; import org.msgpack.*; public class NilType extends MessagePackObject { @@ -24,5 +25,23 @@ public class NilType extends MessagePackObject { public boolean isNull() { return true; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packNil(); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return true; + } + + @Override + public Object clone() { + return new NilType(); + } } diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java index 107ba278..18a419d1 100644 --- a/java/src/main/java/org/msgpack/object/RawType.java +++ b/java/src/main/java/org/msgpack/object/RawType.java @@ -17,9 +17,11 @@ // package org.msgpack.object; +import java.util.Arrays; +import java.io.IOException; import org.msgpack.*; -class RawType extends MessagePackObject { +public class RawType extends MessagePackObject { private byte[] bytes; public RawType(byte[] bytes) { @@ -44,5 +46,24 @@ class RawType extends MessagePackObject { throw new MessageTypeException("type error"); } } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packRaw(bytes.length); + pk.packRawBody(bytes); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((RawType)obj).bytes.equals(bytes); + } + + @Override + public Object clone() { + return new RawType((byte[])bytes.clone()); + } } diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java index a725950d..83a4daf6 100644 --- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java @@ -18,6 +18,7 @@ package org.msgpack.object; import java.math.BigInteger; +import java.io.IOException; import org.msgpack.*; class ShortIntegerTypeIMPL extends IntegerType { @@ -87,5 +88,23 @@ class ShortIntegerTypeIMPL extends IntegerType { public double doubleValue() { return (double)value; } + + @Override + public void messagePack(Packer pk) throws IOException { + pk.packInt(value); + } + + @Override + public boolean equals(Object obj) { + if(obj.getClass() != getClass()) { + return false; + } + return ((ShortIntegerTypeIMPL)obj).value == value; + } + + @Override + public Object clone() { + return new ShortIntegerTypeIMPL(value); + } } From d3bb37d113575e8b36f44c06ac9eeb6b406aa298 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 10 Aug 2010 14:11:25 +0900 Subject: [PATCH 067/259] java: fixes MapSchema --- java/src/main/java/org/msgpack/schema/MapSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java index 339a5c29..2e09af31 100644 --- a/java/src/main/java/org/msgpack/schema/MapSchema.java +++ b/java/src/main/java/org/msgpack/schema/MapSchema.java @@ -72,7 +72,7 @@ public class MapSchema extends Schema implements IMapSchema { dest.put((K)keySchema.convert(e.getKey()), (V)valueSchema.convert(e.getValue())); } - return (Map)d; + return dest; } @Override From 057f73a73e0c3ddabe92f1cd2c394fe4afa13514 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 10 Aug 2010 14:11:44 +0900 Subject: [PATCH 068/259] java: implements MessagePackObject::hashCode() --- java/src/main/java/org/msgpack/object/ArrayType.java | 5 +++++ .../main/java/org/msgpack/object/BigIntegerTypeIMPL.java | 5 +++++ java/src/main/java/org/msgpack/object/BooleanType.java | 9 +++++++++ .../src/main/java/org/msgpack/object/DoubleTypeIMPL.java | 6 ++++++ java/src/main/java/org/msgpack/object/FloatTypeIMPL.java | 5 +++++ .../java/org/msgpack/object/LongIntegerTypeIMPL.java | 5 +++++ java/src/main/java/org/msgpack/object/MapType.java | 5 +++++ java/src/main/java/org/msgpack/object/NilType.java | 5 +++++ java/src/main/java/org/msgpack/object/RawType.java | 5 +++++ .../java/org/msgpack/object/ShortIntegerTypeIMPL.java | 5 +++++ 10 files changed, 55 insertions(+) diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java index 694f53f0..350ce32e 100644 --- a/java/src/main/java/org/msgpack/object/ArrayType.java +++ b/java/src/main/java/org/msgpack/object/ArrayType.java @@ -60,6 +60,11 @@ public class ArrayType extends MessagePackObject { return Arrays.equals(((ArrayType)obj).array, array); } + @Override + public int hashCode() { + return array.hashCode(); + } + @Override public Object clone() { MessagePackObject[] copy = new MessagePackObject[array.length]; diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java index f7c73ae1..fd517e70 100644 --- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -103,6 +103,11 @@ class BigIntegerTypeIMPL extends IntegerType { return ((BigIntegerTypeIMPL)obj).value.equals(value); } + @Override + public int hashCode() { + return value.hashCode(); + } + @Override public Object clone() { return new BigIntegerTypeIMPL(value); diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java index c6e4f300..1d12c1ce 100644 --- a/java/src/main/java/org/msgpack/object/BooleanType.java +++ b/java/src/main/java/org/msgpack/object/BooleanType.java @@ -50,6 +50,15 @@ public class BooleanType extends MessagePackObject { return ((BooleanType)obj).value == value; } + @Override + public int hashCode() { + if(value) { + return 1231; + } else { + return 1237; + } + } + @Override public Object clone() { return new BooleanType(value); diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java index dafe5405..b47a7096 100644 --- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java @@ -82,6 +82,12 @@ class DoubleTypeIMPL extends FloatType { return ((DoubleTypeIMPL)obj).value == value; } + @Override + public int hashCode() { + long v = Double.doubleToLongBits(value); + return (int)(v^(v>>>32)); + } + @Override public Object clone() { return new DoubleTypeIMPL(value); diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java index 234b2ad9..1d799611 100644 --- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java @@ -81,6 +81,11 @@ class FloatTypeIMPL extends FloatType { return ((FloatTypeIMPL)obj).value == value; } + @Override + public int hashCode() { + return Float.floatToIntBits(value); + } + @Override public Object clone() { return new FloatTypeIMPL(value); diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java index 0ce22a25..940ab6fa 100644 --- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java @@ -100,6 +100,11 @@ class LongIntegerTypeIMPL extends IntegerType { return ((LongIntegerTypeIMPL)obj).value == value; } + @Override + public int hashCode() { + return (int)(value^(value>>>32)); + } + @Override public Object clone() { return new LongIntegerTypeIMPL(value); diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java index d456e78f..619d3887 100644 --- a/java/src/main/java/org/msgpack/object/MapType.java +++ b/java/src/main/java/org/msgpack/object/MapType.java @@ -63,6 +63,11 @@ public class MapType extends MessagePackObject { return Arrays.equals(((MapType)obj).map, map); } + @Override + public int hashCode() { + return map.hashCode(); + } + @Override public Object clone() { MessagePackObject[] copy = new MessagePackObject[map.length]; diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java index 7a463764..ece62f0a 100644 --- a/java/src/main/java/org/msgpack/object/NilType.java +++ b/java/src/main/java/org/msgpack/object/NilType.java @@ -39,6 +39,11 @@ public class NilType extends MessagePackObject { return true; } + @Override + public int hashCode() { + return 0; + } + @Override public Object clone() { return new NilType(); diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java index 18a419d1..3a394862 100644 --- a/java/src/main/java/org/msgpack/object/RawType.java +++ b/java/src/main/java/org/msgpack/object/RawType.java @@ -61,6 +61,11 @@ public class RawType extends MessagePackObject { return ((RawType)obj).bytes.equals(bytes); } + @Override + public int hashCode() { + return bytes.hashCode(); + } + @Override public Object clone() { return new RawType((byte[])bytes.clone()); diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java index 83a4daf6..60e92b8e 100644 --- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java @@ -102,6 +102,11 @@ class ShortIntegerTypeIMPL extends IntegerType { return ((ShortIntegerTypeIMPL)obj).value == value; } + @Override + public int hashCode() { + return value; + } + @Override public Object clone() { return new ShortIntegerTypeIMPL(value); From 8c67087a154da0e7cdc32c0b676d2956ce1d0f47 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 16:32:42 +0900 Subject: [PATCH 069/259] java: adds MessagePackObject.bigIntegerValue(), asBigInteger() and equals() --- .../main/java/org/msgpack/MessagePackObject.java | 4 ++++ .../org/msgpack/object/BigIntegerTypeIMPL.java | 15 +++++++++++++++ .../java/org/msgpack/object/DoubleTypeIMPL.java | 5 +++++ .../org/msgpack/object/LongIntegerTypeIMPL.java | 15 +++++++++++++++ .../org/msgpack/object/ShortIntegerTypeIMPL.java | 10 ++++++++++ 5 files changed, 49 insertions(+) diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java index b1a6faba..6181f7a3 100644 --- a/java/src/main/java/org/msgpack/MessagePackObject.java +++ b/java/src/main/java/org/msgpack/MessagePackObject.java @@ -119,6 +119,10 @@ public abstract class MessagePackObject implements Cloneable, MessagePackable { throw new MessageTypeException("type error"); } + public BigInteger bigIntegerValue() { + throw new MessageTypeException("type error"); + } + public float floatValue() { throw new MessageTypeException("type error"); } diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java index fd517e70..7b060eef 100644 --- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -60,6 +60,11 @@ class BigIntegerTypeIMPL extends IntegerType { return value.longValue(); } + @Override + public BigInteger asBigInteger() { + return value; + } + @Override public byte byteValue() { return value.byteValue(); @@ -80,6 +85,11 @@ class BigIntegerTypeIMPL extends IntegerType { return value.longValue(); } + @Override + public BigInteger bigIntegerValue() { + return value; + } + @Override public float floatValue() { return value.floatValue(); @@ -98,6 +108,11 @@ class BigIntegerTypeIMPL extends IntegerType { @Override public boolean equals(Object obj) { if(obj.getClass() != getClass()) { + if(obj.getClass() == ShortIntegerTypeIMPL.class) { + return BigInteger.valueOf((long)((ShortIntegerTypeIMPL)obj).shortValue()).equals(value); + } else if(obj.getClass() == LongIntegerTypeIMPL.class) { + return BigInteger.valueOf(((LongIntegerTypeIMPL)obj).longValue()).equals(value); + } return false; } return ((BigIntegerTypeIMPL)obj).value.equals(value); diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java index b47a7096..fd38089e 100644 --- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java @@ -59,6 +59,11 @@ class DoubleTypeIMPL extends FloatType { return (long)value; } + @Override + public BigInteger bigIntegerValue() { + return BigInteger.valueOf((long)value); + } + @Override public float floatValue() { return (float)value; diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java index 940ab6fa..3928a299 100644 --- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java @@ -57,6 +57,11 @@ class LongIntegerTypeIMPL extends IntegerType { return value; } + @Override + public BigInteger asBigInteger() { + return BigInteger.valueOf(value); + } + @Override public byte byteValue() { return (byte)value; @@ -77,6 +82,11 @@ class LongIntegerTypeIMPL extends IntegerType { return (long)value; } + @Override + public BigInteger bigIntegerValue() { + return BigInteger.valueOf(value); + } + @Override public float floatValue() { return (float)value; @@ -95,6 +105,11 @@ class LongIntegerTypeIMPL extends IntegerType { @Override public boolean equals(Object obj) { if(obj.getClass() != getClass()) { + if(obj.getClass() == ShortIntegerTypeIMPL.class) { + return value == ((ShortIntegerTypeIMPL)obj).longValue(); + } else if(obj.getClass() == BigIntegerTypeIMPL.class) { + return (long)value == ((BigIntegerTypeIMPL)obj).longValue(); + } return false; } return ((LongIntegerTypeIMPL)obj).value == value; diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java index 60e92b8e..dbed4263 100644 --- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java @@ -79,6 +79,11 @@ class ShortIntegerTypeIMPL extends IntegerType { return (long)value; } + @Override + public BigInteger bigIntegerValue() { + return BigInteger.valueOf((long)value); + } + @Override public float floatValue() { return (float)value; @@ -97,6 +102,11 @@ class ShortIntegerTypeIMPL extends IntegerType { @Override public boolean equals(Object obj) { if(obj.getClass() != getClass()) { + if(obj.getClass() == LongIntegerTypeIMPL.class) { + return (long)value == ((LongIntegerTypeIMPL)obj).longValue(); + } else if(obj.getClass() == BigIntegerTypeIMPL.class) { + return ((BigIntegerTypeIMPL)obj).bigIntegerValue().equals(BigInteger.valueOf((long)value)); + } return false; } return ((ShortIntegerTypeIMPL)obj).value == value; From 8b79e6d3c72a02f4dc039799e3cd370c06e966b0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 18:10:30 +0900 Subject: [PATCH 070/259] java: uses MessagePackObject instead of Object for type of deserialized objects --- .../org/msgpack/BufferedUnpackerImpl.java | 8 +- .../org/msgpack/MessageTypeException.java | 10 - java/src/main/java/org/msgpack/Packer.java | 6 - java/src/main/java/org/msgpack/Schema.java | 78 ---- .../main/java/org/msgpack/UnpackIterator.java | 4 +- .../main/java/org/msgpack/UnpackResult.java | 6 +- java/src/main/java/org/msgpack/Unpacker.java | 30 +- .../main/java/org/msgpack/UnpackerImpl.java | 132 ++---- .../org/msgpack/schema/BooleanSchema.java | 64 --- .../org/msgpack/schema/ByteArraySchema.java | 97 ---- .../java/org/msgpack/schema/ByteSchema.java | 96 ---- .../org/msgpack/schema/ClassGenerator.java | 244 ---------- .../java/org/msgpack/schema/ClassSchema.java | 91 ---- .../java/org/msgpack/schema/DoubleSchema.java | 74 --- .../java/org/msgpack/schema/FieldSchema.java | 43 -- .../java/org/msgpack/schema/FloatSchema.java | 74 --- .../msgpack/schema/GenericClassSchema.java | 87 ---- .../org/msgpack/schema/GenericSchema.java | 129 ------ .../java/org/msgpack/schema/IArraySchema.java | 26 -- .../java/org/msgpack/schema/IMapSchema.java | 27 -- .../java/org/msgpack/schema/IntSchema.java | 96 ---- .../java/org/msgpack/schema/ListSchema.java | 111 ----- .../java/org/msgpack/schema/LongSchema.java | 80 ---- .../java/org/msgpack/schema/MapSchema.java | 106 ----- .../msgpack/schema/ReflectionClassSchema.java | 64 --- .../org/msgpack/schema/SSchemaParser.java | 264 ----------- .../java/org/msgpack/schema/SetSchema.java | 115 ----- .../java/org/msgpack/schema/ShortSchema.java | 93 ---- .../msgpack/schema/SpecificClassSchema.java | 122 ----- .../java/org/msgpack/schema/StringSchema.java | 102 ---- .../test/java/org/msgpack/TestPackUnpack.java | 435 +++++++++--------- 31 files changed, 270 insertions(+), 2644 deletions(-) delete mode 100644 java/src/main/java/org/msgpack/Schema.java delete mode 100644 java/src/main/java/org/msgpack/schema/BooleanSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ByteArraySchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ByteSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ClassGenerator.java delete mode 100644 java/src/main/java/org/msgpack/schema/ClassSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/DoubleSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/FieldSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/FloatSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/GenericClassSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/GenericSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/IArraySchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/IMapSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/IntSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ListSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/LongSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/MapSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/SSchemaParser.java delete mode 100644 java/src/main/java/org/msgpack/schema/SetSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/ShortSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/SpecificClassSchema.java delete mode 100644 java/src/main/java/org/msgpack/schema/StringSchema.java diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index f4ed35b7..9496238f 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -47,7 +47,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { offset = noffset; } while(!super.isFinished()); - Object obj = super.getData(); + MessagePackObject obj = super.getData(); super.reset(); result.done(obj); @@ -198,7 +198,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { { long o = castBuffer.getLong(0); if(o < 0) { - // FIXME + // FIXME unpackBigInteger throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported"); } advance(9); @@ -231,6 +231,8 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } + // FIXME unpackBigInteger + final float unpackFloat() throws IOException, MessageTypeException { more(1); int b = buffer[offset]; @@ -414,7 +416,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { return s; } - final Object unpackObject() throws IOException { + final MessagePackObject unpackObject() throws IOException { UnpackResult result = new UnpackResult(); if(!next(result)) { super.reset(); diff --git a/java/src/main/java/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java index 0fa37b76..698ef6dd 100644 --- a/java/src/main/java/org/msgpack/MessageTypeException.java +++ b/java/src/main/java/org/msgpack/MessageTypeException.java @@ -23,15 +23,5 @@ public class MessageTypeException extends RuntimeException { public MessageTypeException(String s) { super(s); } - - public static MessageTypeException invalidConvert(Object from, Schema to) { - return new MessageTypeException(from.getClass().getName()+" cannot be convert to "+to.getExpression()); - } - - /* FIXME - public static MessageTypeException schemaMismatch(Schema to) { - return new MessageTypeException("schema mismatch "+to.getExpression()); - } - */ } diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index fbf7e35c..98af3d64 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -308,12 +308,6 @@ public class Packer { } - public Packer packWithSchema(Object o, Schema s) throws IOException { - s.pack(this, o); - return this; - } - - public Packer packString(String s) throws IOException { byte[] b = ((String)s).getBytes("UTF-8"); packRaw(b.length); diff --git a/java/src/main/java/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java deleted file mode 100644 index 25e10f90..00000000 --- a/java/src/main/java/org/msgpack/Schema.java +++ /dev/null @@ -1,78 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack; - -import java.io.Writer; -import java.io.IOException; -import org.msgpack.schema.SSchemaParser; -//import org.msgpack.schema.ClassGenerator; - -public abstract class Schema { - public Schema() { } - - public abstract String getClassName(); - public abstract String getExpression(); - - public static Schema parse(String source) { - return SSchemaParser.parse(source); - } - - public static Schema load(String source) { - return SSchemaParser.load(source); - } - - public abstract void pack(Packer pk, Object obj) throws IOException; - public abstract Object convert(Object obj) throws MessageTypeException; - - public Object createFromNil() { - return null; - } - - public Object createFromBoolean(boolean v) { - throw new MessageTypeException("type error"); - } - - public Object createFromByte(byte v) { - throw new MessageTypeException("type error"); - } - - public Object createFromShort(short v) { - throw new MessageTypeException("type error"); - } - - public Object createFromInt(int v) { - throw new MessageTypeException("type error"); - } - - public Object createFromLong(long v) { - throw new MessageTypeException("type error"); - } - - public Object createFromFloat(float v) { - throw new MessageTypeException("type error"); - } - - public Object createFromDouble(double v) { - throw new MessageTypeException("type error"); - } - - public Object createFromRaw(byte[] b, int offset, int length) { - throw new MessageTypeException("type error"); - } -} - diff --git a/java/src/main/java/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java index f17e2290..8c778b67 100644 --- a/java/src/main/java/org/msgpack/UnpackIterator.java +++ b/java/src/main/java/org/msgpack/UnpackIterator.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; -public class UnpackIterator extends UnpackResult implements Iterator { +public class UnpackIterator extends UnpackResult implements Iterator { private Unpacker pac; UnpackIterator(Unpacker pac) { @@ -38,7 +38,7 @@ public class UnpackIterator extends UnpackResult implements Iterator { } } - public Object next() { + public MessagePackObject next() { if(!finished && !hasNext()) { throw new NoSuchElementException(); } diff --git a/java/src/main/java/org/msgpack/UnpackResult.java b/java/src/main/java/org/msgpack/UnpackResult.java index cec18a16..bb981c12 100644 --- a/java/src/main/java/org/msgpack/UnpackResult.java +++ b/java/src/main/java/org/msgpack/UnpackResult.java @@ -19,13 +19,13 @@ package org.msgpack; public class UnpackResult { protected boolean finished = false; - protected Object data = null; + protected MessagePackObject data = null; public boolean isFinished() { return finished; } - public Object getData() { + public MessagePackObject getData() { return data; } @@ -34,7 +34,7 @@ public class UnpackResult { data = null; } - void done(Object obj) { + void done(MessagePackObject obj) { finished = true; data = obj; } diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 32bab646..3a952438 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -26,8 +26,8 @@ import java.nio.ByteBuffer; /** * Unpacker enables you to deserialize objects from stream. * - * Unpacker provides Buffered API, Unbuffered API, Schema API - * and Direct Conversion API. + * Unpacker provides Buffered API, Unbuffered API and + * Direct Conversion API. * * Buffered API uses the internal buffer of the Unpacker. * Following code uses Buffered API with an InputStream: @@ -39,7 +39,7 @@ import java.nio.ByteBuffer; * UnpackResult result = pac.next(); * * // use an iterator. - * for(Object obj : pac) { + * for(MessagePackObject obj : pac) { * // use MessageConvertable interface to convert the * // the generic object to the specific type. * } @@ -56,7 +56,7 @@ import java.nio.ByteBuffer; * pac.feed(input_bytes); * * // use next() method or iterators. - * for(Object obj : pac) { + * for(MessagePackObject obj : pac) { * // ... * } * @@ -75,7 +75,7 @@ import java.nio.ByteBuffer; * System.in.read(pac.getBuffer(), pac.getBufferOffset(), pac.getBufferCapacity()); * * // use next() method or iterators. - * for(Object obj : pac) { + * for(MessagePackObject obj : pac) { * // ... * } * @@ -96,12 +96,12 @@ import java.nio.ByteBuffer; * * // take out object if deserialized object is ready. * if(pac.isFinished()) { - * Object obj = pac.getData(); + * MessagePackObject obj = pac.getData(); * // ... * } * */ -public class Unpacker implements Iterable { +public class Unpacker implements Iterable { // buffer: // +---------------------------------------------+ @@ -170,16 +170,6 @@ public class Unpacker implements Iterable { this.stream = stream; } - /** - * Sets schema to convert deserialized object into specific type. - * Default schema is {@link GenericSchema} that leaves objects for generic type. Use {@link MessageConvertable#messageConvert(Object)} method to convert the generic object. - * @param s schem to use - */ - public Unpacker useSchema(Schema s) { - impl.setSchema(s); - return this; - } - /** * Gets the input stream. @@ -255,7 +245,7 @@ public class Unpacker implements Iterable { /** * Returns the iterator that calls {@link next()} method repeatedly. */ - public Iterator iterator() { + public Iterator iterator() { return new UnpackIterator(this); } @@ -387,7 +377,7 @@ public class Unpacker implements Iterable { /** * Gets the object deserialized by {@link execute(byte[])} method. */ - public Object getData() { + public MessagePackObject getData() { return impl.getData(); } @@ -557,7 +547,7 @@ public class Unpacker implements Iterable { * Gets one {@code Object} value from the buffer. * This method calls {@link fill()} method if needed. */ - final public Object unpackObject() throws IOException { + final public MessagePackObject unpackObject() throws IOException { return impl.unpackObject(); } diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index 10cf5f01..f004e6c4 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -18,11 +18,7 @@ package org.msgpack; import java.nio.ByteBuffer; -//import java.math.BigInteger; -import org.msgpack.*; -import org.msgpack.schema.GenericSchema; -import org.msgpack.schema.IMapSchema; -import org.msgpack.schema.IArraySchema; +import org.msgpack.object.*; public class UnpackerImpl { static final int CS_HEADER = 0x00; @@ -55,30 +51,19 @@ public class UnpackerImpl { private int[] stack_ct = new int[MAX_STACK_SIZE]; private int[] stack_count = new int[MAX_STACK_SIZE]; private Object[] stack_obj = new Object[MAX_STACK_SIZE]; - private Schema[] stack_schema = new Schema[MAX_STACK_SIZE]; private int top_ct; private int top_count; private Object top_obj; - private Schema top_schema; private ByteBuffer castBuffer = ByteBuffer.allocate(8); private boolean finished = false; - private Object data = null; - - private static final Schema GENERIC_SCHEMA = new GenericSchema(); - private Schema rootSchema; + private MessagePackObject data = null; public UnpackerImpl() { - setSchema(GENERIC_SCHEMA); - } - - public void setSchema(Schema schema) - { - this.rootSchema = schema; reset(); } - public final Object getData() + public final MessagePackObject getData() { return data; } @@ -94,7 +79,6 @@ public class UnpackerImpl { top_ct = 0; top_count = 0; top_obj = null; - top_schema = rootSchema; } public final void reset() @@ -127,20 +111,20 @@ public class UnpackerImpl { if((b & 0x80) == 0) { // Positive Fixnum //System.out.println("positive fixnum "+b); - obj = top_schema.createFromByte((byte)b); + obj = IntegerType.create((byte)b); break _push; } if((b & 0xe0) == 0xe0) { // Negative Fixnum //System.out.println("negative fixnum "+b); - obj = top_schema.createFromByte((byte)b); + obj = IntegerType.create((byte)b); break _push; } if((b & 0xe0) == 0xa0) { // FixRaw trail = b & 0x1f; if(trail == 0) { - obj = top_schema.createFromRaw(new byte[0], 0, 0); + obj = new RawType(new byte[0]); break _push; } cs = ACS_RAW_VALUE; @@ -151,25 +135,20 @@ public class UnpackerImpl { if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IArraySchema)) { - throw new RuntimeException("type error"); - } count = b & 0x0f; //System.out.println("fixarray count:"+count); - obj = new Object[count]; + obj = new MessagePackObject[count]; if(count == 0) { - obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + obj = new ArrayType((MessagePackObject[])obj); break _push; } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_ARRAY_ITEM; top_count = count; - top_schema = ((IArraySchema)top_schema).getElementSchema(0); break _header_again; } @@ -177,13 +156,10 @@ public class UnpackerImpl { if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IMapSchema)) { - throw new RuntimeException("type error"); - } count = b & 0x0f; - obj = new Object[count*2]; + obj = new MessagePackObject[count*2]; if(count == 0) { - obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + obj = new MapType((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -191,23 +167,21 @@ public class UnpackerImpl { stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_MAP_KEY; top_count = count; - top_schema = ((IMapSchema)top_schema).getKeySchema(); break _header_again; } switch(b & 0xff) { // FIXME case 0xc0: // nil - obj = top_schema.createFromNil(); + obj = new NilType(); break _push; case 0xc2: // false - obj = top_schema.createFromBoolean(false); + obj = new BooleanType(false); break _push; case 0xc3: // true - obj = top_schema.createFromBoolean(true); + obj = new BooleanType(true); break _push; case 0xca: // float case 0xcb: // double @@ -251,13 +225,13 @@ public class UnpackerImpl { case CS_FLOAT: castBuffer.rewind(); castBuffer.put(src, n, 4); - obj = top_schema.createFromFloat( castBuffer.getFloat(0) ); + obj = FloatType.create( castBuffer.getFloat(0) ); //System.out.println("float "+obj); break _push; case CS_DOUBLE: castBuffer.rewind(); castBuffer.put(src, n, 8); - obj = top_schema.createFromDouble( castBuffer.getDouble(0) ); + obj = FloatType.create( castBuffer.getDouble(0) ); //System.out.println("double "+obj); break _push; case CS_UINT_8: @@ -265,7 +239,7 @@ public class UnpackerImpl { //System.out.println(src[n]); //System.out.println(src[n+1]); //System.out.println(src[n-1]); - obj = top_schema.createFromShort( (short)((src[n]) & 0xff) ); + obj = IntegerType.create( (short)((src[n]) & 0xff) ); //System.out.println("uint8 "+obj); break _push; case CS_UINT_16: @@ -273,13 +247,13 @@ public class UnpackerImpl { //System.out.println(src[n+1]); castBuffer.rewind(); castBuffer.put(src, n, 2); - obj = top_schema.createFromInt( ((int)castBuffer.getShort(0)) & 0xffff ); + obj = IntegerType.create( ((int)castBuffer.getShort(0)) & 0xffff ); //System.out.println("uint 16 "+obj); break _push; case CS_UINT_32: castBuffer.rewind(); castBuffer.put(src, n, 4); - obj = top_schema.createFromLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + obj = IntegerType.create( ((long)castBuffer.getInt(0)) & 0xffffffffL ); //System.out.println("uint 32 "+obj); break _push; case CS_UINT_64: @@ -292,34 +266,34 @@ public class UnpackerImpl { //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported"); } else { - obj = top_schema.createFromLong( o ); + obj = IntegerType.create(o); } } break _push; case CS_INT_8: - obj = top_schema.createFromByte( src[n] ); + obj = IntegerType.create( src[n] ); break _push; case CS_INT_16: castBuffer.rewind(); castBuffer.put(src, n, 2); - obj = top_schema.createFromShort( castBuffer.getShort(0) ); + obj = IntegerType.create( castBuffer.getShort(0) ); break _push; case CS_INT_32: castBuffer.rewind(); castBuffer.put(src, n, 4); - obj = top_schema.createFromInt( castBuffer.getInt(0) ); + obj = IntegerType.create( castBuffer.getInt(0) ); break _push; case CS_INT_64: castBuffer.rewind(); castBuffer.put(src, n, 8); - obj = top_schema.createFromLong( castBuffer.getLong(0) ); + obj = IntegerType.create( castBuffer.getLong(0) ); break _push; case CS_RAW_16: castBuffer.rewind(); castBuffer.put(src, n, 2); trail = ((int)castBuffer.getShort(0)) & 0xffff; if(trail == 0) { - obj = top_schema.createFromRaw(new byte[0], 0, 0); + obj = new RawType(new byte[0]); break _push; } cs = ACS_RAW_VALUE; @@ -330,77 +304,67 @@ public class UnpackerImpl { // FIXME overflow check trail = castBuffer.getInt(0) & 0x7fffffff; if(trail == 0) { - obj = top_schema.createFromRaw(new byte[0], 0, 0); + obj = new RawType(new byte[0]); break _push; } cs = ACS_RAW_VALUE; - case ACS_RAW_VALUE: - obj = top_schema.createFromRaw(src, n, trail); + case ACS_RAW_VALUE: { + byte[] raw = new byte[trail]; + System.arraycopy(src, n, raw, 0, trail); + obj = new RawType(raw); + } break _push; case CS_ARRAY_16: if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IArraySchema)) { - throw new RuntimeException("type error"); - } castBuffer.rewind(); castBuffer.put(src, n, 2); count = ((int)castBuffer.getShort(0)) & 0xffff; - obj = new Object[count]; + obj = new MessagePackObject[count]; if(count == 0) { - obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + obj = new ArrayType((MessagePackObject[])obj); break _push; } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_ARRAY_ITEM; top_count = count; - top_schema = ((IArraySchema)top_schema).getElementSchema(0); break _header_again; case CS_ARRAY_32: if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IArraySchema)) { - throw new RuntimeException("type error"); - } castBuffer.rewind(); castBuffer.put(src, n, 4); // FIXME overflow check count = castBuffer.getInt(0) & 0x7fffffff; - obj = new Object[count]; + obj = new MessagePackObject[count]; if(count == 0) { - obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + obj = new ArrayType((MessagePackObject[])obj); break _push; } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_ARRAY_ITEM; top_count = count; - top_schema = ((IArraySchema)top_schema).getElementSchema(0); break _header_again; case CS_MAP_16: if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IMapSchema)) { - throw new RuntimeException("type error"); - } castBuffer.rewind(); castBuffer.put(src, n, 2); count = ((int)castBuffer.getShort(0)) & 0xffff; - obj = new Object[count*2]; + obj = new MessagePackObject[count*2]; if(count == 0) { - obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + obj = new MapType((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -408,26 +372,21 @@ public class UnpackerImpl { stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_MAP_KEY; top_count = count; - top_schema = ((IMapSchema)top_schema).getKeySchema(); break _header_again; case CS_MAP_32: if(top >= MAX_STACK_SIZE) { throw new UnpackException("parse error"); } - if(!(top_schema instanceof IMapSchema)) { - throw new RuntimeException("type error"); - } castBuffer.rewind(); castBuffer.put(src, n, 4); // FIXME overflow check count = castBuffer.getInt(0) & 0x7fffffff; - obj = new Object[count*2]; + obj = new MessagePackObject[count*2]; if(count == 0) { - obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + obj = new MapType((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -435,11 +394,9 @@ public class UnpackerImpl { stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; - stack_schema[top] = top_schema; top_obj = obj; top_ct = CT_MAP_KEY; top_count = count; - top_schema = ((IMapSchema)top_schema).getKeySchema(); break _header_again; default: throw new UnpackException("parse error"); @@ -454,7 +411,7 @@ public class UnpackerImpl { //System.out.println("push top:"+top); if(top == -1) { ++i; - data = obj; + data = (MessagePackObject)obj; finished = true; break _out; } @@ -468,14 +425,10 @@ public class UnpackerImpl { top_obj = stack_obj[top]; top_ct = stack_ct[top]; top_count = stack_count[top]; - top_schema = stack_schema[top]; - obj = ((IArraySchema)top_schema).createFromArray(ar); + obj = new ArrayType((MessagePackObject[])ar); stack_obj[top] = null; - stack_schema[top] = null; --top; break _push; - } else { - top_schema = ((IArraySchema)stack_schema[top]).getElementSchema(ar.length - top_count); } break _header_again; } @@ -484,7 +437,6 @@ public class UnpackerImpl { Object[] mp = (Object[])top_obj; mp[mp.length - top_count*2] = obj; top_ct = CT_MAP_VALUE; - top_schema = ((IMapSchema)stack_schema[top]).getValueSchema(); break _header_again; } case CT_MAP_VALUE: { @@ -495,10 +447,8 @@ public class UnpackerImpl { top_obj = stack_obj[top]; top_ct = stack_ct[top]; top_count = stack_count[top]; - top_schema = stack_schema[top]; - obj = ((IMapSchema)top_schema).createFromMap(mp); + obj = new MapType((MessagePackObject[])mp); stack_obj[top] = null; - stack_schema[top] = null; --top; break _push; } diff --git a/java/src/main/java/org/msgpack/schema/BooleanSchema.java b/java/src/main/java/org/msgpack/schema/BooleanSchema.java deleted file mode 100644 index 2c325f1f..00000000 --- a/java/src/main/java/org/msgpack/schema/BooleanSchema.java +++ /dev/null @@ -1,64 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class BooleanSchema extends Schema { - public BooleanSchema() { } - - @Override - public String getClassName() { - return "Boolean"; - } - - @Override - public String getExpression() { - return "boolean"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Boolean) { - pk.packBoolean((Boolean)obj); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final boolean convertBoolean(Object obj) throws MessageTypeException { - if(obj instanceof Boolean) { - return (Boolean)obj; - } - throw new MessageTypeException(); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertBoolean(obj); - } - - @Override - public Object createFromBoolean(boolean v) { - return v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ByteArraySchema.java b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java deleted file mode 100644 index af9c0edf..00000000 --- a/java/src/main/java/org/msgpack/schema/ByteArraySchema.java +++ /dev/null @@ -1,97 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.nio.ByteBuffer; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import org.msgpack.*; - -public class ByteArraySchema extends Schema { - public ByteArraySchema() { } - - @Override - public String getClassName() { - return "byte[]"; - } - - @Override - public String getExpression() { - return "raw"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof byte[]) { - byte[] b = (byte[])obj; - pk.packRaw(b.length); - pk.packRawBody(b); - } else if(obj instanceof String) { - try { - byte[] b = ((String)obj).getBytes("UTF-8"); - pk.packRaw(b.length); - pk.packRawBody(b); - } catch (UnsupportedEncodingException e) { - throw new MessageTypeException(); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final byte[] convertByteArray(Object obj) throws MessageTypeException { - if(obj instanceof byte[]) { - // FIXME copy? - //byte[] d = (byte[])obj; - //byte[] v = new byte[d.length]; - //System.arraycopy(d, 0, v, 0, d.length); - //return v; - return (byte[])obj; - } else if(obj instanceof ByteBuffer) { - ByteBuffer d = (ByteBuffer)obj; - byte[] v = new byte[d.capacity()]; - int pos = d.position(); - d.get(v); - d.position(pos); - return v; - } else if(obj instanceof String) { - try { - return ((String)obj).getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new MessageTypeException(); - } - } else { - throw new MessageTypeException(); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertByteArray(obj); - } - - @Override - public Object createFromRaw(byte[] b, int offset, int length) { - byte[] d = new byte[length]; - System.arraycopy(b, offset, d, 0, length); - return d; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java deleted file mode 100644 index 6003a0f5..00000000 --- a/java/src/main/java/org/msgpack/schema/ByteSchema.java +++ /dev/null @@ -1,96 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ByteSchema extends Schema { - public ByteSchema() { } - - @Override - public String getClassName() { - return "Byte"; - } - - @Override - public String getExpression() { - return "byte"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - short value = ((Number)obj).shortValue(); - if(value > Byte.MAX_VALUE) { - throw new MessageTypeException(); - } - pk.packByte((byte)value); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final byte convertByte(Object obj) throws MessageTypeException { - if(obj instanceof Number) { - short value = ((Number)obj).shortValue(); - if(value > Byte.MAX_VALUE) { - throw new MessageTypeException(); - } - return (byte)value; - } - throw new MessageTypeException(); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertByte(obj); - } - - @Override - public Object createFromByte(byte v) { - return (byte)v; - } - - @Override - public Object createFromShort(short v) { - if(v > Byte.MAX_VALUE) { - throw new MessageTypeException(); - } - return (byte)v; - } - - @Override - public Object createFromInt(int v) { - if(v > Byte.MAX_VALUE) { - throw new MessageTypeException(); - } - return (byte)v; - } - - @Override - public Object createFromLong(long v) { - if(v > Byte.MAX_VALUE) { - throw new MessageTypeException(); - } - return (byte)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java deleted file mode 100644 index a515996f..00000000 --- a/java/src/main/java/org/msgpack/schema/ClassGenerator.java +++ /dev/null @@ -1,244 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.List; -import java.io.IOException; -import java.io.File; -import java.io.Writer; -import org.msgpack.*; - -public class ClassGenerator { - private ClassSchema schema; - private Writer writer; - private int indent; - - private ClassGenerator(Writer writer) { - this.writer = writer; - this.indent = 0; - } - - public static void write(Schema schema, Writer dest) throws IOException { - if(!(schema instanceof ClassSchema)) { - throw new RuntimeException("schema is not class schema"); - } - ClassSchema cs = (ClassSchema)schema; - new ClassGenerator(dest).run(cs); - } - - private void run(ClassSchema cs) throws IOException { - List subclasses = new ArrayList(); - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(subclasses, f.getSchema()); - } - - for(ClassSchema sub : subclasses) { - sub.setNamespace(cs.getNamespace()); - sub.setImports(cs.getImports()); - } - - this.schema = cs; - - writeHeader(); - - writeClass(); - - for(ClassSchema sub : subclasses) { - this.schema = sub; - writeSubclass(); - } - - writeFooter(); - - this.schema = null; - writer.flush(); - } - - private void findSubclassSchema(List dst, Schema s) { - if(s instanceof ClassSchema) { - ClassSchema cs = (ClassSchema)s; - if(!dst.contains(cs)) { dst.add(cs); } - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(dst, f.getSchema()); - } - } else if(s instanceof ListSchema) { - ListSchema as = (ListSchema)s; - findSubclassSchema(dst, as.getElementSchema(0)); - } else if(s instanceof SetSchema) { - SetSchema as = (SetSchema)s; - findSubclassSchema(dst, as.getElementSchema(0)); - } else if(s instanceof MapSchema) { - MapSchema as = (MapSchema)s; - findSubclassSchema(dst, as.getKeySchema()); - findSubclassSchema(dst, as.getValueSchema()); - } - } - - private void writeHeader() throws IOException { - if(schema.getNamespace() != null) { - line("package "+schema.getNamespace()+";"); - line(); - } - line("import java.util.*;"); - line("import java.io.*;"); - line("import org.msgpack.*;"); - line("import org.msgpack.schema.ClassSchema;"); - line("import org.msgpack.schema.FieldSchema;"); - } - - private void writeFooter() throws IOException { - line(); - } - - private void writeClass() throws IOException { - line(); - line("public final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSubclass() throws IOException { - line(); - line("final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSchema() throws IOException { - line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); - line("public static ClassSchema getSchema() { return _SCHEMA; }"); - } - - private void writeMemberVariables() throws IOException { - line(); - for(FieldSchema f : schema.getFields()) { - line("public "+f.getSchema().getClassName()+" "+f.getName()+";"); - } - } - - private void writeMemberFunctions() throws IOException { - // void messagePack(Packer pk) - // boolean equals(Object obj) - // int hashCode() - // void set(int _index, Object _value) - // Object get(int _index); - // getXxx() - // setXxx(Xxx xxx) - writeConstructors(); - writeAccessors(); - writePackFunction(); - writeConvertFunction(); - writeFactoryFunction(); - } - - private void writeConstructors() throws IOException { - line(); - line("public "+schema.getClassName()+"() { }"); - } - - private void writeAccessors() throws IOException { - // FIXME - //line(); - //for(FieldSchema f : schema.getFields()) { - // line(""); - //} - } - - private void writePackFunction() throws IOException { - line(); - line("@Override"); - line("public void messagePack(Packer _pk) throws IOException"); - line("{"); - pushIndent(); - line("_pk.packArray("+schema.getFields().length+");"); - line("FieldSchema[] _fields = _SCHEMA.getFields();"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("_fields["+i+"].getSchema().pack(_pk, "+f.getName()+");"); - ++i; - } - popIndent(); - line("}"); - } - - private void writeConvertFunction() throws IOException { - line(); - line("@Override"); - line("@SuppressWarnings(\"unchecked\")"); - line("public void messageConvert(Object obj) throws MessageTypeException"); - line("{"); - pushIndent(); - line("Object[] _source = ((List)obj).toArray();"); - line("FieldSchema[] _fields = _SCHEMA.getFields();"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getClassName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); - ++i; - } - popIndent(); - line("}"); - } - - private void writeFactoryFunction() throws IOException { - line(); - line("@SuppressWarnings(\"unchecked\")"); - line("public static "+schema.getClassName()+" createFromMessage(Object[] _message)"); - line("{"); - pushIndent(); - line(schema.getClassName()+" _self = new "+schema.getClassName()+"();"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getClassName()+")_message["+i+"];"); - ++i; - } - line("return _self;"); - popIndent(); - line("}"); - } - - private void line(String str) throws IOException { - for(int i=0; i < indent; ++i) { - writer.write("\t"); - } - writer.write(str+"\n"); - } - - private void line() throws IOException { - writer.write("\n"); - } - - private void pushIndent() { - indent += 1; - } - - private void popIndent() { - indent -= 1; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java deleted file mode 100644 index cd59755a..00000000 --- a/java/src/main/java/org/msgpack/schema/ClassSchema.java +++ /dev/null @@ -1,91 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Arrays; -import java.util.List; -import org.msgpack.*; - -public abstract class ClassSchema extends Schema implements IArraySchema { - protected String name; - protected FieldSchema[] fields; - protected List imports; - protected String namespace; - protected String fqdn; - - public ClassSchema( - String name, String namespace, - List imports, List fields) { - this.name = name; - this.namespace = namespace; - this.imports = imports; // FIXME clone? - this.fields = new FieldSchema[fields.size()]; - System.arraycopy(fields.toArray(), 0, this.fields, 0, fields.size()); - if(namespace == null) { - this.fqdn = name; - } else { - this.fqdn = namespace+"."+name; - } - } - - @Override - public String getClassName() { - return name; - } - - @Override - public String getExpression() { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(name); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(FieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - public boolean equals(ClassSchema o) { - return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && - name.equals(o.name); - } - - public final FieldSchema[] getFields() { - return fields; - } - - String getNamespace() { - return namespace; - } - - List getImports() { - return imports; - } - - void setNamespace(String namespace) { - this.namespace = namespace; - } - - void setImports(List imports) { - this.imports = imports; // FIXME clone? - } -} - diff --git a/java/src/main/java/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java deleted file mode 100644 index cb857c38..00000000 --- a/java/src/main/java/org/msgpack/schema/DoubleSchema.java +++ /dev/null @@ -1,74 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class DoubleSchema extends Schema { - public DoubleSchema() { } - - @Override - public String getClassName() { - return "Double"; - } - - @Override - public String getExpression() { - return "double"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Double) { - pk.packDouble((Double)obj); - } else if(obj instanceof Float) { - pk.packFloat((Float)obj); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final double convertDouble(Object obj) throws MessageTypeException { - if(obj instanceof Double) { - return (Double)obj; - } else if(obj instanceof Float) { - return ((Float)obj).doubleValue(); - } else { - throw new MessageTypeException(); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertDouble(obj); - } - - @Override - public Object createFromFloat(float v) { - return (double)v; - } - - @Override - public Object createFromDouble(double v) { - return (double)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/FieldSchema.java b/java/src/main/java/org/msgpack/schema/FieldSchema.java deleted file mode 100644 index 66c2ff23..00000000 --- a/java/src/main/java/org/msgpack/schema/FieldSchema.java +++ /dev/null @@ -1,43 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import org.msgpack.Schema; - -public class FieldSchema { - private String name; - private Schema schema; - - public FieldSchema(String name, Schema schema) { - this.name = name; - this.schema = schema; - } - - public final String getName() { - return name; - } - - public final Schema getSchema() { - return schema; - } - - public String getExpression() { - return "(field "+name+" "+schema.getExpression()+")"; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java deleted file mode 100644 index cd732014..00000000 --- a/java/src/main/java/org/msgpack/schema/FloatSchema.java +++ /dev/null @@ -1,74 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class FloatSchema extends Schema { - public FloatSchema() { } - - @Override - public String getClassName() { - return "Float"; - } - - @Override - public String getExpression() { - return "float"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Double) { - pk.packDouble((Double)obj); - } else if(obj instanceof Float) { - pk.packFloat((Float)obj); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final float convertFloat(Object obj) throws MessageTypeException { - if(obj instanceof Double) { - return ((Double)obj).floatValue(); - } else if(obj instanceof Float) { - return (Float)obj; - } else { - throw new MessageTypeException(); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertFloat(obj); - } - - @Override - public Object createFromFloat(float v) { - return (float)v; - } - - @Override - public Object createFromDouble(double v) { - return (float)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java deleted file mode 100644 index 1ab4c334..00000000 --- a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java +++ /dev/null @@ -1,87 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import org.msgpack.*; - -public class GenericClassSchema extends ClassSchema { - public GenericClassSchema( - String name, String namespace, - List imports, List fields) { - super(name, namespace, imports, fields); - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Map) { - Map d = (Map)obj; - pk.packArray(fields.length); - for(int i=0; i < fields.length; ++i) { - FieldSchema f = fields[i]; - f.getSchema().pack(pk, d.get(f.getName())); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Collection) { - // FIXME optimize - return createFromArray( ((Collection)obj).toArray() ); - } else if(obj instanceof Map) { - HashMap m = new HashMap(fields.length); - Map d = (Map)obj; - for(int i=0; i < fields.length; ++i) { - FieldSchema f = fields[i]; - String fieldName = f.getName(); - m.put(fieldName, f.getSchema().convert(d.get(fieldName))); - } - return m; - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public Schema getElementSchema(int index) { - // FIXME check index < fields.length - return fields[index].getSchema(); - } - - public Object createFromArray(Object[] obj) { - HashMap m = new HashMap(fields.length); - int i=0; - for(; i < obj.length; ++i) { - m.put(fields[i].getName(), obj[i]); - } - for(; i < fields.length; ++i) { - m.put(fields[i].getName(), null); - } - return m; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java deleted file mode 100644 index f9098edd..00000000 --- a/java/src/main/java/org/msgpack/schema/GenericSchema.java +++ /dev/null @@ -1,129 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Arrays; -import java.util.List; -import java.util.HashMap; -import java.io.IOException; -import org.msgpack.*; - -public class GenericSchema extends Schema implements IArraySchema, IMapSchema { - public GenericSchema() { } - - @Override - public String getClassName() { - return "Object"; - } - - @Override - public String getExpression() { - return "object"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - pk.pack(obj); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return obj; - } - - @Override - public Schema getElementSchema(int index) { - return this; - } - - @Override - public Schema getKeySchema() { - return this; - } - - @Override - public Schema getValueSchema() { - return this; - } - - @Override - public Object createFromNil() { - return null; - } - - @Override - public Object createFromBoolean(boolean v) { - return v; - } - - @Override - public Object createFromByte(byte v) { - return v; - } - - @Override - public Object createFromShort(short v) { - return v; - } - - @Override - public Object createFromInt(int v) { - return v; - } - - @Override - public Object createFromLong(long v) { - return v; - } - - @Override - public Object createFromFloat(float v) { - return v; - } - - @Override - public Object createFromDouble(double v) { - return v; - } - - @Override - public Object createFromRaw(byte[] b, int offset, int length) { - byte[] bytes = new byte[length]; - System.arraycopy(b, offset, bytes, 0, length); - return bytes; - } - - @Override - public Object createFromArray(Object[] obj) { - return Arrays.asList(obj); - } - - @Override - @SuppressWarnings("unchecked") - public Object createFromMap(Object[] obj) { - HashMap m = new HashMap(obj.length / 2); - int i = 0; - while(i < obj.length) { - Object k = obj[i++]; - Object v = obj[i++]; - m.put(k, v); - } - return m; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/IArraySchema.java b/java/src/main/java/org/msgpack/schema/IArraySchema.java deleted file mode 100644 index 67e9f55a..00000000 --- a/java/src/main/java/org/msgpack/schema/IArraySchema.java +++ /dev/null @@ -1,26 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import org.msgpack.Schema; - -public interface IArraySchema { - public Schema getElementSchema(int index); - public Object createFromArray(Object[] obj); -} - diff --git a/java/src/main/java/org/msgpack/schema/IMapSchema.java b/java/src/main/java/org/msgpack/schema/IMapSchema.java deleted file mode 100644 index 3a2f556c..00000000 --- a/java/src/main/java/org/msgpack/schema/IMapSchema.java +++ /dev/null @@ -1,27 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import org.msgpack.Schema; - -public interface IMapSchema { - public Schema getKeySchema(); - public Schema getValueSchema(); - public Object createFromMap(Object[] obj); -} - diff --git a/java/src/main/java/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java deleted file mode 100644 index 269f4fb7..00000000 --- a/java/src/main/java/org/msgpack/schema/IntSchema.java +++ /dev/null @@ -1,96 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class IntSchema extends Schema { - public IntSchema() { } - - @Override - public String getClassName() { - return "Integer"; - } - - @Override - public String getExpression() { - return "int"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - int value = ((Number)obj).intValue(); - if(value >= Short.MAX_VALUE) { - long lvalue = ((Number)obj).longValue(); - if(lvalue > Integer.MAX_VALUE) { - throw new MessageTypeException(); - } - } - pk.packInt(value); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final int convertInt(Object obj) throws MessageTypeException { - if(obj instanceof Number) { - int value = ((Number)obj).intValue(); - if(value >= Integer.MAX_VALUE) { - long lvalue = ((Number)obj).longValue(); - if(lvalue > Integer.MAX_VALUE) { - throw new MessageTypeException(); - } - } - return value; - } - throw new MessageTypeException(); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertInt(obj); - } - - @Override - public Object createFromByte(byte v) { - return (int)v; - } - - @Override - public Object createFromShort(short v) { - return (int)v; - } - - @Override - public Object createFromInt(int v) { - return (int)v; - } - - @Override - public Object createFromLong(long v) { - if(v > Integer.MAX_VALUE) { - throw new MessageTypeException(); - } - return (int)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ListSchema.java b/java/src/main/java/org/msgpack/schema/ListSchema.java deleted file mode 100644 index bef8cc41..00000000 --- a/java/src/main/java/org/msgpack/schema/ListSchema.java +++ /dev/null @@ -1,111 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Set; -import java.util.List; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.RandomAccess; -import java.io.IOException; -import org.msgpack.*; - -public class ListSchema extends Schema implements IArraySchema { - private Schema elementSchema; - - public ListSchema(Schema elementSchema) { - this.elementSchema = elementSchema; - } - - @Override - public String getClassName() { - return "List<"+elementSchema.getClassName()+">"; - } - - @Override - public String getExpression() { - return "(array "+elementSchema.getExpression()+")"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof List) { - List d = (List)obj; - pk.packArray(d.size()); - if(obj instanceof RandomAccess) { - for(int i=0; i < d.size(); ++i) { - elementSchema.pack(pk, d.get(i)); - } - } else { - for(Object e : d) { - elementSchema.pack(pk, e); - } - } - } else if(obj instanceof Set) { - Set d = (Set)obj; - pk.packArray(d.size()); - for(Object e : d) { - elementSchema.pack(pk, e); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @SuppressWarnings("unchecked") - public static final List convertList(Object obj, - Schema elementSchema, List dest) throws MessageTypeException { - if(!(obj instanceof List)) { - throw new MessageTypeException(); - } - List d = (List)obj; - if(dest == null) { - dest = new ArrayList(d.size()); - } - if(obj instanceof RandomAccess) { - for(int i=0; i < d.size(); ++i) { - dest.add( (T)elementSchema.convert(d.get(i)) ); - } - } else { - for(Object e : d) { - dest.add( (T)elementSchema.convert(e) ); - } - } - return dest; - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertList(obj, elementSchema, null); - } - - @Override - public Schema getElementSchema(int index) { - return elementSchema; - } - - @Override - public Object createFromArray(Object[] obj) { - return Arrays.asList(obj); - } -} - diff --git a/java/src/main/java/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java deleted file mode 100644 index 728fa21e..00000000 --- a/java/src/main/java/org/msgpack/schema/LongSchema.java +++ /dev/null @@ -1,80 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class LongSchema extends Schema { - public LongSchema() { } - - @Override - public String getClassName() { - return "Long"; - } - - @Override - public String getExpression() { - return "long"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - long value = ((Number)obj).longValue(); - pk.packLong(value); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final long convertLong(Object obj) throws MessageTypeException { - if(obj instanceof Number) { - return ((Number)obj).longValue(); - } - throw new MessageTypeException(); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertLong(obj); - } - - @Override - public Object createFromByte(byte v) { - return (long)v; - } - - @Override - public Object createFromShort(short v) { - return (long)v; - } - - @Override - public Object createFromInt(int v) { - return (long)v; - } - - @Override - public Object createFromLong(long v) { - return (long)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java deleted file mode 100644 index 2e09af31..00000000 --- a/java/src/main/java/org/msgpack/schema/MapSchema.java +++ /dev/null @@ -1,106 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import org.msgpack.*; - -public class MapSchema extends Schema implements IMapSchema { - private Schema keySchema; - private Schema valueSchema; - - public MapSchema(Schema keySchema, Schema valueSchema) { - this.keySchema = keySchema; - this.valueSchema = valueSchema; - } - - @Override - public String getClassName() { - return "Map<"+keySchema.getClassName()+", "+valueSchema.getClassName()+">"; - } - - @Override - public String getExpression() { - return "(map "+keySchema.getExpression()+" "+valueSchema.getExpression()+")"; - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Map) { - Map d = (Map)obj; - pk.packMap(d.size()); - for(Map.Entry e : d.entrySet()) { - keySchema.pack(pk, e.getKey()); - valueSchema.pack(pk, e.getValue()); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @SuppressWarnings("unchecked") - public static final Map convertMap(Object obj, - Schema keySchema, Schema valueSchema, Map dest) throws MessageTypeException { - if(!(obj instanceof Map)) { - throw new MessageTypeException(); - } - Map d = (Map)obj; - if(dest == null) { - dest = new HashMap(d.size()); - } - for(Map.Entry e : d.entrySet()) { - dest.put((K)keySchema.convert(e.getKey()), - (V)valueSchema.convert(e.getValue())); - } - return dest; - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertMap(obj, keySchema, valueSchema, null); - } - - @Override - public Schema getKeySchema() { - return keySchema; - } - - @Override - public Schema getValueSchema() { - return valueSchema; - } - - @Override - @SuppressWarnings("unchecked") - public Object createFromMap(Object[] obj) { - HashMap dest = new HashMap(obj.length / 2); - int i = 0; - while(i < obj.length) { - Object k = obj[i++]; - Object v = obj[i++]; - dest.put(k, v); - } - return dest; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java b/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java deleted file mode 100644 index fb94adfa..00000000 --- a/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.msgpack.schema; - -import java.util.Arrays; -import java.util.List; -import java.lang.reflect.*; -import org.msgpack.*; - -// FIXME -public abstract class ReflectionClassSchema extends ClassSchema { - private Constructor constructorCache; - - public ReflectionClassSchema(String name, List fields, String namespace, List imports) { - super(name, namespace, imports, fields); - } - - /* - Schema getElementSchema(int index) - { - // FIXME check index < fields.length - fields[index].getSchema(); - } - - Object createFromArray(Object[] obj) - { - Object o = newInstance(); - ((MessageConvertable)o).messageConvert(obj); - return o; - } - - Object newInstance() - { - if(constructorCache == null) { - cacheConstructor(); - } - try { - return constructorCache.newInstance((Object[])null); - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - private void cacheConstructor() - { - try { - Class c = Class.forName(fqdn); - int index = 0; - for(SpecificFieldSchema f : fields) { - f.cacheField(c, index++); - } - constructorCache = c.getDeclaredConstructor((Class[])null); - constructorCache.setAccessible(true); - } catch(ClassNotFoundException e) { - throw new RuntimeException("class not found: "+fqdn); - } catch (NoSuchMethodException e) { - throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); - } - } - */ -} - diff --git a/java/src/main/java/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java deleted file mode 100644 index 4345e924..00000000 --- a/java/src/main/java/org/msgpack/schema/SSchemaParser.java +++ /dev/null @@ -1,264 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Stack; -import java.util.regex.Pattern; -import java.util.regex.Matcher; -import org.msgpack.*; - -// FIXME exception class - -public class SSchemaParser { - public static Schema parse(String source) { - return new SSchemaParser(false).run(source); - } - - public static Schema load(String source) { - return new SSchemaParser(true).run(source); - } - - private static abstract class SExp { - boolean isAtom() { return false; } - public String getAtom() { return null; } - - boolean isTuple() { return false; } - public SExp getTuple(int i) { return null; } - public int size() { return 0; } - public boolean empty() { return size() == 0; } - Iterator iterator(int offset) { return null; } - } - - private static class SAtom extends SExp { - private String atom; - - SAtom(String atom) { this.atom = atom; } - - boolean isAtom() { return true; } - public String getAtom() { return atom; } - - public String toString() { return atom; } - } - - private static class STuple extends SExp { - private List tuple; - - STuple() { this.tuple = new ArrayList(); } - - public void add(SExp e) { tuple.add(e); } - - boolean isTuple() { return true; } - public SExp getTuple(int i) { return tuple.get(i); } - public int size() { return tuple.size(); } - - Iterator iterator(int skip) { - Iterator i = tuple.iterator(); - for(int s=0; s < skip; ++s) { i.next(); } - return i; - } - - public String toString() { - if(tuple.isEmpty()) { return "()"; } - Iterator i = tuple.iterator(); - StringBuffer o = new StringBuffer(); - o.append("(").append(i.next()); - while(i.hasNext()) { o.append(" ").append(i.next()); } - o.append(")"); - return o.toString(); - } - } - - boolean specificClass; - - private SSchemaParser(boolean specificClass) { - this.specificClass = specificClass; - } - - private static Pattern pattern = Pattern.compile( - "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); - - private Schema run(String source) { - Matcher m = pattern.matcher(source); - - Stack stack = new Stack(); - String token; - - while(true) { - while(true) { - if(!m.find()) { throw new RuntimeException("unexpected end of file"); } - token = m.group(1); - if(token != null) { break; } - } - - if(token.equals("(")) { - stack.push(new STuple()); - } else if(token.equals(")")) { - STuple top = stack.pop(); - if(stack.empty()) { - stack.push(top); - break; - } - stack.peek().add(top); - } else { - if(stack.empty()) { - throw new RuntimeException("unexpected token '"+token+"'"); - } - stack.peek().add(new SAtom(token)); - } - } - - while(true) { - if(!m.find()) { break; } - token = m.group(1); - if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } - } - - return readType( stack.pop() ); - } - - private Schema readType(SExp exp) { - if(exp.isAtom()) { - String type = exp.getAtom(); - if(type.equals("string")) { - return new StringSchema(); - } else if(type.equals("raw")) { - return new ByteArraySchema(); - } else if(type.equals("byte")) { - return new ByteSchema(); - } else if(type.equals("short")) { - return new ShortSchema(); - } else if(type.equals("int")) { - return new IntSchema(); - } else if(type.equals("long")) { - return new LongSchema(); - } else if(type.equals("float")) { - return new FloatSchema(); - } else if(type.equals("double")) { - return new DoubleSchema(); - } else if(type.equals("object")) { - return new GenericSchema(); - } else { - throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); - } - } else { - String type = exp.getTuple(0).getAtom(); - if(type.equals("class")) { - return parseClass(exp); - } else if(type.equals("array")) { - return parseList(exp); - } else if(type.equals("set")) { - return parseSet(exp); - } else if(type.equals("map")) { - return parseMap(exp); - } else { - throw new RuntimeException("class, list, set or map is expected but got '"+type+"': "+exp); - } - } - } - - private ClassSchema parseClass(SExp exp) { - if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); - } - - String namespace = null; - List imports = new ArrayList(); - String name = exp.getTuple(1).getAtom(); - List fields = new ArrayList(); - - for(Iterator i=exp.iterator(2); i.hasNext();) { - SExp subexp = i.next(); - if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { - throw new RuntimeException("field, package or import is expected: "+subexp); - } - String type = subexp.getTuple(0).getAtom(); - if(type.equals("field")) { - fields.add( parseField(subexp) ); - } else if(type.equals("package")) { - if(namespace != null) { - throw new RuntimeException("duplicated package definition: "+subexp); - } - namespace = parseNamespace(subexp); - } else if(type.equals("import")) { - imports.add( parseImport(subexp) ); - } else { - throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); - } - } - - if(specificClass) { - return new SpecificClassSchema(name, namespace, imports, fields); - } else { - return new GenericClassSchema(name, namespace, imports, fields); - } - } - - private ListSchema parseList(SExp exp) { - if(exp.size() != 2) { - throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp); - } - Schema elementType = readType(exp.getTuple(1)); - return new ListSchema(elementType); - } - - private SetSchema parseSet(SExp exp) { - if(exp.size() != 2) { - throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp); - } - Schema elementType = readType(exp.getTuple(1)); - return new SetSchema(elementType); - } - - private MapSchema parseMap(SExp exp) { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); - } - Schema keyType = readType(exp.getTuple(1)); - Schema valueType = readType(exp.getTuple(2)); - return new MapSchema(keyType, valueType); - } - - private String parseNamespace(SExp exp) { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("package is (package NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private String parseImport(SExp exp) { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("import is (import NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private FieldSchema parseField(SExp exp) { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("field is (field NAME TYPE): "+exp); - } - String name = exp.getTuple(1).getAtom(); - Schema type = readType(exp.getTuple(2)); - return new FieldSchema(name, type); - } -} - diff --git a/java/src/main/java/org/msgpack/schema/SetSchema.java b/java/src/main/java/org/msgpack/schema/SetSchema.java deleted file mode 100644 index a3e19741..00000000 --- a/java/src/main/java/org/msgpack/schema/SetSchema.java +++ /dev/null @@ -1,115 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Set; -import java.util.List; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.RandomAccess; -import java.io.IOException; -import org.msgpack.*; - -public class SetSchema extends Schema implements IArraySchema { - private Schema elementSchema; - - public SetSchema(Schema elementSchema) { - this.elementSchema = elementSchema; - } - - @Override - public String getClassName() { - return "Set<"+elementSchema.getClassName()+">"; - } - - @Override - public String getExpression() { - return "(set "+elementSchema.getExpression()+")"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof List) { - List d = (List)obj; - pk.packArray(d.size()); - if(obj instanceof RandomAccess) { - for(int i=0; i < d.size(); ++i) { - elementSchema.pack(pk, d.get(i)); - } - } else { - for(Object e : d) { - elementSchema.pack(pk, e); - } - } - } else if(obj instanceof Set) { - Set d = (Set)obj; - pk.packArray(d.size()); - for(Object e : d) { - elementSchema.pack(pk, e); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @SuppressWarnings("unchecked") - public static final Set convertSet(Object obj, - Schema elementSchema, Set dest) throws MessageTypeException { - if(!(obj instanceof List)) { - throw new MessageTypeException(); - } - List d = (List)obj; - if(dest == null) { - dest = new HashSet(d.size()); - } - if(obj instanceof RandomAccess) { - for(int i=0; i < d.size(); ++i) { - dest.add( (T)elementSchema.convert(d.get(i)) ); - } - } else { - for(Object e : d) { - dest.add( (T)elementSchema.convert(e) ); - } - } - return dest; - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertSet(obj, elementSchema, null); - } - - @Override - public Schema getElementSchema(int index) { - return elementSchema; - } - - @Override - public Object createFromArray(Object[] obj) { - Set m = new HashSet(obj.length); - for(int i=0; i < obj.length; i++) { - m.add(obj[i]); - } - return m; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java deleted file mode 100644 index 21b93279..00000000 --- a/java/src/main/java/org/msgpack/schema/ShortSchema.java +++ /dev/null @@ -1,93 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ShortSchema extends Schema { - public ShortSchema() { } - - @Override - public String getClassName() { - return "Short"; - } - - @Override - public String getExpression() { - return "short"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof Number) { - int value = ((Number)obj).intValue(); - if(value > Short.MAX_VALUE) { - throw new MessageTypeException(); - } - pk.packShort((short)value); - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final short convertShort(Object obj) throws MessageTypeException { - if(obj instanceof Number) { - int value = ((Number)obj).intValue(); - if(value > Short.MAX_VALUE) { - throw new MessageTypeException(); - } - return (short)value; - } - throw new MessageTypeException(); - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertShort(obj); - } - - @Override - public Object createFromByte(byte v) { - return (short)v; - } - - @Override - public Object createFromShort(short v) { - return (short)v; - } - - @Override - public Object createFromInt(int v) { - if(v > Short.MAX_VALUE) { - throw new MessageTypeException(); - } - return (short)v; - } - - @Override - public Object createFromLong(long v) { - if(v > Short.MAX_VALUE) { - throw new MessageTypeException(); - } - return (short)v; - } -} - diff --git a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java deleted file mode 100644 index 850f6218..00000000 --- a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java +++ /dev/null @@ -1,122 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.util.Collection; -import java.util.List; -import java.lang.reflect.*; -import java.io.IOException; -import org.msgpack.*; - -public class SpecificClassSchema extends ClassSchema { - private Class classCache; - private Method factoryCache; - private Constructor constructorCache; - - public SpecificClassSchema( - String name, String namespace, - List imports, List fields) { - super(name, namespace, imports, fields); - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException { - if(obj == null) { - pk.packNil(); - return; - } - if(classCache == null) { - cacheFactory(); - } - if(classCache.isInstance(obj)) { - ((MessagePackable)obj).messagePack(pk); - } else { - // FIXME Map - throw MessageTypeException.invalidConvert(obj, this); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - if(obj instanceof Collection) { - if(constructorCache == null) { - cacheConstructor(); - } - try { - MessageConvertable o = (MessageConvertable)constructorCache.newInstance((Object[])null); - o.messageConvert(obj); - return o; - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public Schema getElementSchema(int index) { - // FIXME check index < fields.length - return fields[index].getSchema(); - } - - public Object createFromArray(Object[] obj) { - if(factoryCache == null) { - cacheFactory(); - } - try { - return factoryCache.invoke(null, new Object[]{obj}); - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getCause()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - @SuppressWarnings("unchecked") - private void cacheFactory() { - try { - classCache = Class.forName(fqdn); - factoryCache = classCache.getDeclaredMethod("createFromMessage", new Class[]{Object[].class}); - factoryCache.setAccessible(true); - } catch(ClassNotFoundException e) { - throw new RuntimeException("class not found: "+fqdn); - } catch (NoSuchMethodException e) { - throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); - } - } - - @SuppressWarnings("unchecked") - private void cacheConstructor() { - try { - classCache = Class.forName(fqdn); - constructorCache = classCache.getDeclaredConstructor((Class[])null); - constructorCache.setAccessible(true); - } catch(ClassNotFoundException e) { - throw new RuntimeException("class not found: "+fqdn); - } catch (NoSuchMethodException e) { - throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); - } - } -} - diff --git a/java/src/main/java/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java deleted file mode 100644 index 23e4e64e..00000000 --- a/java/src/main/java/org/msgpack/schema/StringSchema.java +++ /dev/null @@ -1,102 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack.schema; - -import java.nio.ByteBuffer; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import org.msgpack.*; - -public class StringSchema extends Schema { - public StringSchema() { } - - @Override - public String getClassName() { - return "String"; - } - - @Override - public String getExpression() { - return "string"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException { - if(obj instanceof byte[]) { - byte[] b = (byte[])obj; - pk.packRaw(b.length); - pk.packRawBody(b); - } else if(obj instanceof String) { - try { - byte[] b = ((String)obj).getBytes("UTF-8"); - pk.packRaw(b.length); - pk.packRawBody(b); - } catch (UnsupportedEncodingException e) { - throw new MessageTypeException(); - } - } else if(obj == null) { - pk.packNil(); - } else { - throw MessageTypeException.invalidConvert(obj, this); - } - } - - public static final String convertString(Object obj) throws MessageTypeException { - if(obj instanceof byte[]) { - try { - return new String((byte[])obj, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new MessageTypeException(); - } - } else if(obj instanceof String) { - return (String)obj; - } else if(obj instanceof ByteBuffer) { - ByteBuffer d = (ByteBuffer)obj; - try { - if(d.hasArray()) { - return new String(d.array(), d.position(), d.capacity(), "UTF-8"); - } else { - byte[] v = new byte[d.capacity()]; - int pos = d.position(); - d.get(v); - d.position(pos); - return new String(v, "UTF-8"); - } - } catch (UnsupportedEncodingException e) { - throw new MessageTypeException(); - } - } else { - throw new MessageTypeException(); - } - } - - @Override - public Object convert(Object obj) throws MessageTypeException { - return convertString(obj); - } - - @Override - public Object createFromRaw(byte[] b, int offset, int length) { - try { - return new String(b, offset, length, "UTF-8"); - } catch (Exception e) { - throw new RuntimeException(e.getMessage()); - } - } -} - diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index b02bbb40..ca3d235c 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -8,240 +8,223 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestPackUnpack { - protected Object unpackOne(ByteArrayOutputStream out) { - return unpackOne(out, null); - } - protected Object unpackOne(ByteArrayOutputStream out, Schema schema) { - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - if (schema != null) - upk = upk.useSchema(schema); - Iterator it = upk.iterator(); - assertEquals(true, it.hasNext()); - Object obj = it.next(); - assertEquals(false, it.hasNext()); - return obj; - } + public MessagePackObject unpackOne(ByteArrayOutputStream out) { + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + Iterator it = upk.iterator(); + assertEquals(true, it.hasNext()); + MessagePackObject obj = it.next(); + assertEquals(false, it.hasNext()); + return obj; + } - @Test - public void testInt() throws Exception { - testInt(0); - testInt(-1); - testInt(1); - testInt(Integer.MIN_VALUE); - testInt(Integer.MAX_VALUE); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testInt(rand.nextInt()); - } - public void testInt(int val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out); - if (obj instanceof Byte) - assertEquals(val, ((Byte)obj).intValue()); - else if (obj instanceof Integer) - assertEquals(val, ((Integer)obj).intValue()); - else if (obj instanceof Short) - assertEquals(val, ((Short)obj).intValue()); - else if (obj instanceof Long) - assertEquals(val, ((Long)obj).intValue()); - else { - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + public void testInt(int val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asInt()); + } + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } - @Test - public void testFloat() throws Exception { - testFloat((float)0.0); - testFloat((float)-0.0); - testFloat((float)1.0); - testFloat((float)-1.0); - testFloat((float)Float.MAX_VALUE); - testFloat((float)Float.MIN_VALUE); - testFloat((float)Float.NaN); - testFloat((float)Float.NEGATIVE_INFINITY); - testFloat((float)Float.POSITIVE_INFINITY); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testFloat(rand.nextFloat()); - } - public void testFloat(float val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out); - if (obj instanceof Float) - assertEquals(val, ((Float)obj).floatValue(), 10e-10); - else { - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + public void testFloat(float val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asFloat(), 10e-10); + } + @Test + public void testFloat() throws Exception { + testFloat((float)0.0); + testFloat((float)-0.0); + testFloat((float)1.0); + testFloat((float)-1.0); + testFloat((float)Float.MAX_VALUE); + testFloat((float)Float.MIN_VALUE); + testFloat((float)Float.NaN); + testFloat((float)Float.NEGATIVE_INFINITY); + testFloat((float)Float.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testFloat(rand.nextFloat()); + } - @Test - public void testDouble() throws Exception { - testDouble((double)0.0); - testDouble((double)-0.0); - testDouble((double)1.0); - testDouble((double)-1.0); - testDouble((double)Double.MAX_VALUE); - testDouble((double)Double.MIN_VALUE); - testDouble((double)Double.NaN); - testDouble((double)Double.NEGATIVE_INFINITY); - testDouble((double)Double.POSITIVE_INFINITY); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testDouble(rand.nextDouble()); - } - public void testDouble(double val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out); - if (obj instanceof Double) - assertEquals(val, ((Double)obj).doubleValue(), 10e-10); - else { - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asDouble(), 10e-10); + } + @Test + public void testDouble() throws Exception { + testDouble((double)0.0); + testDouble((double)-0.0); + testDouble((double)1.0); + testDouble((double)-1.0); + testDouble((double)Double.MAX_VALUE); + testDouble((double)Double.MIN_VALUE); + testDouble((double)Double.NaN); + testDouble((double)Double.NEGATIVE_INFINITY); + testDouble((double)Double.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testDouble(rand.nextDouble()); + } - @Test - public void testNil() throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).packNil(); - Object obj = unpackOne(out); - assertEquals(null, obj); - } + @Test + public void testNil() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).packNil(); + MessagePackObject obj = unpackOne(out); + assertTrue(obj.isNull()); + } - @Test - public void testBoolean() throws Exception { - testBoolean(false); - testBoolean(true); - } - public void testBoolean(boolean val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out); - if (obj instanceof Boolean) - assertEquals(val, ((Boolean)obj).booleanValue()); - else { - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + @Test + public void testBoolean() throws Exception { + testBoolean(false); + testBoolean(true); + } + public void testBoolean(boolean val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asBoolean()); + } - @Test - public void testString() throws Exception { - testString(""); - testString("a"); - testString("ab"); - testString("abc"); - // small size string - for (int i = 0; i < 100; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 31 + 1; - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - // medium size string - for (int i = 0; i < 100; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 100 + (1 << 15); - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - // large size string - for (int i = 0; i < 10; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 100 + (1 << 31); - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - } - public void testString(String val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out); - if (obj instanceof byte[]) - assertEquals(val, new String((byte[])obj)); - else { - System.out.println("obj=" + obj); - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asString()); + } + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); - @Test - public void testArray() throws Exception { - List emptyList = new ArrayList(); - testArray(emptyList, Schema.parse("(array int)")); + // small size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 31 + 1; + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } - for (int i = 0; i < 1000; i++) { - Schema schema = Schema.parse("(array int)"); - List l = new ArrayList(); - int len = (int)Math.random() % 1000 + 1; - for (int j = 0; j < len; j++) - l.add(j); - testArray(l, schema); - } - for (int i = 0; i < 1000; i++) { - Schema schema = Schema.parse("(array string)"); - List l = new ArrayList(); - int len = (int)Math.random() % 1000 + 1; - for (int j = 0; j < len; j++) - l.add(Integer.toString(j)); - testArray(l, schema); - } - } - public void testArray(List val, Schema schema) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out, schema); - if (obj instanceof List) - assertTrue(val.equals(obj)); - else { - System.out.println("obj=" + obj); - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + // medium size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 15); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } - @Test - public void testMap() throws Exception { - Map emptyMap = new HashMap(); - testMap(emptyMap, Schema.parse("(map int int)")); + // large size string + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 31); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + } - for (int i = 0; i < 1000; i++) { - Schema schema = Schema.parse("(map int int)"); - Map m = new HashMap(); - int len = (int)Math.random() % 1000 + 1; - for (int j = 0; j < len; j++) - m.put(j, j); - testMap(m, schema); - } - for (int i = 0; i < 1000; i++) { - Schema schema = Schema.parse("(map string int)"); - Map m = new HashMap(); - int len = (int)Math.random() % 1000 + 1; - for (int j = 0; j < len; j++) - m.put(Integer.toString(j), j); - testMap(m, schema); - } - } - public void testMap(Map val, Schema schema) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - Object obj = unpackOne(out, schema); - if (obj instanceof Map) - assertTrue(val.equals(obj)); - else { - System.out.println("obj=" + obj); - System.out.println("Got unexpected class: " + obj.getClass()); - assertTrue(false); - } - } + @Test + public void testArray() throws Exception { + List emptyList = new ArrayList(); + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(emptyList); + MessagePackObject obj = unpackOne(out); + assertEquals(emptyList, obj.asList()); + } + + for (int i = 0; i < 1000; i++) { + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(l); + MessagePackObject obj = unpackOne(out); + List list = obj.asList(); + assertEquals(l.size(), list.size()); + for (int j = 0; j < len; j++) { + assertEquals(l.get(j).intValue(), list.get(j).asInt()); + } + } + + for (int i = 0; i < 1000; i++) { + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(Integer.toString(j)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(l); + MessagePackObject obj = unpackOne(out); + List list = obj.asList(); + assertEquals(l.size(), list.size()); + for (int j = 0; j < len; j++) { + assertEquals(l.get(j), list.get(j).asString()); + } + } + } + + @Test + public void testMap() throws Exception { + Map emptyMap = new HashMap(); + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(emptyMap); + MessagePackObject obj = unpackOne(out); + assertEquals(emptyMap, obj.asMap()); + } + + for (int i = 0; i < 1000; i++) { + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(j, j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(m); + MessagePackObject obj = unpackOne(out); + Map map = obj.asMap(); + assertEquals(m.size(), map.size()); + for (Map.Entry pair : map.entrySet()) { + Integer val = m.get(pair.getKey().asInt()); + assertNotNull(val); + assertEquals(val.intValue(), pair.getValue().asInt()); + } + } + + for (int i = 0; i < 1000; i++) { + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(Integer.toString(j), j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(m); + MessagePackObject obj = unpackOne(out); + Map map = obj.asMap(); + assertEquals(m.size(), map.size()); + for (Map.Entry pair : map.entrySet()) { + Integer val = m.get(pair.getKey().asString()); + assertNotNull(val); + assertEquals(val.intValue(), pair.getValue().asInt()); + } + } + } }; + From 5658ca5b903b294dcccad0dfa6fa6a7bcd9acc12 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 22:24:51 +0900 Subject: [PATCH 071/259] java: adds ObjectEquals test --- .../main/java/org/msgpack/UnpackerImpl.java | 30 +++--- .../java/org/msgpack/object/ArrayType.java | 6 +- .../msgpack/object/BigIntegerTypeIMPL.java | 2 +- .../java/org/msgpack/object/BooleanType.java | 6 +- .../msgpack/object/LongIntegerTypeIMPL.java | 2 +- .../main/java/org/msgpack/object/MapType.java | 6 +- .../main/java/org/msgpack/object/NilType.java | 6 ++ .../main/java/org/msgpack/object/RawType.java | 20 +++- .../java/org/msgpack/TestObjectEquals.java | 96 +++++++++++++++++++ 9 files changed, 152 insertions(+), 22 deletions(-) create mode 100644 java/src/test/java/org/msgpack/TestObjectEquals.java diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index f004e6c4..cfd3d22b 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -124,7 +124,7 @@ public class UnpackerImpl { if((b & 0xe0) == 0xa0) { // FixRaw trail = b & 0x1f; if(trail == 0) { - obj = new RawType(new byte[0]); + obj = RawType.create(new byte[0]); break _push; } cs = ACS_RAW_VALUE; @@ -139,7 +139,7 @@ public class UnpackerImpl { //System.out.println("fixarray count:"+count); obj = new MessagePackObject[count]; if(count == 0) { - obj = new ArrayType((MessagePackObject[])obj); + obj = ArrayType.create((MessagePackObject[])obj); break _push; } ++top; @@ -159,7 +159,7 @@ public class UnpackerImpl { count = b & 0x0f; obj = new MessagePackObject[count*2]; if(count == 0) { - obj = new MapType((MessagePackObject[])obj); + obj = MapType.create((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -175,13 +175,13 @@ public class UnpackerImpl { switch(b & 0xff) { // FIXME case 0xc0: // nil - obj = new NilType(); + obj = NilType.create(); break _push; case 0xc2: // false - obj = new BooleanType(false); + obj = BooleanType.create(false); break _push; case 0xc3: // true - obj = new BooleanType(true); + obj = BooleanType.create(true); break _push; case 0xca: // float case 0xcb: // double @@ -293,7 +293,7 @@ public class UnpackerImpl { castBuffer.put(src, n, 2); trail = ((int)castBuffer.getShort(0)) & 0xffff; if(trail == 0) { - obj = new RawType(new byte[0]); + obj = RawType.create(new byte[0]); break _push; } cs = ACS_RAW_VALUE; @@ -304,14 +304,14 @@ public class UnpackerImpl { // FIXME overflow check trail = castBuffer.getInt(0) & 0x7fffffff; if(trail == 0) { - obj = new RawType(new byte[0]); + obj = RawType.create(new byte[0]); break _push; } cs = ACS_RAW_VALUE; case ACS_RAW_VALUE: { byte[] raw = new byte[trail]; System.arraycopy(src, n, raw, 0, trail); - obj = new RawType(raw); + obj = RawType.create(raw); } break _push; case CS_ARRAY_16: @@ -323,7 +323,7 @@ public class UnpackerImpl { count = ((int)castBuffer.getShort(0)) & 0xffff; obj = new MessagePackObject[count]; if(count == 0) { - obj = new ArrayType((MessagePackObject[])obj); + obj = ArrayType.create((MessagePackObject[])obj); break _push; } ++top; @@ -344,7 +344,7 @@ public class UnpackerImpl { count = castBuffer.getInt(0) & 0x7fffffff; obj = new MessagePackObject[count]; if(count == 0) { - obj = new ArrayType((MessagePackObject[])obj); + obj = ArrayType.create((MessagePackObject[])obj); break _push; } ++top; @@ -364,7 +364,7 @@ public class UnpackerImpl { count = ((int)castBuffer.getShort(0)) & 0xffff; obj = new MessagePackObject[count*2]; if(count == 0) { - obj = new MapType((MessagePackObject[])obj); + obj = MapType.create((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -386,7 +386,7 @@ public class UnpackerImpl { count = castBuffer.getInt(0) & 0x7fffffff; obj = new MessagePackObject[count*2]; if(count == 0) { - obj = new MapType((MessagePackObject[])obj); + obj = MapType.create((MessagePackObject[])obj); break _push; } //System.out.println("fixmap count:"+count); @@ -425,7 +425,7 @@ public class UnpackerImpl { top_obj = stack_obj[top]; top_ct = stack_ct[top]; top_count = stack_count[top]; - obj = new ArrayType((MessagePackObject[])ar); + obj = ArrayType.create((MessagePackObject[])ar); stack_obj[top] = null; --top; break _push; @@ -447,7 +447,7 @@ public class UnpackerImpl { top_obj = stack_obj[top]; top_ct = stack_ct[top]; top_count = stack_count[top]; - obj = new MapType((MessagePackObject[])mp); + obj = MapType.create((MessagePackObject[])mp); stack_obj[top] = null; --top; break _push; diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java index 350ce32e..36134dc7 100644 --- a/java/src/main/java/org/msgpack/object/ArrayType.java +++ b/java/src/main/java/org/msgpack/object/ArrayType.java @@ -25,10 +25,14 @@ import org.msgpack.*; public class ArrayType extends MessagePackObject { private MessagePackObject[] array; - public ArrayType(MessagePackObject[] array) { + ArrayType(MessagePackObject[] array) { this.array = array; } + public static ArrayType create(MessagePackObject[] array) { + return new ArrayType(array); + } + @Override public boolean isArrayType() { return true; diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java index 7b060eef..b29879f4 100644 --- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java @@ -109,7 +109,7 @@ class BigIntegerTypeIMPL extends IntegerType { public boolean equals(Object obj) { if(obj.getClass() != getClass()) { if(obj.getClass() == ShortIntegerTypeIMPL.class) { - return BigInteger.valueOf((long)((ShortIntegerTypeIMPL)obj).shortValue()).equals(value); + return BigInteger.valueOf(((ShortIntegerTypeIMPL)obj).longValue()).equals(value); } else if(obj.getClass() == LongIntegerTypeIMPL.class) { return BigInteger.valueOf(((LongIntegerTypeIMPL)obj).longValue()).equals(value); } diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java index 1d12c1ce..65bd42b4 100644 --- a/java/src/main/java/org/msgpack/object/BooleanType.java +++ b/java/src/main/java/org/msgpack/object/BooleanType.java @@ -23,10 +23,14 @@ import org.msgpack.*; public class BooleanType extends MessagePackObject { private boolean value; - public BooleanType(boolean value) { + BooleanType(boolean value) { this.value = value; } + public static BooleanType create(boolean value) { + return new BooleanType(value); + } + @Override public boolean isBooleanType() { return true; diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java index 3928a299..d052e77a 100644 --- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java +++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java @@ -108,7 +108,7 @@ class LongIntegerTypeIMPL extends IntegerType { if(obj.getClass() == ShortIntegerTypeIMPL.class) { return value == ((ShortIntegerTypeIMPL)obj).longValue(); } else if(obj.getClass() == BigIntegerTypeIMPL.class) { - return (long)value == ((BigIntegerTypeIMPL)obj).longValue(); + return BigInteger.valueOf(value).equals(((BigIntegerTypeIMPL)obj).bigIntegerValue()); } return false; } diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java index 619d3887..148c0bf4 100644 --- a/java/src/main/java/org/msgpack/object/MapType.java +++ b/java/src/main/java/org/msgpack/object/MapType.java @@ -26,10 +26,14 @@ import org.msgpack.*; public class MapType extends MessagePackObject { private MessagePackObject[] map; - public MapType(MessagePackObject[] map) { + MapType(MessagePackObject[] map) { this.map = map; } + public static MapType create(MessagePackObject[] map) { + return new MapType(map); + } + @Override public boolean isMapType() { return true; diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java index ece62f0a..d0572f19 100644 --- a/java/src/main/java/org/msgpack/object/NilType.java +++ b/java/src/main/java/org/msgpack/object/NilType.java @@ -21,6 +21,12 @@ import java.io.IOException; import org.msgpack.*; public class NilType extends MessagePackObject { + private static NilType instance = new NilType(); + + public static NilType create() { + return instance; + } + @Override public boolean isNull() { return true; diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java index 3a394862..26f6e0d9 100644 --- a/java/src/main/java/org/msgpack/object/RawType.java +++ b/java/src/main/java/org/msgpack/object/RawType.java @@ -24,10 +24,26 @@ import org.msgpack.*; public class RawType extends MessagePackObject { private byte[] bytes; - public RawType(byte[] bytes) { + RawType(byte[] bytes) { this.bytes = bytes; } + RawType(String str) { + try { + this.bytes = str.getBytes("UTF-8"); + } catch (Exception e) { + throw new MessageTypeException("type error"); + } + } + + public static RawType create(byte[] bytes) { + return new RawType(bytes); + } + + public static RawType create(String str) { + return new RawType(str); + } + @Override public boolean isRawType() { return true; @@ -58,7 +74,7 @@ public class RawType extends MessagePackObject { if(obj.getClass() != getClass()) { return false; } - return ((RawType)obj).bytes.equals(bytes); + return Arrays.equals(((RawType)obj).bytes, bytes); } @Override diff --git a/java/src/test/java/org/msgpack/TestObjectEquals.java b/java/src/test/java/org/msgpack/TestObjectEquals.java new file mode 100644 index 00000000..b2b018d1 --- /dev/null +++ b/java/src/test/java/org/msgpack/TestObjectEquals.java @@ -0,0 +1,96 @@ +package org.msgpack; + +import org.msgpack.*; +import org.msgpack.object.*; +import java.math.BigInteger; +import java.util.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestObjectEquals { + public void testInt(int val) throws Exception { + MessagePackObject objInt = IntegerType.create(val); + MessagePackObject objLong = IntegerType.create((long)val); + MessagePackObject objBigInt = IntegerType.create(BigInteger.valueOf((long)val)); + assertTrue(objInt.equals(objInt)); + assertTrue(objInt.equals(objLong)); + assertTrue(objInt.equals(objBigInt)); + assertTrue(objLong.equals(objInt)); + assertTrue(objLong.equals(objLong)); + assertTrue(objLong.equals(objBigInt)); + assertTrue(objBigInt.equals(objInt)); + assertTrue(objBigInt.equals(objLong)); + assertTrue(objBigInt.equals(objBigInt)); + } + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } + + public void testLong(long val) throws Exception { + MessagePackObject objInt = IntegerType.create((int)val); + MessagePackObject objLong = IntegerType.create(val); + MessagePackObject objBigInt = IntegerType.create(BigInteger.valueOf(val)); + if(val > (long)Integer.MAX_VALUE || val < (long)Integer.MIN_VALUE) { + assertTrue(objInt.equals(objInt)); + assertFalse(objInt.equals(objLong)); + assertFalse(objInt.equals(objBigInt)); + assertFalse(objLong.equals(objInt)); + assertTrue(objLong.equals(objLong)); + assertTrue(objLong.equals(objBigInt)); + assertFalse(objBigInt.equals(objInt)); + assertTrue(objBigInt.equals(objLong)); + assertTrue(objBigInt.equals(objBigInt)); + } else { + assertTrue(objInt.equals(objInt)); + assertTrue(objInt.equals(objLong)); + assertTrue(objInt.equals(objBigInt)); + assertTrue(objLong.equals(objInt)); + assertTrue(objLong.equals(objLong)); + assertTrue(objLong.equals(objBigInt)); + assertTrue(objBigInt.equals(objInt)); + assertTrue(objBigInt.equals(objLong)); + assertTrue(objBigInt.equals(objBigInt)); + } + } + @Test + public void testLong() throws Exception { + testLong(0); + testLong(-1); + testLong(1); + testLong(Integer.MIN_VALUE); + testLong(Integer.MAX_VALUE); + testLong(Long.MIN_VALUE); + testLong(Long.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testLong(rand.nextLong()); + } + + @Test + public void testNil() throws Exception { + assertTrue(NilType.create().equals(NilType.create())); + assertFalse(NilType.create().equals(IntegerType.create(0))); + assertFalse(NilType.create().equals(BooleanType.create(false))); + } + + public void testString(String str) throws Exception { + assertTrue(RawType.create(str).equals(RawType.create(str))); + } + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); + } +} + From fdfabc9f8862da5982830017e21b0628a00356eb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 22:51:23 +0900 Subject: [PATCH 072/259] java: adds cross-language test case --- .../main/java/org/msgpack/UnpackerImpl.java | 1 + java/src/test/java/org/msgpack/TestCases.java | 46 ++++++++++++++++++ java/src/test/resources/cases.json | 1 + java/src/test/resources/cases.mpac | Bin 0 -> 213 bytes java/src/test/resources/cases_compact.mpac | Bin 0 -> 116 bytes 5 files changed, 48 insertions(+) create mode 100644 java/src/test/java/org/msgpack/TestCases.java create mode 100644 java/src/test/resources/cases.json create mode 100644 java/src/test/resources/cases.mpac create mode 100644 java/src/test/resources/cases_compact.mpac diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index cfd3d22b..d4f99e38 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -308,6 +308,7 @@ public class UnpackerImpl { break _push; } cs = ACS_RAW_VALUE; + break _fixed_trail_again; case ACS_RAW_VALUE: { byte[] raw = new byte[trail]; System.arraycopy(src, n, raw, 0, trail); diff --git a/java/src/test/java/org/msgpack/TestCases.java b/java/src/test/java/org/msgpack/TestCases.java new file mode 100644 index 00000000..632645ff --- /dev/null +++ b/java/src/test/java/org/msgpack/TestCases.java @@ -0,0 +1,46 @@ +package org.msgpack; + +import java.io.*; +import java.util.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestCases { + public void feedFile(Unpacker pac, String path) throws Exception { + FileInputStream input = new FileInputStream(path); + byte[] buffer = new byte[32*1024]; + while(true) { + int count = input.read(buffer); + if(count < 0) { + break; + } + pac.feed(buffer, 0, count); + } + } + + @Test + public void testCases() throws Exception { + System.out.println( new File(".").getAbsoluteFile().getParent() ); + + + Unpacker pac = new Unpacker(); + Unpacker pac_compact = new Unpacker(); + + feedFile(pac, "src/test/resources/cases.mpac"); + feedFile(pac_compact, "src/test/resources/cases_compact.mpac"); + + UnpackResult result = new UnpackResult(); + while(pac.next(result)) { + UnpackResult result_compact = new UnpackResult(); + assertTrue( pac_compact.next(result_compact) ); + System.out.println("obj: "+result_compact.getData()); + if(!result.getData().equals(result_compact.getData())) { + System.out.println("compact: "+result_compact.getData().asString()); + System.out.println("data : "+result.getData().asString()); + } + assertTrue( result.getData().equals(result_compact.getData()) ); + } + } +}; + diff --git a/java/src/test/resources/cases.json b/java/src/test/resources/cases.json new file mode 100644 index 00000000..fd390d48 --- /dev/null +++ b/java/src/test/resources/cases.json @@ -0,0 +1 @@ +[false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,65535,4294967295,-32,-32,-128,-32768,-2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",[0],[0],[0],[],[],[],{},{},{},{"a":97},{"a":97},{"a":97},[[]],[["a"]]] \ No newline at end of file diff --git a/java/src/test/resources/cases.mpac b/java/src/test/resources/cases.mpac new file mode 100644 index 0000000000000000000000000000000000000000..5ec08c6a9af42d9568eb429a209a639616e94711 GIT binary patch literal 213 zcmXYp!4<+V3`3R4n8lOSY|v~#CV>aXw2-v7Qc6c)17ihrkbe}**V_dHM&J(DgGLop zU?R;l%8FI9$y_sy>V|HFdDW~{neAn-roN|Wd+OcH160;F91fo! Date: Wed, 18 Aug 2010 22:55:50 +0900 Subject: [PATCH 073/259] java: fixes cross-language test case --- java/src/test/java/org/msgpack/TestCases.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/java/src/test/java/org/msgpack/TestCases.java b/java/src/test/java/org/msgpack/TestCases.java index 632645ff..c368972f 100644 --- a/java/src/test/java/org/msgpack/TestCases.java +++ b/java/src/test/java/org/msgpack/TestCases.java @@ -21,9 +21,6 @@ public class TestCases { @Test public void testCases() throws Exception { - System.out.println( new File(".").getAbsoluteFile().getParent() ); - - Unpacker pac = new Unpacker(); Unpacker pac_compact = new Unpacker(); @@ -34,13 +31,10 @@ public class TestCases { while(pac.next(result)) { UnpackResult result_compact = new UnpackResult(); assertTrue( pac_compact.next(result_compact) ); - System.out.println("obj: "+result_compact.getData()); - if(!result.getData().equals(result_compact.getData())) { - System.out.println("compact: "+result_compact.getData().asString()); - System.out.println("data : "+result.getData().asString()); - } assertTrue( result.getData().equals(result_compact.getData()) ); } + + assertFalse( pac_compact.next(result) ); } }; From 40dc9de6c9bc90a95e55d5d2999f7e45d34cabc8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 23:37:47 +0900 Subject: [PATCH 074/259] java: supports packing/unpacking of BigInteger less than 0xffffffffffffffff --- .../java/org/msgpack/MessageConvertable.java | 2 +- java/src/main/java/org/msgpack/Packer.java | 22 ++++++++------- .../main/java/org/msgpack/UnpackerImpl.java | 5 ++-- .../test/java/org/msgpack/TestPackUnpack.java | 27 +++++++++++++++++++ 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/java/src/main/java/org/msgpack/MessageConvertable.java b/java/src/main/java/org/msgpack/MessageConvertable.java index da251dc1..8acf1f28 100644 --- a/java/src/main/java/org/msgpack/MessageConvertable.java +++ b/java/src/main/java/org/msgpack/MessageConvertable.java @@ -18,6 +18,6 @@ package org.msgpack; public interface MessageConvertable { - public void messageConvert(Object obj) throws MessageTypeException; + public void messageConvert(MessagePackObject obj) throws MessageTypeException; } diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 98af3d64..60e04bfe 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -196,19 +196,19 @@ public class Packer { } public Packer packBigInteger(BigInteger d) throws IOException { - if(d.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { + if(d.bitLength() <= 63) { return packLong(d.longValue()); - } else if(d.bitLength() <= 64) { + } else if(d.bitLength() <= 64 && d.signum() >= 0) { castBytes[0] = (byte)0xcf; byte[] barray = d.toByteArray(); - castBytes[1] = barray[0]; - castBytes[2] = barray[1]; - castBytes[3] = barray[2]; - castBytes[4] = barray[3]; - castBytes[5] = barray[4]; - castBytes[6] = barray[5]; - castBytes[7] = barray[6]; - castBytes[8] = barray[7]; + castBytes[1] = barray[barray.length-8]; + castBytes[2] = barray[barray.length-7]; + castBytes[3] = barray[barray.length-6]; + castBytes[4] = barray[barray.length-5]; + castBytes[5] = barray[barray.length-4]; + castBytes[6] = barray[barray.length-3]; + castBytes[7] = barray[barray.length-2]; + castBytes[8] = barray[barray.length-1]; out.write(castBytes); return this; } else { @@ -436,6 +436,8 @@ public class Packer { return packFloat((Float)o); } else if(o instanceof Double) { return packDouble((Double)o); + } else if(o instanceof BigInteger) { + return packBigInteger((BigInteger)o); } else { throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")"); } diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index d4f99e38..6a7085fb 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -18,6 +18,7 @@ package org.msgpack; import java.nio.ByteBuffer; +import java.math.BigInteger; import org.msgpack.object.*; public class UnpackerImpl { @@ -262,9 +263,7 @@ public class UnpackerImpl { { long o = castBuffer.getLong(0); if(o < 0) { - // FIXME - //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); - throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported"); + obj = IntegerType.create(new BigInteger(1, castBuffer.array())); } else { obj = IntegerType.create(o); } diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index ca3d235c..75c5fe74 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -3,6 +3,7 @@ package org.msgpack; import org.msgpack.*; import java.io.*; import java.util.*; +import java.math.BigInteger; import org.junit.Test; import static org.junit.Assert.*; @@ -36,6 +37,32 @@ public class TestPackUnpack { testInt(rand.nextInt()); } + public void testBigInteger(BigInteger val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + if(!val.equals(obj.asBigInteger())) { + System.out.println("expect: "+val); + System.out.println("but : "+obj.asBigInteger()); + } + assertEquals(val, obj.asBigInteger()); + } + @Test + public void testBigInteger() throws Exception { + testBigInteger(BigInteger.valueOf(0)); + testBigInteger(BigInteger.valueOf(-1)); + testBigInteger(BigInteger.valueOf(1)); + testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE)); + testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE)); + testBigInteger(BigInteger.valueOf(Long.MIN_VALUE)); + testBigInteger(BigInteger.valueOf(Long.MAX_VALUE)); + BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63); + testBigInteger(max); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) ); + } + public void testFloat(float val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); From 48da2b8353a640de6bdc4e59c7fe91b640321b27 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 18 Aug 2010 23:47:31 +0900 Subject: [PATCH 075/259] java: adds Packer.pack() --- java/src/main/java/org/msgpack/Packer.java | 103 ++++++++++++------ .../test/java/org/msgpack/TestPackUnpack.java | 24 +++- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 60e04bfe..0ac8d89e 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -315,39 +315,36 @@ public class Packer { } - public Packer pack(String o) throws IOException { - if(o == null) { return packNil(); } - return packString(o); - } - - public Packer pack(MessagePackable o) throws IOException { - if(o == null) { return packNil(); } - o.messagePack(this); - return this; - } - - public Packer pack(byte[] o) throws IOException { - if(o == null) { return packNil(); } - packRaw(o.length); - return packRawBody(o); - } - - public Packer pack(List o) throws IOException { - if(o == null) { return packNil(); } - packArray(o.size()); - for(Object i : o) { pack(i); } - return this; - } - - @SuppressWarnings("unchecked") - public Packer pack(Map o) throws IOException { - if(o == null) { return packNil(); } - packMap(o.size()); - for(Map.Entry e : ((Map)o).entrySet()) { - pack(e.getKey()); - pack(e.getValue()); + public Packer pack(boolean o) throws IOException { + if(o) { + return packTrue(); + } else { + return packFalse(); } - return this; + } + + public Packer pack(byte o) throws IOException { + return packByte(o); + } + + public Packer pack(short o) throws IOException { + return packShort(o); + } + + public Packer pack(int o) throws IOException { + return packInt(o); + } + + public Packer pack(long o) throws IOException { + return packLong(o); + } + + public Packer pack(float o) throws IOException { + return packFloat(o); + } + + public Packer pack(double o) throws IOException { + return packDouble(o); } public Packer pack(Boolean o) throws IOException { @@ -379,6 +376,11 @@ public class Packer { return packLong(o); } + public Packer pack(BigInteger o) throws IOException { + if(o == null) { return packNil(); } + return packBigInteger(o); + } + public Packer pack(Float o) throws IOException { if(o == null) { return packNil(); } return packFloat(o); @@ -389,8 +391,41 @@ public class Packer { return packDouble(o); } + public Packer pack(String o) throws IOException { + if(o == null) { return packNil(); } + return packString(o); + } + + public Packer pack(MessagePackable o) throws IOException { + if(o == null) { return packNil(); } + o.messagePack(this); + return this; + } + + public Packer pack(byte[] o) throws IOException { + if(o == null) { return packNil(); } + packRaw(o.length); + return packRawBody(o); + } + + public Packer pack(List o) throws IOException { + if(o == null) { return packNil(); } + packArray(o.size()); + for(Object i : o) { pack(i); } + return this; + } + + public Packer pack(Map o) throws IOException { + if(o == null) { return packNil(); } + packMap(o.size()); + for(Map.Entry e : ((Map)o).entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } + - @SuppressWarnings("unchecked") public Packer pack(Object o) throws IOException { if(o == null) { return packNil(); @@ -413,7 +448,7 @@ public class Packer { } else if(o instanceof Map) { Map m = (Map)o; packMap(m.size()); - for(Map.Entry e : m.entrySet()) { + for(Map.Entry e : m.entrySet()) { pack(e.getKey()); pack(e.getValue()); } diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index 75c5fe74..81636783 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -37,14 +37,30 @@ public class TestPackUnpack { testInt(rand.nextInt()); } + public void testLong(long val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asLong()); + } + @Test + public void testLong() throws Exception { + testLong(0); + testLong(-1); + testLong(1); + testLong(Integer.MIN_VALUE); + testLong(Integer.MAX_VALUE); + testLong(Long.MIN_VALUE); + testLong(Long.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testLong(rand.nextLong()); + } + public void testBigInteger(BigInteger val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); MessagePackObject obj = unpackOne(out); - if(!val.equals(obj.asBigInteger())) { - System.out.println("expect: "+val); - System.out.println("but : "+obj.asBigInteger()); - } assertEquals(val, obj.asBigInteger()); } @Test From 193a739749687e45a71a6821e625815587c4c4bf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 19 Aug 2010 00:05:48 +0900 Subject: [PATCH 076/259] java: updates TestDirectConversion --- cpp/test/cases.cc | 2 + .../org/msgpack/TestDirectConversion.java | 369 +++++++++++------- .../java/org/msgpack/TestObjectEquals.java | 48 +-- .../test/java/org/msgpack/TestPackUnpack.java | 54 +-- 4 files changed, 292 insertions(+), 181 deletions(-) diff --git a/cpp/test/cases.cc b/cpp/test/cases.cc index b408876c..eb1286c7 100644 --- a/cpp/test/cases.cc +++ b/cpp/test/cases.cc @@ -32,5 +32,7 @@ TEST(cases, format) EXPECT_TRUE( pac_compact.next(&result_compact) ); EXPECT_EQ(result_compact.get(), result.get()); } + + EXPECT_FALSE( pac_compact.next(&result) ); } diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java index 77bbc585..dbacf012 100644 --- a/java/src/test/java/org/msgpack/TestDirectConversion.java +++ b/java/src/test/java/org/msgpack/TestDirectConversion.java @@ -8,141 +8,248 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestDirectConversion { - @Test - public void testInt() throws Exception { - testInt(0); - testInt(-1); - testInt(1); - testInt(Integer.MIN_VALUE); - testInt(Integer.MAX_VALUE); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testInt(rand.nextInt()); - } - public void testInt(int val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - assertEquals(val, upk.unpackInt()); - } + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } + public void testInt(int val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackInt()); + } - @Test - public void testFloat() throws Exception { - testFloat((float)0.0); - testFloat((float)-0.0); - testFloat((float)1.0); - testFloat((float)-1.0); - testFloat((float)Float.MAX_VALUE); - testFloat((float)Float.MIN_VALUE); - testFloat((float)Float.NaN); - testFloat((float)Float.NEGATIVE_INFINITY); - testFloat((float)Float.POSITIVE_INFINITY); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testFloat(rand.nextFloat()); - } - public void testFloat(float val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - float f = upk.unpackFloat(); - if(Float.isNaN(val)) { - assertTrue(Float.isNaN(f)); - } else { - assertEquals(val, f, 10e-10); - } - } + @Test + public void testLong() throws Exception { + testLong(0); + testLong(-1); + testLong(1); + testLong(Integer.MIN_VALUE); + testLong(Integer.MAX_VALUE); + testLong(Long.MIN_VALUE); + testLong(Long.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testLong(rand.nextLong()); + } + public void testLong(long val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackLong()); + } - @Test - public void testDouble() throws Exception { - testDouble((double)0.0); - testDouble((double)-0.0); - testDouble((double)1.0); - testDouble((double)-1.0); - testDouble((double)Double.MAX_VALUE); - testDouble((double)Double.MIN_VALUE); - testDouble((double)Double.NaN); - testDouble((double)Double.NEGATIVE_INFINITY); - testDouble((double)Double.POSITIVE_INFINITY); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testDouble(rand.nextDouble()); - } - public void testDouble(double val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - double f = upk.unpackDouble(); - if(Double.isNaN(val)) { - assertTrue(Double.isNaN(f)); - } else { - assertEquals(val, f, 10e-10); - } - } + @Test + public void testFloat() throws Exception { + testFloat((float)0.0); + testFloat((float)-0.0); + testFloat((float)1.0); + testFloat((float)-1.0); + testFloat((float)Float.MAX_VALUE); + testFloat((float)Float.MIN_VALUE); + testFloat((float)Float.NaN); + testFloat((float)Float.NEGATIVE_INFINITY); + testFloat((float)Float.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testFloat(rand.nextFloat()); + } + public void testFloat(float val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackFloat(), 10e-10); + } - @Test - public void testNil() throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).packNil(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - assertEquals(null, upk.unpackNull()); - } + @Test + public void testDouble() throws Exception { + testDouble((double)0.0); + testDouble((double)-0.0); + testDouble((double)1.0); + testDouble((double)-1.0); + testDouble((double)Double.MAX_VALUE); + testDouble((double)Double.MIN_VALUE); + testDouble((double)Double.NaN); + testDouble((double)Double.NEGATIVE_INFINITY); + testDouble((double)Double.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testDouble(rand.nextDouble()); + } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackDouble(), 10e-10); + } - @Test - public void testBoolean() throws Exception { - testBoolean(false); - testBoolean(true); - } - public void testBoolean(boolean val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - assertEquals(val, upk.unpackBoolean()); - } + @Test + public void testNil() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).packNil(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(null, pac.unpackNull()); + } - @Test - public void testString() throws Exception { - testString(""); - testString("a"); - testString("ab"); - testString("abc"); - // small size string - for (int i = 0; i < 100; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 31 + 1; - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - // medium size string - for (int i = 0; i < 100; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 100 + (1 << 15); - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - // large size string - for (int i = 0; i < 10; i++) { - StringBuilder sb = new StringBuilder(); - int len = (int)Math.random() % 100 + (1 << 31); - for (int j = 0; j < len; j++) - sb.append('a' + ((int)Math.random()) & 26); - testString(sb.toString()); - } - } - public void testString(String val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - assertEquals(val, upk.unpackString()); - } + @Test + public void testBoolean() throws Exception { + testBoolean(false); + testBoolean(true); + } + public void testBoolean(boolean val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackBoolean()); + } - // FIXME container types + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); + + // small size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 31 + 1; + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + + // medium size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 15); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + + // large size string + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 31); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackString()); + } + + @Test + public void testArray() throws Exception { + List emptyList = new ArrayList(); + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(emptyList); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackArray(); + assertEquals(0, ulen); + } + + for (int i = 0; i < 1000; i++) { + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(l); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackArray(); + assertEquals(len, ulen); + for (int j = 0; j < len; j++) { + assertEquals(l.get(j).intValue(), pac.unpackInt()); + } + } + + for (int i = 0; i < 1000; i++) { + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(Integer.toString(j)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(l); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackArray(); + assertEquals(len, ulen); + for (int j = 0; j < len; j++) { + assertEquals(l.get(j), pac.unpackString()); + } + } + } + + @Test + public void testMap() throws Exception { + Map emptyMap = new HashMap(); + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(emptyMap); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackMap(); + assertEquals(0, ulen); + } + + for (int i = 0; i < 1000; i++) { + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(j, j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(m); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackMap(); + assertEquals(len, ulen); + for (int j = 0; j < len; j++) { + Integer val = m.get(pac.unpackInt()); + assertNotNull(val); + assertEquals(val.intValue(), pac.unpackInt()); + } + } + + for (int i = 0; i < 1000; i++) { + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(Integer.toString(j), j); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(m); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + int ulen = pac.unpackMap(); + assertEquals(len, ulen); + for (int j = 0; j < len; j++) { + Integer val = m.get(pac.unpackString()); + assertNotNull(val); + assertEquals(val.intValue(), pac.unpackInt()); + } + } + } }; + diff --git a/java/src/test/java/org/msgpack/TestObjectEquals.java b/java/src/test/java/org/msgpack/TestObjectEquals.java index b2b018d1..25fe9271 100644 --- a/java/src/test/java/org/msgpack/TestObjectEquals.java +++ b/java/src/test/java/org/msgpack/TestObjectEquals.java @@ -9,6 +9,17 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestObjectEquals { + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } public void testInt(int val) throws Exception { MessagePackObject objInt = IntegerType.create(val); MessagePackObject objLong = IntegerType.create((long)val); @@ -23,18 +34,20 @@ public class TestObjectEquals { assertTrue(objBigInt.equals(objLong)); assertTrue(objBigInt.equals(objBigInt)); } + @Test - public void testInt() throws Exception { - testInt(0); - testInt(-1); - testInt(1); - testInt(Integer.MIN_VALUE); - testInt(Integer.MAX_VALUE); + public void testLong() throws Exception { + testLong(0); + testLong(-1); + testLong(1); + testLong(Integer.MIN_VALUE); + testLong(Integer.MAX_VALUE); + testLong(Long.MIN_VALUE); + testLong(Long.MAX_VALUE); Random rand = new Random(); for (int i = 0; i < 1000; i++) - testInt(rand.nextInt()); + testLong(rand.nextLong()); } - public void testLong(long val) throws Exception { MessagePackObject objInt = IntegerType.create((int)val); MessagePackObject objLong = IntegerType.create(val); @@ -61,19 +74,6 @@ public class TestObjectEquals { assertTrue(objBigInt.equals(objBigInt)); } } - @Test - public void testLong() throws Exception { - testLong(0); - testLong(-1); - testLong(1); - testLong(Integer.MIN_VALUE); - testLong(Integer.MAX_VALUE); - testLong(Long.MIN_VALUE); - testLong(Long.MAX_VALUE); - Random rand = new Random(); - for (int i = 0; i < 1000; i++) - testLong(rand.nextLong()); - } @Test public void testNil() throws Exception { @@ -82,9 +82,6 @@ public class TestObjectEquals { assertFalse(NilType.create().equals(BooleanType.create(false))); } - public void testString(String str) throws Exception { - assertTrue(RawType.create(str).equals(RawType.create(str))); - } @Test public void testString() throws Exception { testString(""); @@ -92,5 +89,8 @@ public class TestObjectEquals { testString("ab"); testString("abc"); } + public void testString(String str) throws Exception { + assertTrue(RawType.create(str).equals(RawType.create(str))); + } } diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index 81636783..7edd4116 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -11,20 +11,14 @@ import static org.junit.Assert.*; public class TestPackUnpack { public MessagePackObject unpackOne(ByteArrayOutputStream out) { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - Iterator it = upk.iterator(); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); assertEquals(true, it.hasNext()); MessagePackObject obj = it.next(); assertEquals(false, it.hasNext()); return obj; } - public void testInt(int val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asInt()); - } @Test public void testInt() throws Exception { testInt(0); @@ -36,13 +30,13 @@ public class TestPackUnpack { for (int i = 0; i < 1000; i++) testInt(rand.nextInt()); } - - public void testLong(long val) throws Exception { + public void testInt(int val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asLong()); + assertEquals(val, obj.asInt()); } + @Test public void testLong() throws Exception { testLong(0); @@ -56,13 +50,13 @@ public class TestPackUnpack { for (int i = 0; i < 1000; i++) testLong(rand.nextLong()); } - - public void testBigInteger(BigInteger val) throws Exception { + public void testLong(long val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asBigInteger()); + assertEquals(val, obj.asLong()); } + @Test public void testBigInteger() throws Exception { testBigInteger(BigInteger.valueOf(0)); @@ -78,13 +72,13 @@ public class TestPackUnpack { for (int i = 0; i < 1000; i++) testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) ); } - - public void testFloat(float val) throws Exception { + public void testBigInteger(BigInteger val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asFloat(), 10e-10); + assertEquals(val, obj.asBigInteger()); } + @Test public void testFloat() throws Exception { testFloat((float)0.0); @@ -100,13 +94,14 @@ public class TestPackUnpack { for (int i = 0; i < 1000; i++) testFloat(rand.nextFloat()); } - - public void testDouble(double val) throws Exception { + public void testFloat(float val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asDouble(), 10e-10); + float f = obj.asFloat(); + assertEquals(val, f, 10e-10); } + @Test public void testDouble() throws Exception { testDouble((double)0.0); @@ -122,6 +117,13 @@ public class TestPackUnpack { for (int i = 0; i < 1000; i++) testDouble(rand.nextDouble()); } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + double f = obj.asDouble(); + assertEquals(val, f, 10e-10); + } @Test public void testNil() throws Exception { @@ -143,12 +145,6 @@ public class TestPackUnpack { assertEquals(val, obj.asBoolean()); } - public void testString(String val) throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new Packer(out).pack(val); - MessagePackObject obj = unpackOne(out); - assertEquals(val, obj.asString()); - } @Test public void testString() throws Exception { testString(""); @@ -183,6 +179,12 @@ public class TestPackUnpack { testString(sb.toString()); } } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + MessagePackObject obj = unpackOne(out); + assertEquals(val, obj.asString()); + } @Test public void testArray() throws Exception { From 1d17836b7d2eb4e5e0f25d2466df70e137642061 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 19 Aug 2010 00:54:23 +0900 Subject: [PATCH 077/259] java: NilType::create() returns NilType's one and only instance --- java/src/main/java/org/msgpack/MessagePackObject.java | 2 +- java/src/main/java/org/msgpack/object/NilType.java | 10 ++++++---- java/src/test/java/org/msgpack/TestPackUnpack.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java index 6181f7a3..2424446f 100644 --- a/java/src/main/java/org/msgpack/MessagePackObject.java +++ b/java/src/main/java/org/msgpack/MessagePackObject.java @@ -23,7 +23,7 @@ import java.util.Map; import java.math.BigInteger; public abstract class MessagePackObject implements Cloneable, MessagePackable { - public boolean isNull() { + public boolean isNil() { return false; } diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java index d0572f19..c443db18 100644 --- a/java/src/main/java/org/msgpack/object/NilType.java +++ b/java/src/main/java/org/msgpack/object/NilType.java @@ -21,14 +21,16 @@ import java.io.IOException; import org.msgpack.*; public class NilType extends MessagePackObject { - private static NilType instance = new NilType(); + private final static NilType INSTANCE = new NilType(); public static NilType create() { - return instance; + return INSTANCE; } + private NilType() { } + @Override - public boolean isNull() { + public boolean isNil() { return true; } @@ -52,7 +54,7 @@ public class NilType extends MessagePackObject { @Override public Object clone() { - return new NilType(); + return INSTANCE; } } diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index 7edd4116..494c8a8a 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -130,7 +130,7 @@ public class TestPackUnpack { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).packNil(); MessagePackObject obj = unpackOne(out); - assertTrue(obj.isNull()); + assertTrue(obj.isNil()); } @Test From b4c98584db002cbb4d3960b6ed025212248dcce8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 19 Aug 2010 01:19:34 +0900 Subject: [PATCH 078/259] java: adds Unpacker.unpackBigInteger() --- .../org/msgpack/BufferedUnpackerImpl.java | 25 ++++++++++++++++--- java/src/main/java/org/msgpack/Unpacker.java | 10 ++++++++ .../org/msgpack/TestDirectConversion.java | 24 ++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index 9496238f..5b449c79 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -19,7 +19,7 @@ package org.msgpack; import java.io.IOException; import java.nio.ByteBuffer; -//import java.math.BigInteger; +import java.math.BigInteger; abstract class BufferedUnpackerImpl extends UnpackerImpl { int offset = 0; @@ -198,8 +198,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { { long o = castBuffer.getLong(0); if(o < 0) { - // FIXME unpackBigInteger - throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported"); + throw new MessageTypeException(); } advance(9); return o; @@ -231,7 +230,25 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - // FIXME unpackBigInteger + final BigInteger unpackBigInteger() throws IOException, MessageTypeException { + more(1); + int b = buffer[offset]; + if((b & 0xff) != 0xcf) { + return BigInteger.valueOf(unpackLong()); + } + + // unsigned int 64 + more(9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + advance(9); + long o = castBuffer.getLong(0); + if(o < 0) { + return new BigInteger(1, castBuffer.array()); + } else { + return BigInteger.valueOf(o); + } + } final float unpackFloat() throws IOException, MessageTypeException { more(1); diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 3a952438..3cae502e 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.IOException; import java.util.Iterator; import java.nio.ByteBuffer; +import java.math.BigInteger; /** * Unpacker enables you to deserialize objects from stream. @@ -452,6 +453,15 @@ public class Unpacker implements Iterable { return impl.unpackLong(); } + /** + * Gets one {@code BigInteger} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code BigInteger}. + */ + public BigInteger unpackBigInteger() throws IOException, MessageTypeException { + return impl.unpackBigInteger(); + } + /** * Gets one {@code float} value from the buffer. * This method calls {@link fill()} method if needed. diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java index dbacf012..1822ecb4 100644 --- a/java/src/test/java/org/msgpack/TestDirectConversion.java +++ b/java/src/test/java/org/msgpack/TestDirectConversion.java @@ -3,6 +3,7 @@ package org.msgpack; import org.msgpack.*; import java.io.*; import java.util.*; +import java.math.BigInteger; import org.junit.Test; import static org.junit.Assert.*; @@ -48,6 +49,29 @@ public class TestDirectConversion { assertEquals(val, pac.unpackLong()); } + @Test + public void testBigInteger() throws Exception { + testBigInteger(BigInteger.valueOf(0)); + testBigInteger(BigInteger.valueOf(-1)); + testBigInteger(BigInteger.valueOf(1)); + testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE)); + testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE)); + testBigInteger(BigInteger.valueOf(Long.MIN_VALUE)); + testBigInteger(BigInteger.valueOf(Long.MAX_VALUE)); + BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63); + testBigInteger(max); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) ); + } + public void testBigInteger(BigInteger val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + assertEquals(val, pac.unpackBigInteger()); + } + @Test public void testFloat() throws Exception { testFloat((float)0.0); From c8e351b31e28042c10f7406238636c92a64a696c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 21 Aug 2010 03:36:44 +0900 Subject: [PATCH 079/259] java: adds TestMessageUnpackable test --- java/src/main/java/org/msgpack/Packer.java | 2 +- java/src/test/java/org/msgpack/Image.java | 50 +++++++++++++++++++ .../org/msgpack/TestMessageUnpackable.java | 34 +++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 java/src/test/java/org/msgpack/Image.java create mode 100644 java/src/test/java/org/msgpack/TestMessageUnpackable.java diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 0ac8d89e..139b3b15 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -212,7 +212,7 @@ public class Packer { out.write(castBytes); return this; } else { - throw new MessageTypeException("can't BigInteger larger than 0xffffffffffffffff"); + throw new MessageTypeException("can't pack BigInteger larger than 0xffffffffffffffff"); } } diff --git a/java/src/test/java/org/msgpack/Image.java b/java/src/test/java/org/msgpack/Image.java new file mode 100644 index 00000000..3fcfe388 --- /dev/null +++ b/java/src/test/java/org/msgpack/Image.java @@ -0,0 +1,50 @@ +package org.msgpack; + +import org.msgpack.*; +import java.io.*; +import java.util.*; + +public class Image implements MessagePackable, MessageUnpackable { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + pk.packArray(5); + pk.pack(uri); + pk.pack(title); + pk.pack(width); + pk.pack(height); + pk.pack(size); + } + + public void messageUnpack(Unpacker pac) throws IOException, MessageTypeException { + int length = pac.unpackArray(); + if(length != 5) { + throw new MessageTypeException(); + } + uri = pac.unpackString(); + title = pac.unpackString(); + width = pac.unpackInt(); + height = pac.unpackInt(); + size = pac.unpackInt(); + } + + public boolean equals(Image obj) { + return uri.equals(obj.uri) && + title.equals(obj.title) && + width == obj.width && + height == obj.height && + size == obj.size; + } + + public boolean equals(Object obj) { + if(obj.getClass() != Image.class) { + return false; + } + return equals((Image)obj); + } +} + diff --git a/java/src/test/java/org/msgpack/TestMessageUnpackable.java b/java/src/test/java/org/msgpack/TestMessageUnpackable.java new file mode 100644 index 00000000..9099f910 --- /dev/null +++ b/java/src/test/java/org/msgpack/TestMessageUnpackable.java @@ -0,0 +1,34 @@ +package org.msgpack; + +import org.msgpack.*; +import java.io.*; +import java.util.*; +import java.math.BigInteger; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestMessageUnpackable { + @Test + public void testImage() throws Exception { + Image src = new Image(); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + + Image dst = new Image(); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + + dst.messageUnpack(pac); + + assertEquals(src, dst); + } +} + From a91c1ec6d9b86a3f8f71504e0889191da13a210e Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 21 Aug 2010 16:02:23 +0900 Subject: [PATCH 080/259] fixed segv on cyclic reference(patch by dankogai) --- perl/pack.c | 28 +++++++++++++++++----------- perl/t/08_cycle.t | 25 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 perl/t/08_cycle.t diff --git a/perl/pack.c b/perl/pack.c index 589cee8b..af6669cd 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -49,6 +49,8 @@ static void need(enc_t *enc, STRLEN len); # error "msgpack only supports IVSIZE = 8,4,2 environment." #endif +#define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)" + static void need(enc_t *enc, STRLEN len) { @@ -146,9 +148,10 @@ static int try_int(enc_t* enc, const char *p, size_t len) { } -static void _msgpack_pack_rv(enc_t *enc, SV* sv); +static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); -static void _msgpack_pack_sv(enc_t *enc, SV* sv) { +static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { + if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); if (sv==NULL) { @@ -171,7 +174,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) { } else if (SvIOKp(sv)) { PACK_IV(enc, SvIV(sv)); } else if (SvROK(sv)) { - _msgpack_pack_rv(enc, SvRV(sv)); + _msgpack_pack_rv(enc, SvRV(sv), depth-1); } else if (!SvOK(sv)) { msgpack_pack_nil(enc); } else if (isGV(sv)) { @@ -182,8 +185,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) { } } -static void _msgpack_pack_rv(enc_t *enc, SV* sv) { +static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; + if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); svt = SvTYPE(sv); @@ -207,8 +211,8 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv) { msgpack_pack_map(enc, count); while (he = hv_iternext(hval)) { - _msgpack_pack_sv(enc, hv_iterkeysv(he)); - _msgpack_pack_sv(enc, HeVAL(he)); + _msgpack_pack_sv(enc, hv_iterkeysv(he), depth); + _msgpack_pack_sv(enc, HeVAL(he), depth); } } else if (svt == SVt_PVAV) { AV* ary = (AV*)sv; @@ -218,7 +222,7 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv) { for (i=0; ipack($dat)"); + if (items < 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat [,$max_depth])"); } - SV* val = ST(1); + SV* val = ST(1); + int depth = 512; + if (items >= 3) depth = SvIV(ST(2)); enc_t enc; enc.sv = sv_2mortal(NEWSV(0, INIT_SIZE)); @@ -256,7 +262,7 @@ XS(xs_pack) { enc.end = SvEND(enc.sv); SvPOK_only(enc.sv); - _msgpack_pack_sv(&enc, val); + _msgpack_pack_sv(&enc, val, depth); SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv)); *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */ diff --git a/perl/t/08_cycle.t b/perl/t/08_cycle.t new file mode 100644 index 00000000..55d8427a --- /dev/null +++ b/perl/t/08_cycle.t @@ -0,0 +1,25 @@ +use t::Util; +use Test::More; +use Data::MessagePack; + +plan tests => 5; + +my $aref = [0]; +$aref->[1] = $aref; +eval { Data::MessagePack->pack($aref) }; +ok $@, $@; + +my $href = {}; +$href->{cycle} = $href; +eval { Data::MessagePack->pack($aref) }; +ok $@, $@; + +$aref = [0,[1,2]]; +eval { Data::MessagePack->pack($aref) }; +ok !$@; + +eval { Data::MessagePack->pack($aref, 3) }; +ok !$@; + +eval { Data::MessagePack->pack($aref, 2) }; +ok $@, $@; From 8de1f764fdcab4e9ec3f5739b2fb0a8ae2a02df9 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 21 Aug 2010 16:09:30 +0900 Subject: [PATCH 081/259] Perl: bump up version to 0.14 --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index f32d5f49..13fc98b5 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.14 + + - fixed segv on serializing cyclic reference + (Dan Kogai) + 0.13 - clearly specify requires_c99(), because msgpack C header requires C99. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index ee07d35f..3c38a795 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.13'; +our $VERSION = '0.14'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 18c712cd99d1317d69f628fe58fd6ff9d10865c5 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Wed, 25 Aug 2010 21:35:52 +0900 Subject: [PATCH 082/259] object serialization with reflection and with dynamic code generation --- java/pom.xml | 12 + .../org/msgpack/tmp/DynamicCodeGenerator.java | 209 ++++++++++++++++++ .../src/main/java/org/msgpack/tmp/Image1.java | 47 ++++ .../src/main/java/org/msgpack/tmp/Image2.java | 85 +++++++ .../src/main/java/org/msgpack/tmp/Image3.java | 38 ++++ .../msgpack/tmp/ImagePackUnpackPerfTest.java | 134 +++++++++++ 6 files changed, 525 insertions(+) create mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java create mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java diff --git a/java/pom.xml b/java/pom.xml index 44806d51..6591a727 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,6 +29,12 @@ 4.8.1 test + + + jboss + javassist + 3.7.ga + @@ -101,6 +107,12 @@ MessagePack Maven2 Repository http://msgpack.org/maven2 + + + repo2.maven.org + repo2.maven.org Maven2 Repository + http://repo2.maven.org/maven2 + diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java new file mode 100644 index 00000000..9e4336d3 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java @@ -0,0 +1,209 @@ +package org.msgpack.tmp; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.util.HashMap; + +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; + +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class DynamicCodeGenerator { + + private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced"; + + private static final String SPACE = " "; + + private static final String MODIFIER_PUBLIC = "public"; + + private static final String METHOD_NAME_MESSAGEPACK = "messagePack"; + + private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack"; + + private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); + + private static final String PACKER_OBJECT_NAME = "pk"; + + private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class + .getName(); + + private static final String UNPACKER_OBJECT_NAME = "pk"; + + private HashMap> classMap; + + private HashMap objectCacheMap; + + private ClassPool pool; + + public DynamicCodeGenerator() { + classMap = new HashMap>(); + objectCacheMap = new HashMap(); + pool = ClassPool.getDefault(); + } + + public Object newEnhancedInstance(Class targetClass) throws Exception { + String targetClassName = targetClass.getName(); + //Class enhancedClass = classMap.get(targetClassName); + Object enhancedObject = objectCacheMap.get(targetClassName); + //if (enhancedClass == null) { + if (enhancedObject == null) { + CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); +// System.out.println("enhanced class name: " +// + enhancedCtClass.getName()); + addSuperclass(enhancedCtClass, targetClassName); + addConstructor(enhancedCtClass); + createMessagePackMethod(enhancedCtClass, targetClass); + createMessageUnpackMethod(enhancedCtClass, targetClass); + Class enhancedClass = loadEnhancedClass(enhancedCtClass); + //classMap.put(targetClassName, enhancedClass); + enhancedObject = enhancedClass.newInstance(); + objectCacheMap.put(targetClassName, enhancedObject); + } + //return newEnhancedInstance0(enhancedClass); + return enhancedObject; + } + + private CtClass createEnhancedCtClass(final String targetClassName) + throws Exception { + return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX); + } + + private void addSuperclass(CtClass enhancedCtClass, + final String targetClassName) throws Exception { + CtClass targetCtClass = pool.get(targetClassName); + enhancedCtClass.setSuperclass(targetCtClass); + } + + private void addConstructor(CtClass enhancedCtClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append(MODIFIER_PUBLIC).append(SPACE).append( + // enhancedCtClass.getName()).append("(").append(")") + "Image3_$$_Enhanced").append("(").append(")").append(SPACE) + .append("{ super(); }"); // TODO +// System.out.println("cons: " + sb.toString()); + CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(), + enhancedCtClass); + enhancedCtClass.addConstructor(newCtConstructor); + } + + private void createMessagePackMethod(CtClass enhancedCtClass, + Class targetClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE) + .append(METHOD_NAME_MESSAGEPACK).append("(").append( + PACKER_CLASS_TYPE_NAME).append(SPACE).append( + PACKER_OBJECT_NAME).append(")").append(SPACE).append( + "throws").append(SPACE).append("java.io.IOException") + .append(SPACE).append("{"); + Field[] fields = targetClass.getFields(); + sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append( + "(").append(fields.length).append(")").append(";"); + for (Field field : fields) { + insertCodeOfMessagePack(field, sb); + } + sb.append("}"); +// System.out.println("messagePack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); + enhancedCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessagePack(Field field, StringBuilder sb) { + Class type = field.getType(); + if (type.equals(int.class)) { + sb.append(PACKER_OBJECT_NAME).append(".").append("pack") + .append("(").append(field.getName()).append(")") + .append(";"); + } else if (type.equals(String.class)) { + sb.append(PACKER_OBJECT_NAME).append(".").append("pack") + .append("(").append(field.getName()).append(")") + .append(";"); + } else { + throw new UnsupportedOperationException(); + } + } + + private void createMessageUnpackMethod(CtClass enhancedCtClass, + Class targetClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb + .append(MODIFIER_PUBLIC) + .append(SPACE) + .append("void") + .append(SPACE) + .append(METHOD_NAME_MESSAGEUNPACK) + .append("(") + .append(UNPACKER_CLASS_TYPE_NAME) + .append(SPACE) + .append(UNPACKER_OBJECT_NAME) + .append(")") + .append(SPACE) + .append("throws") + .append(SPACE) + .append("org.msgpack.MessageTypeException, java.io.IOException") + .append(SPACE).append("{"); + Field[] fields = targetClass.getFields(); + sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";"); + // TODO + for (Field field : fields) { + insertCodeOfMessageUnpack(field, sb); + } + sb.append("}"); +// System.out.println("messageUnpack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); + enhancedCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) { + Class type = field.getType(); + if (type.equals(int.class)) { + sb.append(field.getName()).append(SPACE).append("=").append(SPACE) + .append(PACKER_OBJECT_NAME).append(".").append("unpackInt") + .append("(").append(")").append(";"); + } else if (type.equals(String.class)) { + sb.append(field.getName()).append(SPACE).append("=").append(SPACE) + .append(PACKER_OBJECT_NAME).append(".").append( + "unpackString").append("(").append(")").append(";"); + } else { + throw new UnsupportedOperationException(); + } + } + + private Class loadEnhancedClass(CtClass enhancedCtClass) + throws Exception { + return enhancedCtClass.toClass(null, null); + } + + private Object newEnhancedInstance0(Class enhancedClass) { + try { + return enhancedClass.newInstance(); + } catch (Exception e) { + throw new UnsupportedOperationException(); + } + } + + public static void main(String[] args) throws Exception { + DynamicCodeGenerator gen = new DynamicCodeGenerator(); + Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + +// System.out.println(dst.equals(src)); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java new file mode 100644 index 00000000..f1819ed6 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image1.java @@ -0,0 +1,47 @@ +package org.msgpack.tmp; + +import org.msgpack.*; +import java.io.*; + +public class Image1 implements MessagePackable, MessageUnpackable { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + pk.packArray(5); + pk.pack(uri); + pk.pack(title); + pk.pack(width); + pk.pack(height); + pk.pack(size); + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + int length = pac.unpackArray(); + if (length != 5) { + throw new MessageTypeException(); + } + uri = pac.unpackString(); + title = pac.unpackString(); + width = pac.unpackInt(); + height = pac.unpackInt(); + size = pac.unpackInt(); + } + + public boolean equals(Image1 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image1.class) { + return false; + } + return equals((Image1) obj); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java new file mode 100644 index 00000000..9c357bf3 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image2.java @@ -0,0 +1,85 @@ +package org.msgpack.tmp; + +import java.io.IOException; +import java.lang.reflect.Field; + +import org.msgpack.MessageTypeException; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class Image2 { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + messagePackWithReflection(pk); + } + + public void messagePackWithReflection(Packer pk) throws IOException { + Class cl = this.getClass(); + Field[] fields = cl.getFields(); + pk.packArray(fields.length); + for (Field field : fields) { + try { + Object obj = field.get(this); + pk.pack(obj); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + messageUnpackWithReflection(pac); + } + + public void messageUnpackWithReflection(Unpacker pac) throws IOException, + MessageTypeException { + Class cl = this.getClass(); + Field[] fields = cl.getFields(); + int length = pac.unpackArray(); + if (length != fields.length) { + throw new MessageTypeException(); + } + for (Field field : fields) { + try { + field.set(this, unpack(pac, field.getType())); + } catch (IOException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + + public static Object unpack(Unpacker pk, Class cl) throws IOException { + if (cl == int.class) { + return pk.unpackInt(); + } else if (cl == String.class) { + return pk.unpackString(); + } else { + throw new UnsupportedOperationException(); + } + } + + public boolean equals(Image2 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image2.class) { + return false; + } + return equals((Image2) obj); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java new file mode 100644 index 00000000..a28ff53f --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image3.java @@ -0,0 +1,38 @@ +package org.msgpack.tmp; + +import java.io.IOException; +import org.msgpack.MessagePackable; +import org.msgpack.MessageTypeException; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class Image3 implements MessagePackable, MessageUnpackable { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + // empty + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + // empty + } + + public boolean equals(Image3 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image3.class) { + return false; + } + return equals((Image3) obj); + } +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java new file mode 100644 index 00000000..1f27c1b1 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java @@ -0,0 +1,134 @@ +package org.msgpack.tmp; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class ImagePackUnpackPerfTest { + + private static int BIG_LOOP = 5; + + private static int SMALL_LOOP = 50000; + + private static DynamicCodeGenerator gen; + + private static void doGc() { + try { + Thread.sleep(50L); + } catch (InterruptedException ie) { + System.err + .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); + } + System.gc(); + try { // longer sleep afterwards (not needed by GC, but may help with + // scheduling) + Thread.sleep(200L); + } catch (InterruptedException ie) { + System.err + .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); + } + } + + public void doIt(int versionID) { + try { + doIt0(versionID); + } catch (Exception e) { + e.printStackTrace(); + } + try { + for (int j = 0; j < BIG_LOOP; ++j) { + long t = System.currentTimeMillis(); + doIt0(versionID); + t = System.currentTimeMillis() - t; + System.out.println("time: " + t); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void doIt0(int versionID) throws Exception { + for (int i = 0; i < SMALL_LOOP; ++i) { + switch (versionID) { + case 0: + doCurrentVersion(); + break; + case 1: + doWithReflection(); + break; + case 2: + doWithDynamicCodeGeneration(); + break; + default: + throw new RuntimeException(); + } + } + } + + public void doCurrentVersion() throws Exception { + Image1 src = new Image1(); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image1 dst = new Image1(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public void doWithReflection() throws Exception { + Image2 src = new Image2(); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image2 dst = new Image2(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public void doWithDynamicCodeGeneration() throws Exception { + Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public static void main(String[] args) { + ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest(); + + doGc(); + System.out.println("test current version"); + test.doIt(0); + + doGc(); + System.out.println("test with reflection"); + test.doIt(1); + + doGc(); + System.out.println("test with dynamic codegen"); + gen = new DynamicCodeGenerator(); + test.doIt(2); + } +} From ff0e1bbbc070e95be3631459815b0f765c3aab8d Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 26 Aug 2010 17:53:29 +0900 Subject: [PATCH 083/259] change the part for creating a new constructor within DynamicCodeGenerator --- .../org/msgpack/tmp/DynamicCodeGenerator.java | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java index 9e4336d3..78238b4f 100644 --- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java +++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java @@ -29,6 +29,8 @@ public class DynamicCodeGenerator { private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); + private static final String VOID_TYPE_NAME = "void"; + private static final String PACKER_OBJECT_NAME = "pk"; private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class @@ -37,7 +39,7 @@ public class DynamicCodeGenerator { private static final String UNPACKER_OBJECT_NAME = "pk"; private HashMap> classMap; - + private HashMap objectCacheMap; private ClassPool pool; @@ -50,23 +52,23 @@ public class DynamicCodeGenerator { public Object newEnhancedInstance(Class targetClass) throws Exception { String targetClassName = targetClass.getName(); - //Class enhancedClass = classMap.get(targetClassName); + // Class enhancedClass = classMap.get(targetClassName); Object enhancedObject = objectCacheMap.get(targetClassName); - //if (enhancedClass == null) { + // if (enhancedClass == null) { if (enhancedObject == null) { CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); -// System.out.println("enhanced class name: " -// + enhancedCtClass.getName()); + // System.out.println("enhanced class name: " + // + enhancedCtClass.getName()); addSuperclass(enhancedCtClass, targetClassName); addConstructor(enhancedCtClass); createMessagePackMethod(enhancedCtClass, targetClass); createMessageUnpackMethod(enhancedCtClass, targetClass); Class enhancedClass = loadEnhancedClass(enhancedCtClass); - //classMap.put(targetClassName, enhancedClass); + // classMap.put(targetClassName, enhancedClass); enhancedObject = enhancedClass.newInstance(); objectCacheMap.put(targetClassName, enhancedObject); } - //return newEnhancedInstance0(enhancedClass); + // return newEnhancedInstance0(enhancedClass); return enhancedObject; } @@ -82,24 +84,18 @@ public class DynamicCodeGenerator { } private void addConstructor(CtClass enhancedCtClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append( - // enhancedCtClass.getName()).append("(").append(")") - "Image3_$$_Enhanced").append("(").append(")").append(SPACE) - .append("{ super(); }"); // TODO -// System.out.println("cons: " + sb.toString()); - CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(), - enhancedCtClass); + CtConstructor newCtConstructor = CtNewConstructor + .defaultConstructor(enhancedCtClass); enhancedCtClass.addConstructor(newCtConstructor); } private void createMessagePackMethod(CtClass enhancedCtClass, Class targetClass) throws Exception { StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE) - .append(METHOD_NAME_MESSAGEPACK).append("(").append( - PACKER_CLASS_TYPE_NAME).append(SPACE).append( - PACKER_OBJECT_NAME).append(")").append(SPACE).append( + sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append( + SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append( + PACKER_CLASS_TYPE_NAME).append(SPACE) + .append(PACKER_OBJECT_NAME).append(")").append(SPACE).append( "throws").append(SPACE).append("java.io.IOException") .append(SPACE).append("{"); Field[] fields = targetClass.getFields(); @@ -109,7 +105,7 @@ public class DynamicCodeGenerator { insertCodeOfMessagePack(field, sb); } sb.append("}"); -// System.out.println("messagePack method: " + sb.toString()); + // System.out.println("messagePack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); enhancedCtClass.addMethod(newCtMethod); } @@ -135,7 +131,7 @@ public class DynamicCodeGenerator { sb .append(MODIFIER_PUBLIC) .append(SPACE) - .append("void") + .append(VOID_TYPE_NAME) .append(SPACE) .append(METHOD_NAME_MESSAGEUNPACK) .append("(") @@ -149,13 +145,14 @@ public class DynamicCodeGenerator { .append("org.msgpack.MessageTypeException, java.io.IOException") .append(SPACE).append("{"); Field[] fields = targetClass.getFields(); - sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";"); + sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()") + .append(";"); // TODO for (Field field : fields) { insertCodeOfMessageUnpack(field, sb); } sb.append("}"); -// System.out.println("messageUnpack method: " + sb.toString()); + // System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); enhancedCtClass.addMethod(newCtMethod); } From 59ba8dec4ee082e8777047e6ae72e8b6998cdc79 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 27 Aug 2010 16:45:48 +0900 Subject: [PATCH 084/259] cpp: fixes include paths --- cpp/src/msgpack/object.h | 2 +- cpp/src/msgpack/object.hpp | 6 +++--- cpp/src/msgpack/pack.h | 9 ++++++--- cpp/src/msgpack/pack.hpp | 4 ++-- cpp/src/msgpack/sbuffer.hpp | 2 +- cpp/src/msgpack/type.hpp | 28 ++++++++++++++-------------- cpp/src/msgpack/unpack.h | 4 ++-- cpp/src/msgpack/unpack.hpp | 6 +++--- cpp/src/msgpack/vrefbuffer.h | 2 +- cpp/src/msgpack/vrefbuffer.hpp | 2 +- cpp/src/msgpack/zbuffer.h | 2 +- cpp/src/msgpack/zbuffer.hpp | 2 +- cpp/src/msgpack/zone.h | 2 +- cpp/src/msgpack/zone.hpp.erb | 2 +- 14 files changed, 38 insertions(+), 35 deletions(-) diff --git a/cpp/src/msgpack/object.h b/cpp/src/msgpack/object.h index cf0b4c18..baeeb9b2 100644 --- a/cpp/src/msgpack/object.h +++ b/cpp/src/msgpack/object.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_OBJECT_H__ #define MSGPACK_OBJECT_H__ -#include "msgpack/zone.h" +#include "zone.h" #include #ifdef __cplusplus diff --git a/cpp/src/msgpack/object.hpp b/cpp/src/msgpack/object.hpp index f80a390d..96c026ef 100644 --- a/cpp/src/msgpack/object.hpp +++ b/cpp/src/msgpack/object.hpp @@ -18,9 +18,9 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ -#include "msgpack/object.h" -#include "msgpack/pack.hpp" -#include "msgpack/zone.hpp" +#include "object.h" +#include "pack.hpp" +#include "zone.hpp" #include #include #include diff --git a/cpp/src/msgpack/pack.h b/cpp/src/msgpack/pack.h index 1252895d..9c4ce595 100644 --- a/cpp/src/msgpack/pack.h +++ b/cpp/src/msgpack/pack.h @@ -18,8 +18,8 @@ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ -#include "msgpack/pack_define.h" -#include "msgpack/object.h" +#include "pack_define.h" +#include "object.h" #include #ifdef __cplusplus @@ -96,12 +96,15 @@ int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name +#define msgpack_pack_inline_func_cint(name) \ + inline int msgpack_pack ## name + #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/cpp/src/msgpack/pack.hpp b/cpp/src/msgpack/pack.hpp index ee906900..2f7dfa07 100644 --- a/cpp/src/msgpack/pack.hpp +++ b/cpp/src/msgpack/pack.hpp @@ -18,7 +18,7 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include "msgpack/pack_define.h" +#include "pack_define.h" #include #include @@ -137,7 +137,7 @@ inline void pack(Stream& s, const T& v) #define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack_template.h" +#include "pack_template.h" template diff --git a/cpp/src/msgpack/sbuffer.hpp b/cpp/src/msgpack/sbuffer.hpp index e4a3f969..a9efc6db 100644 --- a/cpp/src/msgpack/sbuffer.hpp +++ b/cpp/src/msgpack/sbuffer.hpp @@ -18,7 +18,7 @@ #ifndef MSGPACK_SBUFFER_HPP__ #define MSGPACK_SBUFFER_HPP__ -#include "msgpack/sbuffer.h" +#include "sbuffer.h" #include namespace msgpack { diff --git a/cpp/src/msgpack/type.hpp b/cpp/src/msgpack/type.hpp index fafa6740..a55c68e7 100644 --- a/cpp/src/msgpack/type.hpp +++ b/cpp/src/msgpack/type.hpp @@ -1,15 +1,15 @@ -#include "msgpack/type/bool.hpp" -#include "msgpack/type/float.hpp" -#include "msgpack/type/int.hpp" -#include "msgpack/type/list.hpp" -#include "msgpack/type/deque.hpp" -#include "msgpack/type/map.hpp" -#include "msgpack/type/nil.hpp" -#include "msgpack/type/pair.hpp" -#include "msgpack/type/raw.hpp" -#include "msgpack/type/set.hpp" -#include "msgpack/type/string.hpp" -#include "msgpack/type/vector.hpp" -#include "msgpack/type/tuple.hpp" -#include "msgpack/type/define.hpp" +#include "type/bool.hpp" +#include "type/float.hpp" +#include "type/int.hpp" +#include "type/list.hpp" +#include "type/deque.hpp" +#include "type/map.hpp" +#include "type/nil.hpp" +#include "type/pair.hpp" +#include "type/raw.hpp" +#include "type/set.hpp" +#include "type/string.hpp" +#include "type/vector.hpp" +#include "type/tuple.hpp" +#include "type/define.hpp" diff --git a/cpp/src/msgpack/unpack.h b/cpp/src/msgpack/unpack.h index 82698fc2..bea7d922 100644 --- a/cpp/src/msgpack/unpack.h +++ b/cpp/src/msgpack/unpack.h @@ -18,8 +18,8 @@ #ifndef MSGPACK_UNPACKER_H__ #define MSGPACK_UNPACKER_H__ -#include "msgpack/zone.h" -#include "msgpack/object.h" +#include "zone.h" +#include "object.h" #include #ifdef __cplusplus diff --git a/cpp/src/msgpack/unpack.hpp b/cpp/src/msgpack/unpack.hpp index 7b45017a..51580b6e 100644 --- a/cpp/src/msgpack/unpack.hpp +++ b/cpp/src/msgpack/unpack.hpp @@ -18,9 +18,9 @@ #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ -#include "msgpack/unpack.h" -#include "msgpack/object.hpp" -#include "msgpack/zone.hpp" +#include "unpack.h" +#include "object.hpp" +#include "zone.hpp" #include #include diff --git a/cpp/src/msgpack/vrefbuffer.h b/cpp/src/msgpack/vrefbuffer.h index 123499d1..0643927e 100644 --- a/cpp/src/msgpack/vrefbuffer.h +++ b/cpp/src/msgpack/vrefbuffer.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_VREFBUFFER_H__ #define MSGPACK_VREFBUFFER_H__ -#include "msgpack/zone.h" +#include "zone.h" #include #ifndef _WIN32 diff --git a/cpp/src/msgpack/vrefbuffer.hpp b/cpp/src/msgpack/vrefbuffer.hpp index 7e0ffb23..82335277 100644 --- a/cpp/src/msgpack/vrefbuffer.hpp +++ b/cpp/src/msgpack/vrefbuffer.hpp @@ -18,7 +18,7 @@ #ifndef MSGPACK_VREFBUFFER_HPP__ #define MSGPACK_VREFBUFFER_HPP__ -#include "msgpack/vrefbuffer.h" +#include "vrefbuffer.h" #include namespace msgpack { diff --git a/cpp/src/msgpack/zbuffer.h b/cpp/src/msgpack/zbuffer.h index abb9c501..efdd3049 100644 --- a/cpp/src/msgpack/zbuffer.h +++ b/cpp/src/msgpack/zbuffer.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_ZBUFFER_H__ #define MSGPACK_ZBUFFER_H__ -#include "msgpack/sysdep.h" +#include "sysdep.h" #include #include #include diff --git a/cpp/src/msgpack/zbuffer.hpp b/cpp/src/msgpack/zbuffer.hpp index 278f076e..08e6fd03 100644 --- a/cpp/src/msgpack/zbuffer.hpp +++ b/cpp/src/msgpack/zbuffer.hpp @@ -18,7 +18,7 @@ #ifndef MSGPACK_ZBUFFER_HPP__ #define MSGPACK_ZBUFFER_HPP__ -#include "msgpack/zbuffer.h" +#include "zbuffer.h" #include namespace msgpack { diff --git a/cpp/src/msgpack/zone.h b/cpp/src/msgpack/zone.h index 0e811dff..d8c60b68 100644 --- a/cpp/src/msgpack/zone.h +++ b/cpp/src/msgpack/zone.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_ZONE_H__ #define MSGPACK_ZONE_H__ -#include "msgpack/sysdep.h" +#include "sysdep.h" #ifdef __cplusplus extern "C" { diff --git a/cpp/src/msgpack/zone.hpp.erb b/cpp/src/msgpack/zone.hpp.erb index 8e69aa44..1cef05ec 100644 --- a/cpp/src/msgpack/zone.hpp.erb +++ b/cpp/src/msgpack/zone.hpp.erb @@ -18,7 +18,7 @@ #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ -#include "msgpack/zone.h" +#include "zone.h" #include #include #include From fe2a0f5089ebfc5c03db783a1f85b1c7c217128a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 27 Aug 2010 17:42:05 +0900 Subject: [PATCH 085/259] cpp: adds fixed length serialization for integers --- cpp/src/Makefile.am | 3 +- cpp/src/msgpack/pack.h | 12 +++++ cpp/src/msgpack/pack.hpp | 56 +++++++++++++++++++++ cpp/src/msgpack/type.hpp | 3 +- cpp/src/msgpack/type/fixint.hpp | 89 +++++++++++++++++++++++++++++++++ cpp/test/Makefile.am | 6 +++ cpp/test/fixint.cc | 24 +++++++++ cpp/test/fixint_c.cc | 32 ++++++++++++ 8 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 cpp/src/msgpack/type/fixint.hpp create mode 100644 cpp/test/fixint.cc create mode 100644 cpp/test/fixint_c.cc diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am index 31096f0c..0979d235 100644 --- a/cpp/src/Makefile.am +++ b/cpp/src/Makefile.am @@ -58,10 +58,11 @@ nobase_include_HEADERS += \ msgpack/zone.hpp \ msgpack/type.hpp \ msgpack/type/bool.hpp \ + msgpack/type/deque.hpp \ msgpack/type/float.hpp \ + msgpack/type/fixint.hpp \ msgpack/type/int.hpp \ msgpack/type/list.hpp \ - msgpack/type/deque.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/pair.hpp \ diff --git a/cpp/src/msgpack/pack.h b/cpp/src/msgpack/pack.h index 9c4ce595..c1564963 100644 --- a/cpp/src/msgpack/pack.h +++ b/cpp/src/msgpack/pack.h @@ -70,6 +70,15 @@ static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static int msgpack_pack_fix_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_fix_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_fix_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_fix_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_fix_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_fix_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_fix_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_fix_int64(msgpack_packer* pk, int64_t d); + static int msgpack_pack_float(msgpack_packer* pk, float d); static int msgpack_pack_double(msgpack_packer* pk, double d); @@ -99,6 +108,9 @@ int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name +#define msgpack_pack_inline_func_fixint(name) \ + inline int msgpack_pack_fix ## name + #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ diff --git a/cpp/src/msgpack/pack.hpp b/cpp/src/msgpack/pack.hpp index 2f7dfa07..0090b961 100644 --- a/cpp/src/msgpack/pack.hpp +++ b/cpp/src/msgpack/pack.hpp @@ -45,6 +45,15 @@ public: packer& pack_int32(int32_t d); packer& pack_int64(int64_t d); + packer& pack_fix_uint8(uint8_t d); + packer& pack_fix_uint16(uint16_t d); + packer& pack_fix_uint32(uint32_t d); + packer& pack_fix_uint64(uint64_t d); + packer& pack_fix_int8(int8_t d); + packer& pack_fix_int16(int16_t d); + packer& pack_fix_int32(int32_t d); + packer& pack_fix_int64(int64_t d); + packer& pack_short(short d); packer& pack_int(int d); packer& pack_long(long d); @@ -78,6 +87,15 @@ private: static void _pack_int32(Stream& x, int32_t d); static void _pack_int64(Stream& x, int64_t d); + static void _pack_fix_uint8(Stream& x, uint8_t d); + static void _pack_fix_uint16(Stream& x, uint16_t d); + static void _pack_fix_uint32(Stream& x, uint32_t d); + static void _pack_fix_uint64(Stream& x, uint64_t d); + static void _pack_fix_int8(Stream& x, int8_t d); + static void _pack_fix_int16(Stream& x, int16_t d); + static void _pack_fix_int32(Stream& x, int32_t d); + static void _pack_fix_int64(Stream& x, int64_t d); + static void _pack_short(Stream& x, short d); static void _pack_int(Stream& x, int d); static void _pack_long(Stream& x, long d); @@ -133,6 +151,10 @@ inline void pack(Stream& s, const T& v) template \ inline void packer::_pack ## name +#define msgpack_pack_inline_func_fixint(name) \ + template \ + inline void packer::_pack_fix ## name + #define msgpack_pack_user Stream& #define msgpack_pack_append_buffer append_buffer @@ -149,6 +171,7 @@ packer::packer(Stream& s) : m_stream(s) { } template packer::~packer() { } + template inline packer& packer::pack_uint8(uint8_t d) { _pack_uint8(m_stream, d); return *this; } @@ -182,6 +205,39 @@ inline packer& packer::pack_int64(int64_t d) { _pack_int64(m_stream, d); return *this;} +template +inline packer& packer::pack_fix_uint8(uint8_t d) +{ _pack_fix_uint8(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_uint16(uint16_t d) +{ _pack_fix_uint16(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_uint32(uint32_t d) +{ _pack_fix_uint32(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_uint64(uint64_t d) +{ _pack_fix_uint64(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_int8(int8_t d) +{ _pack_fix_int8(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_int16(int16_t d) +{ _pack_fix_int16(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_int32(int32_t d) +{ _pack_fix_int32(m_stream, d); return *this; } + +template +inline packer& packer::pack_fix_int64(int64_t d) +{ _pack_fix_int64(m_stream, d); return *this;} + + template inline packer& packer::pack_short(short d) { _pack_short(m_stream, d); return *this; } diff --git a/cpp/src/msgpack/type.hpp b/cpp/src/msgpack/type.hpp index a55c68e7..bca69bfd 100644 --- a/cpp/src/msgpack/type.hpp +++ b/cpp/src/msgpack/type.hpp @@ -1,8 +1,9 @@ #include "type/bool.hpp" +#include "type/deque.hpp" +#include "type/fixint.hpp" #include "type/float.hpp" #include "type/int.hpp" #include "type/list.hpp" -#include "type/deque.hpp" #include "type/map.hpp" #include "type/nil.hpp" #include "type/pair.hpp" diff --git a/cpp/src/msgpack/type/fixint.hpp b/cpp/src/msgpack/type/fixint.hpp new file mode 100644 index 00000000..da584159 --- /dev/null +++ b/cpp/src/msgpack/type/fixint.hpp @@ -0,0 +1,89 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2020 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_FIXINT_HPP__ +#define MSGPACK_TYPE_FIXINT_HPP__ + +#include "msgpack/object.hpp" + +namespace msgpack { + +namespace type { + + +template +struct fix_int { + fix_int() : value(0) { } + fix_int(T value) : value(value) { } + operator T() const { return value; } + T get() const { return value; } +private: + const T value; +}; + + +typedef fix_int fix_uint8; +typedef fix_int fix_uint16; +typedef fix_int fix_uint32; +typedef fix_int fix_uint64; + +typedef fix_int fix_int8; +typedef fix_int fix_int16; +typedef fix_int fix_int32; +typedef fix_int fix_int64; + + +} // namespace type + + +template +inline packer& operator<< (packer& o, const type::fix_int8& v) + { o.pack_fix_int8(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_int16& v) + { o.pack_fix_int16(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_int32& v) + { o.pack_fix_int32(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_int64& v) + { o.pack_fix_int64(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_uint8& v) + { o.pack_fix_uint8(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_uint16& v) + { o.pack_fix_uint16(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_uint32& v) + { o.pack_fix_uint32(v); return o; } + +template +inline packer& operator<< (packer& o, const type::fix_uint64& v) + { o.pack_fix_uint64(v); return o; } + + +} // namespace msgpack + +#endif /* msgpack/type/fixint.hpp */ + diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am index bb9439e3..5225f28b 100644 --- a/cpp/test/Makefile.am +++ b/cpp/test/Makefile.am @@ -13,6 +13,8 @@ check_PROGRAMS = \ convert \ buffer \ cases \ + fixint \ + fixint_c \ version \ msgpackc_test \ msgpack_test @@ -38,6 +40,10 @@ buffer_LDADD = -lz cases_SOURCES = cases.cc +fixint_SOURCES = fixint.cc + +fixint_c_SOURCES = fixint_c.cc + version_SOURCES = version.cc msgpackc_test_SOURCES = msgpackc_test.cpp diff --git a/cpp/test/fixint.cc b/cpp/test/fixint.cc new file mode 100644 index 00000000..64a39acd --- /dev/null +++ b/cpp/test/fixint.cc @@ -0,0 +1,24 @@ +#include +#include + +template +void check_size(size_t size) { + T v(0); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, v); + EXPECT_EQ(size, sbuf.size()); +} + +TEST(fixint, size) +{ + check_size(2); + check_size(3); + check_size(5); + check_size(9); + + check_size(2); + check_size(3); + check_size(5); + check_size(9); +} + diff --git a/cpp/test/fixint_c.cc b/cpp/test/fixint_c.cc new file mode 100644 index 00000000..caa4d262 --- /dev/null +++ b/cpp/test/fixint_c.cc @@ -0,0 +1,32 @@ +#include +#include + +TEST(fixint, size) +{ + msgpack_sbuffer* sbuf = msgpack_sbuffer_new(); + msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write); + + size_t sum = 0; + + EXPECT_EQ(0, msgpack_pack_fix_int8(pk, 0)); + EXPECT_EQ(sum+=2, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_int16(pk, 0)); + EXPECT_EQ(sum+=3, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_int32(pk, 0)); + EXPECT_EQ(sum+=5, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_int64(pk, 0)); + EXPECT_EQ(sum+=9, sbuf->size); + + EXPECT_EQ(0, msgpack_pack_fix_uint8(pk, 0)); + EXPECT_EQ(sum+=2, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_uint16(pk, 0)); + EXPECT_EQ(sum+=3, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_uint32(pk, 0)); + EXPECT_EQ(sum+=5, sbuf->size); + EXPECT_EQ(0, msgpack_pack_fix_uint64(pk, 0)); + EXPECT_EQ(sum+=9, sbuf->size); + + msgpack_sbuffer_free(sbuf); + msgpack_packer_free(pk); +} + From 2c7573a032b7aa3b0588bace376d3c24b53fbc2d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 27 Aug 2010 17:53:02 +0900 Subject: [PATCH 086/259] cpp: updates msgpack_vc8.postbuild.bat --- cpp/msgpack_vc8.postbuild.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat index 20fabbb9..33ff232b 100644 --- a/cpp/msgpack_vc8.postbuild.bat +++ b/cpp/msgpack_vc8.postbuild.bat @@ -26,10 +26,11 @@ copy src\msgpack\object.hpp include\msgpack\ copy src\msgpack\zone.hpp include\msgpack\ copy src\msgpack\type.hpp include\msgpack\type\ copy src\msgpack\type\bool.hpp include\msgpack\type\ +copy src\msgpack\type\deque.hpp include\msgpack\type\ +copy src\msgpack\type\fixint.hpp include\msgpack\type\ copy src\msgpack\type\float.hpp include\msgpack\type\ copy src\msgpack\type\int.hpp include\msgpack\type\ copy src\msgpack\type\list.hpp include\msgpack\type\ -copy src\msgpack\type\deque.hpp include\msgpack\type\ copy src\msgpack\type\map.hpp include\msgpack\type\ copy src\msgpack\type\nil.hpp include\msgpack\type\ copy src\msgpack\type\pair.hpp include\msgpack\type\ From 421bee38719ee66d79cd8c376c871678dbb55a1d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 27 Aug 2010 17:53:19 +0900 Subject: [PATCH 087/259] cpp: version 0.5.3 --- cpp/ChangeLog | 7 +++++++ cpp/configure.in | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/ChangeLog b/cpp/ChangeLog index 504ac4ba..2756236a 100644 --- a/cpp/ChangeLog +++ b/cpp/ChangeLog @@ -1,4 +1,11 @@ +2010-07-27 version 0.5.3: + + * adds type::fix_{u,}int{8,16,32,64} types + * adds msgpack_pack_fix_{u,}int{8,16,32,64} functions + * adds packer::pack_fix_{u,}int{8,16,32,64} functions + * fixes include paths + 2010-07-14 version 0.5.2: * type::raw::str(), operator==, operator!=, operator< and operator> are now const diff --git a/cpp/configure.in b/cpp/configure.in index 93174da2..0104ef90 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,6 +1,6 @@ AC_INIT(src/object.cpp) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.5.2) +AM_INIT_AUTOMAKE(msgpack, 0.5.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From c87f7cb9ac87525c9531ddf6b2fd16499535967c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 27 Aug 2010 20:52:40 +0900 Subject: [PATCH 088/259] cpp: fixes fix_int; updates test/fixint.cc --- cpp/src/msgpack/type/fixint.hpp | 85 ++++++++++++++++++++++++++++++++- cpp/test/fixint.cc | 31 ++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/cpp/src/msgpack/type/fixint.hpp b/cpp/src/msgpack/type/fixint.hpp index da584159..ebe2696f 100644 --- a/cpp/src/msgpack/type/fixint.hpp +++ b/cpp/src/msgpack/type/fixint.hpp @@ -19,6 +19,7 @@ #define MSGPACK_TYPE_FIXINT_HPP__ #include "msgpack/object.hpp" +#include "msgpack/type/int.hpp" namespace msgpack { @@ -29,10 +30,13 @@ template struct fix_int { fix_int() : value(0) { } fix_int(T value) : value(value) { } + operator T() const { return value; } + T get() const { return value; } + private: - const T value; + T value; }; @@ -50,6 +54,32 @@ typedef fix_int fix_int64; } // namespace type +inline type::fix_int8& operator>> (object o, type::fix_int8& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_int16& operator>> (object o, type::fix_int16& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_int32& operator>> (object o, type::fix_int32& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_int64& operator>> (object o, type::fix_int64& v) + { v = type::detail::convert_integer(o); return v; } + + +inline type::fix_uint8& operator>> (object o, type::fix_uint8& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_uint16& operator>> (object o, type::fix_uint16& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_uint32& operator>> (object o, type::fix_uint32& v) + { v = type::detail::convert_integer(o); return v; } + +inline type::fix_uint64& operator>> (object o, type::fix_uint64& v) + { v = type::detail::convert_integer(o); return v; } + + template inline packer& operator<< (packer& o, const type::fix_int8& v) { o.pack_fix_int8(v); return o; } @@ -66,6 +96,7 @@ template inline packer& operator<< (packer& o, const type::fix_int64& v) { o.pack_fix_int64(v); return o; } + template inline packer& operator<< (packer& o, const type::fix_uint8& v) { o.pack_fix_uint8(v); return o; } @@ -83,6 +114,58 @@ inline packer& operator<< (packer& o, const type::fix_uint64& v) { o.pack_fix_uint64(v); return o; } +inline void operator<< (object& o, type::fix_int8 v) + { v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_int16 v) + { v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_int32 v) + { v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_int64 v) + { v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + + +inline void operator<< (object& o, type::fix_uint8 v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_uint16 v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_uint32 v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + +inline void operator<< (object& o, type::fix_uint64 v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); } + + +inline void operator<< (object::with_zone& o, type::fix_int8 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_int16 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_int32 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_int64 v) + { static_cast(o) << v; } + + +inline void operator<< (object::with_zone& o, type::fix_uint8 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_uint16 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_uint32 v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, type::fix_uint64 v) + { static_cast(o) << v; } + + } // namespace msgpack #endif /* msgpack/type/fixint.hpp */ diff --git a/cpp/test/fixint.cc b/cpp/test/fixint.cc index 64a39acd..63288a1b 100644 --- a/cpp/test/fixint.cc +++ b/cpp/test/fixint.cc @@ -22,3 +22,34 @@ TEST(fixint, size) check_size(9); } + +template +void check_convert() { + T v1(-11); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, v1); + + msgpack::unpacked msg; + msgpack::unpack(&msg, sbuf.data(), sbuf.size()); + + T v2; + msg.get().convert(&v2); + + EXPECT_EQ(v1.get(), v2.get()); + + EXPECT_EQ(msg.get(), msgpack::object(T(v1.get()))); +} + +TEST(fixint, convert) +{ + check_convert(); + check_convert(); + check_convert(); + check_convert(); + + check_convert(); + check_convert(); + check_convert(); + check_convert(); +} + From c42cba1d5453f718a85f49f7441387ed4d0a7210 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 27 Aug 2010 23:02:16 +0900 Subject: [PATCH 089/259] erlang: fixed bug around error case when serializing atom. --- erlang/msgpack.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index b54874d5..8c85458e 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -126,7 +126,7 @@ pack_(List) when is_list(List) -> pack_({Map}) when is_list(Map) -> pack_map(Map); pack_(Other) -> - throw({error, {badarg, Other}}). + throw({badarg, Other}). -spec pack_uint_(non_neg_integer()) -> binary(). @@ -387,4 +387,9 @@ benchmark_test()-> {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)), ?debugFmt("for ~p KB test data.", [byte_size(S) div 1024]). +error_test()-> + ?assertEqual({error,{badarg, atom}}, msgpack:pack(atom)), + Term = {"hoge", "hage", atom}, + ?assertEqual({error,{badarg, Term}}, msgpack:pack(Term)). + -endif. From c44c9ab74da304c70f78fbfda7fcad206435ff0e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 29 Aug 2010 18:23:16 +0900 Subject: [PATCH 090/259] cpp: adds msgpack_vc2008.vcproj file in source package --- cpp/Makefile.am | 4 +++- cpp/{msgpack_vc8.postbuild.bat => msgpack_vc.postbuild.bat} | 0 cpp/msgpack_vc8.vcproj | 4 ++-- cpp/preprocess | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) rename cpp/{msgpack_vc8.postbuild.bat => msgpack_vc.postbuild.bat} (100%) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 7dd48910..ecec1b56 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -6,7 +6,9 @@ DOC_FILES = \ NOTICE \ msgpack_vc8.vcproj \ msgpack_vc8.sln \ - msgpack_vc8.postbuild.bat + msgpack_vc2008.vcproj \ + msgpack_vc2008.sln \ + msgpack_vc.postbuild.bat EXTRA_DIST = \ $(DOC_FILES) diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc.postbuild.bat similarity index 100% rename from cpp/msgpack_vc8.postbuild.bat rename to cpp/msgpack_vc.postbuild.bat diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index ed0daa47..72d47b6b 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -28,7 +28,7 @@ msgpack_vc2008.vcproj +sed -e 's/9\.00/10.00/' -e 's/msgpack_vc8/msgpack_vc2008/' < msgpack_vc8.sln > msgpack_vc2008.sln + From 3c75361e5a195eba9622927cac7aebe0944f9232 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 29 Aug 2010 18:24:32 +0900 Subject: [PATCH 091/259] cpp: updates README.md --- cpp/README.md | 45 +++++++++---------- .../org/msgpack/TestMessageUnpackable.java | 4 +- msgpack/pack_template.h | 20 ++++----- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 454ce1af..eac77935 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -13,9 +13,10 @@ On UNIX-like platform, run ./configure && make && sudo make install: $ make $ sudo make install -On Windows, open msgpack_vc8.vcproj file and build it using batch build. DLLs are built on lib folder, and the headers are built on include folder. +On Windows, open msgpack_vc8.vcproj or msgpack_vc2008 file and build it using batch build. DLLs are built on lib folder, +and the headers are built on include folder. -To use the library in your program, include msgpack.hpp header and link msgpack and msgpackc library. +To use the library in your program, include msgpack.hpp header and link "msgpack" library. ## Example @@ -34,15 +35,9 @@ To use the library in your program, include msgpack.hpp header and link msgpack msgpack::pack(&buffer, target); // Deserialize the serialized data. - msgpack::zone mempool; // this manages the life of deserialized object - msgpack::object obj; - msgpack::unpack_return ret = - msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj); - - if(ret != msgapck::UNPACK_SUCCESS) { - // error check - exit(1); - } + msgpack::unpacked msg; // includes memory pool and deserialized object + msgpack::unpack(&msg, sbuf.data(), sbuf.size()); + msgpack::object obj = msg.get(); // Print the deserialized object to stdout. std::cout << obj << std::endl; // ["Hello," "World!"] @@ -55,24 +50,24 @@ To use the library in your program, include msgpack.hpp header and link msgpack obj.as(); // type is mismatched, msgpack::type_error is thrown } -API document and other example codes are available at the [wiki.](http://msgpack.sourceforge.net/start) +API documents and other example codes are available at the [wiki.](http://redmine.msgpack.org/projects/msgpack/wiki) ## License -Copyright (C) 2008-2010 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Copyright (C) 2008-2010 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See also NOTICE file. diff --git a/java/src/test/java/org/msgpack/TestMessageUnpackable.java b/java/src/test/java/org/msgpack/TestMessageUnpackable.java index 9099f910..32917c7a 100644 --- a/java/src/test/java/org/msgpack/TestMessageUnpackable.java +++ b/java/src/test/java/org/msgpack/TestMessageUnpackable.java @@ -24,9 +24,7 @@ public class TestMessageUnpackable { Image dst = new Image(); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - - dst.messageUnpack(pac); + dst.messageUnpack(new Unpacker(in)); assertEquals(src, dst); } diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index daa8f810..b636967f 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -272,63 +272,63 @@ do { \ } while(0) -#ifdef msgpack_pack_inline_func_fastint +#ifdef msgpack_pack_inline_func_fixint -msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d) { unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; buf[0] = 0xcd; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; buf[0] = 0xce; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; buf[0] = 0xcf; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d) { unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; buf[0] = 0xd1; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; buf[0] = 0xd2; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; buf[0] = 0xd3; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } -#undef msgpack_pack_inline_func_fastint +#undef msgpack_pack_inline_func_fixint #endif From 9684c8664fe7643274fe546ccbe0009c741ebc72 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 29 Aug 2010 18:27:10 +0900 Subject: [PATCH 092/259] cpp: version 0.5.4 --- cpp/ChangeLog | 7 ++++++- cpp/configure.in | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/ChangeLog b/cpp/ChangeLog index 2756236a..71c7d5bf 100644 --- a/cpp/ChangeLog +++ b/cpp/ChangeLog @@ -1,5 +1,10 @@ -2010-07-27 version 0.5.3: +2010-08-29 version 0.5.4: + + * includes msgpack_vc2008.vcproj file in source package + * fixes type::fix_int types + +2010-08-27 version 0.5.3: * adds type::fix_{u,}int{8,16,32,64} types * adds msgpack_pack_fix_{u,}int{8,16,32,64} functions diff --git a/cpp/configure.in b/cpp/configure.in index 0104ef90..2dd92d18 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,6 +1,6 @@ AC_INIT(src/object.cpp) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.5.3) +AM_INIT_AUTOMAKE(msgpack, 0.5.4) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From a1bd14e516a0baef6f96b441da70e29e5be7d175 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 06:27:15 +0900 Subject: [PATCH 093/259] template: casts integer types explicitly --- msgpack/pack_template.h | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index b636967f..da54c364 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -103,7 +103,7 @@ do { \ #define msgpack_pack_real_uint64(x, d) \ do { \ if(d < (1ULL<<8)) { \ - if(d < (1<<7)) { \ + if(d < (1ULL<<7)) { \ /* fixnum */ \ msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ @@ -115,12 +115,12 @@ do { \ if(d < (1ULL<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -225,14 +225,14 @@ do { \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,14 +252,14 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ @@ -690,11 +690,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; _msgpack_store16(&buf[1], n); + buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; _msgpack_store32(&buf[1], n); + buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -711,11 +711,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; _msgpack_store16(&buf[1], n); + buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; _msgpack_store32(&buf[1], n); + buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -732,11 +732,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; _msgpack_store16(&buf[1], l); + buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; _msgpack_store32(&buf[1], l); + buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l); msgpack_pack_append_buffer(x, buf, 5); } } From b5c78de2ddf82783a6f80a199b68927d1a1747ca Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 06:30:16 +0900 Subject: [PATCH 094/259] ruby: converts encodings into UTF-8 on Ruby 1.9 --- ruby/encoding.h | 33 ++++++++++++++++++ ruby/pack.c | 21 ++++++++++-- ruby/rbinit.c | 15 +++++++++ ruby/test/test_encoding.rb | 68 ++++++++++++++++++++++++++++++++++++++ ruby/test/test_helper.rb | 4 ++- ruby/unpack.c | 38 +++------------------ 6 files changed, 141 insertions(+), 38 deletions(-) create mode 100644 ruby/encoding.h create mode 100644 ruby/test/test_encoding.rb diff --git a/ruby/encoding.h b/ruby/encoding.h new file mode 100644 index 00000000..2ad3fd7f --- /dev/null +++ b/ruby/encoding.h @@ -0,0 +1,33 @@ +/* + * MessagePack for Ruby + * + * Copyright (C) 2008-2010 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ENCODING_H__ +#define ENCODING_H__ + + +#ifdef HAVE_RUBY_ENCODING_H +#include "ruby/encoding.h" +#define MSGPACK_RUBY_ENCODING +extern int s_enc_utf8; +extern int s_enc_ascii8bit; +extern int s_enc_usascii; +extern VALUE s_enc_utf8_value; +#endif + + +#endif /* encoding.h */ + diff --git a/ruby/pack.c b/ruby/pack.c index bbeac4a5..35878c7d 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,6 +16,8 @@ * limitations under the License. */ #include "ruby.h" +#include "encoding.h" + #include "msgpack/pack_define.h" static ID s_to_msgpack; @@ -131,7 +133,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - // FIXME bignum if(RBIGNUM_SIGN(self)) { // positive msgpack_pack_uint64(out, rb_big2ull(self)); } else { // negative @@ -168,6 +169,14 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); +#ifdef MSGPACK_RUBY_ENCODING + int enc = ENCODING_GET(self); + if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) { + if(!ENC_CODERANGE_ASCIIONLY(self)) { + self = rb_str_encode(self, s_enc_utf8_value, 0, Qnil); + } + } +#endif msgpack_pack_raw(out, RSTRING_LEN(self)); msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self)); return out; @@ -184,12 +193,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { +#ifdef MSGPACK_RUBY_ENCODING + return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self))); +#else ARG_BUFFER(out, argc, argv); const char* name = rb_id2name(SYM2ID(self)); size_t len = strlen(name); msgpack_pack_raw(out, len); msgpack_pack_raw_body(out, name, len); return out; +#endif } @@ -205,7 +218,8 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_array(out, RARRAY_LEN(self)); + // FIXME check sizeof(long) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX + msgpack_pack_array(out, (unsigned int)RARRAY_LEN(self)); VALUE* p = RARRAY_PTR(self); VALUE* const pend = p + RARRAY_LEN(self); for(;p != pend; ++p) { @@ -239,7 +253,8 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_map(out, RHASH_SIZE(self)); + // FIXME check sizeof(st_index_t) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX + msgpack_pack_map(out, (unsigned int)RHASH_SIZE(self)); rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out); return out; } diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 28a8bfec..46781594 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -17,9 +17,17 @@ */ #include "pack.h" #include "unpack.h" +#include "encoding.h" static VALUE mMessagePack; +#ifdef MSGPACK_RUBY_ENCODING +int s_enc_utf8; +int s_enc_ascii8bit; +int s_enc_usascii; +VALUE s_enc_utf8_value; +#endif + /** * Document-module: MessagePack * @@ -46,6 +54,13 @@ void Init_msgpack(void) rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); +#ifdef MSGPACK_RUBY_ENCODING + s_enc_ascii8bit = rb_ascii8bit_encindex(); + s_enc_utf8 = rb_utf8_encindex(); + s_enc_usascii = rb_usascii_encindex(); + s_enc_utf8_value = rb_enc_from_encoding(rb_utf8_encoding()); +#endif + Init_msgpack_unpack(mMessagePack); Init_msgpack_pack(mMessagePack); } diff --git a/ruby/test/test_encoding.rb b/ruby/test/test_encoding.rb new file mode 100644 index 00000000..2cf0767c --- /dev/null +++ b/ruby/test/test_encoding.rb @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby +require File.dirname(__FILE__)+'/test_helper' + +if RUBY_VERSION < "1.9" + exit +end + +class MessagePackTestEncoding < Test::Unit::TestCase + def self.it(name, &block) + define_method("test_#{name}", &block) + end + + it "US-ASCII" do + check_unpack "abc".force_encoding("US-ASCII") + end + + it "UTF-8 ascii" do + check_unpack "abc".force_encoding("UTF-8") + end + + it "UTF-8 mbstr" do + check_unpack "\xE3\x81\x82".force_encoding("UTF-8") + end + + it "UTF-8 invalid" do + check_unpack "\xD0".force_encoding("UTF-8") + end + + it "ASCII-8BIT" do + check_unpack "\xD0".force_encoding("ASCII-8BIT") + end + + it "EUC-JP" do + x = "\xA4\xA2".force_encoding("EUC-JP") + check_unpack(x) + end + + it "EUC-JP invalid" do + begin + "\xD0".force_encoding("EUC-JP").to_msgpack + assert(false) + rescue Encoding::InvalidByteSequenceError + assert(true) + end + end + + private + def check_unpack(str) + if str.encoding.to_s == "ASCII-8BIT" + should_str = str.dup.force_encoding("UTF-8") + else + should_str = str.encode("UTF-8") + end + + raw = str.to_msgpack + r = MessagePack.unpack(str.to_msgpack) + assert_equal(r.encoding.to_s, "UTF-8") + assert_equal(r, should_str.force_encoding("UTF-8")) + + if str.valid_encoding? + sym = str.to_sym + r = MessagePack.unpack(sym.to_msgpack) + assert_equal(r.encoding.to_s, "UTF-8") + assert_equal(r, should_str.force_encoding("UTF-8")) + end + end +end + diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 80d7806a..4def861a 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,4 +5,6 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end -#GC.stress = true +if ENV["GC_STRESS"] + GC.stress = true +end diff --git a/ruby/unpack.c b/ruby/unpack.c index 09481510..3c5e3507 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,17 +16,13 @@ * limitations under the License. */ #include "ruby.h" +#include "encoding.h" #include "msgpack/unpack_define.h" static ID s_sysread; static ID s_readpartial; -#ifdef HAVE_RUBY_ENCODING_H -#include "ruby/encoding.h" -int s_ascii_8bit; -#endif - struct unpack_buffer { size_t size; size_t free; @@ -136,6 +132,9 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } else { *o = rb_str_substr(u->source, p - b, l); } +#ifdef MSGPACK_RUBY_ENCODING + ENCODING_SET(*o, s_enc_utf8); +#endif return 0; } @@ -163,16 +162,6 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #endif -#ifdef HAVE_RUBY_ENCODING_H -static VALUE template_execute_rescue_enc(VALUE data) -{ - rb_gc_enable(); - VALUE* resc = (VALUE*)data; - rb_enc_set_index(resc[0], (int)resc[1]); - RERAISE; -} -#endif - static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); @@ -203,31 +192,16 @@ static int template_execute_wrap(msgpack_unpack_t* mp, (VALUE)from, }; -#ifdef HAVE_RUBY_ENCODING_H - int enc_orig = rb_enc_get_index(str); - rb_enc_set_index(str, s_ascii_8bit); -#endif - // FIXME execute実行中はmp->topが更新されないのでGC markが機能しない rb_gc_disable(); mp->user.source = str; -#ifdef HAVE_RUBY_ENCODING_H - VALUE resc[2] = {str, enc_orig}; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue_enc, (VALUE)resc); -#else int ret = (int)rb_rescue(template_execute_do, (VALUE)args, template_execute_rescue, Qnil); -#endif rb_gc_enable(); -#ifdef HAVE_RUBY_ENCODING_H - rb_enc_set_index(str, enc_orig); -#endif - return ret; } @@ -746,10 +720,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_sysread = rb_intern("sysread"); s_readpartial = rb_intern("readpartial"); -#ifdef HAVE_RUBY_ENCODING_H - s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); -#endif - eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From 09b47cc536ebd951c231fd5e09b4382a25b98020 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 07:00:19 +0900 Subject: [PATCH 095/259] ruby: fixes compatibility with ruby-1.8.5 --- ruby/{encoding.h => compat.h} | 35 +++++++++++++++++++++++++++++++---- ruby/pack.c | 6 +++--- ruby/rbinit.c | 6 +++--- ruby/test/test_pack_unpack.rb | 12 ++++++++---- ruby/unpack.c | 12 +++--------- 5 files changed, 48 insertions(+), 23 deletions(-) rename ruby/{encoding.h => compat.h} (60%) diff --git a/ruby/encoding.h b/ruby/compat.h similarity index 60% rename from ruby/encoding.h rename to ruby/compat.h index 2ad3fd7f..98c88816 100644 --- a/ruby/encoding.h +++ b/ruby/compat.h @@ -15,19 +15,46 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef ENCODING_H__ -#define ENCODING_H__ +#ifndef COMPAT_H__ +#define COMPAT_H__ #ifdef HAVE_RUBY_ENCODING_H #include "ruby/encoding.h" -#define MSGPACK_RUBY_ENCODING +#define COMPAT_HAVE_ENCODING extern int s_enc_utf8; extern int s_enc_ascii8bit; extern int s_enc_usascii; extern VALUE s_enc_utf8_value; #endif +#ifdef RUBY_VM +#define COMPAT_RERAISE rb_exc_raise(rb_errinfo()) +#else +#define COMPAT_RERAISE rb_exc_raise(ruby_errinfo) +#endif -#endif /* encoding.h */ + +/* ruby 1.8.5 */ +#ifndef RSTRING_PTR +#define RSTRING_PTR(s) (RSTRING(s)->ptr) +#endif + +/* ruby 1.8.5 */ +#ifndef RSTRING_LEN +#define RSTRING_LEN(s) (RSTRING(s)->len) +#endif + +/* ruby 1.8.5 */ +#ifndef RARRAY_PTR +#define RARRAY_PTR(s) (RARRAY(s)->ptr) +#endif + +/* ruby 1.8.5 */ +#ifndef RARRAY_LEN +#define RARRAY_LEN(s) (RARRAY(s)->len) +#endif + + +#endif /* compat.h */ diff --git a/ruby/pack.c b/ruby/pack.c index 35878c7d..49b69fc9 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,7 +16,7 @@ * limitations under the License. */ #include "ruby.h" -#include "encoding.h" +#include "compat.h" #include "msgpack/pack_define.h" @@ -169,7 +169,7 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING int enc = ENCODING_GET(self); if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) { if(!ENC_CODERANGE_ASCIIONLY(self)) { @@ -193,7 +193,7 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self))); #else ARG_BUFFER(out, argc, argv); diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 46781594..1d1cbc69 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -17,11 +17,11 @@ */ #include "pack.h" #include "unpack.h" -#include "encoding.h" +#include "compat.h" static VALUE mMessagePack; -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING int s_enc_utf8; int s_enc_ascii8bit; int s_enc_usascii; @@ -54,7 +54,7 @@ void Init_msgpack(void) rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING s_enc_ascii8bit = rb_ascii8bit_encindex(); s_enc_utf8 = rb_utf8_encindex(); s_enc_usascii = rb_usascii_encindex(); diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index 25bde81e..545e5939 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -153,7 +153,8 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "{1=>1}" do - match ({1=>1}), "\x81\x01\x01" + obj = {1=>1} + match obj, "\x81\x01\x01" end it "1.0" do @@ -165,15 +166,18 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "[0, 1, ..., 14]" do - match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + obj = (0..14).to_a + match obj, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" end it "[0, 1, ..., 15]" do - match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + obj = (0..15).to_a + match obj, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" end it "{}" do - match ({}), "\x80" + obj = {} + match obj, "\x80" end ## FIXME diff --git a/ruby/unpack.c b/ruby/unpack.c index 3c5e3507..2d10e75d 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,7 +16,7 @@ * limitations under the License. */ #include "ruby.h" -#include "encoding.h" +#include "compat.h" #include "msgpack/unpack_define.h" @@ -132,7 +132,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } else { *o = rb_str_substr(u->source, p - b, l); } -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING ENCODING_SET(*o, s_enc_utf8); #endif return 0; @@ -155,17 +155,11 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } -#ifdef RUBY_VM -#define RERAISE rb_exc_raise(rb_errinfo()) -#else -#define RERAISE rb_exc_raise(ruby_errinfo) -#endif - static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); - RERAISE; + COMPAT_RERAISE; } static VALUE template_execute_do(VALUE argv) From 71a1cb01842787e2fa897f023addd88337542915 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 09:29:01 +0900 Subject: [PATCH 096/259] fixes compatibility with Rubinius --- ruby/compat.h | 17 +++++++++++++++++ ruby/extconf.rb | 2 +- ruby/pack.c | 8 ++------ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ruby/compat.h b/ruby/compat.h index 98c88816..d7a2ca78 100644 --- a/ruby/compat.h +++ b/ruby/compat.h @@ -35,6 +35,23 @@ extern VALUE s_enc_utf8_value; #endif +/* ruby 1.8 and Rubinius */ +#ifndef RBIGNUM_POSITIVE_P +# ifdef RUBINIUS +# define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue) +# else +# define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign) +# endif +#endif + + +/* Rubinius */ +#ifdef RUBINIUS +static inline void rb_gc_enable() { return; } +static inline void rb_gc_disable() { return; } +#endif + + /* ruby 1.8.5 */ #ifndef RSTRING_PTR #define RSTRING_PTR(s) (RSTRING(s)->ptr) diff --git a/ruby/extconf.rb b/ruby/extconf.rb index eb6a389f..f1d44ec1 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,5 +1,5 @@ require 'mkmf' require './version.rb' -$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"] +$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\" -g] create_makefile('msgpack') diff --git a/ruby/pack.c b/ruby/pack.c index 49b69fc9..8ce46aaf 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -118,10 +118,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) } -#ifndef RBIGNUM_SIGN // Ruby 1.8 -#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) -#endif - /* * Document-method: Bignum#to_msgpack * @@ -133,9 +129,9 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - if(RBIGNUM_SIGN(self)) { // positive + if(RBIGNUM_POSITIVE_P(self)) { msgpack_pack_uint64(out, rb_big2ull(self)); - } else { // negative + } else { msgpack_pack_int64(out, rb_big2ll(self)); } return out; From 23a7137e6a6d7f2910fef2dde305f45f83416194 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Tue, 31 Aug 2010 23:42:32 +0900 Subject: [PATCH 097/259] Perl: better argument validation(patch from dankogai) --- perl/pack.c | 4 ++-- perl/t/08_cycle.t | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index af6669cd..93b2e2f4 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -151,7 +151,7 @@ static int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { - if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); + if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); if (sv==NULL) { @@ -187,7 +187,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; - if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); + if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); svt = SvTYPE(sv); diff --git a/perl/t/08_cycle.t b/perl/t/08_cycle.t index 55d8427a..2bd66c10 100644 --- a/perl/t/08_cycle.t +++ b/perl/t/08_cycle.t @@ -2,7 +2,7 @@ use t::Util; use Test::More; use Data::MessagePack; -plan tests => 5; +plan tests => 6; my $aref = [0]; $aref->[1] = $aref; @@ -23,3 +23,6 @@ ok !$@; eval { Data::MessagePack->pack($aref, 2) }; ok $@, $@; + +eval { Data::MessagePack->pack($aref, -1) }; +ok $@, $@; From 558e9c21edf3cee5813aaa0e7797509eec5d43fb Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 1 Sep 2010 08:19:05 +0900 Subject: [PATCH 098/259] Perl: 0.15 --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 13fc98b5..189990a8 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.15 + + - better argument validation. + (Dan Kogai) + 0.14 - fixed segv on serializing cyclic reference diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 3c38a795..276353a2 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.14'; +our $VERSION = '0.15'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 712b8eec3d90f7e61908cb32c4433ee38a5f1848 Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 1 Sep 2010 11:22:43 +0900 Subject: [PATCH 099/259] added pp version --- perl/lib/Data/MessagePack/PP.pm | 556 ++++++++++++++++++++++++++++++++ 1 file changed, 556 insertions(+) create mode 100644 perl/lib/Data/MessagePack/PP.pm diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm new file mode 100644 index 00000000..f4f1060f --- /dev/null +++ b/perl/lib/Data/MessagePack/PP.pm @@ -0,0 +1,556 @@ +package Data::MessagePack::PP; + +use 5.008000; +use strict; +use B (); +use Scalar::Util qw( blessed ); +use Carp (); + +our $VERSION = '0.03'; + + +# copied from Data::MessagePack +our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; +our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; + +sub true () { $true } +sub false () { $false } + +our $PreferInteger; + +# See also +# http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec +# http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm +# http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack + + +BEGIN { + # for pack and unpack compatibility + if ( $] < 5.010 ) { + require Data::Float; + *pack_double = sub { + my $float_hex = Data::Float::float_hex( $_[0] ); + my ( $sign, $sgnf, $exp ) = $float_hex =~ /^([-+])0x1\.([a-z0-9]+)p([-+][\d]+)$/; + my @bits; + + $sign = $sign eq '-' ? 1 : 0; + $exp = sprintf( '%011b', 1023 + $exp ); + + my $bit = $sign . $exp . join( '', map { unpack('B4', pack('H', $_) ) } split //, $sgnf ); + + while ( $bit =~ /(.{8})/g ) { + push @bits, $1; + } + + return pack( 'C*', 0xcb, map { unpack( 'C', pack("B*", $_ ) ) } @bits ); + }; + *unpack_double = sub { + my $bits = join('', map { sprintf('%08b', $_) } unpack( 'C*', substr( $_[0], $_[1], 8 ) ) ); + my $sign = substr($bits, 0, 1) ? '-' : '+'; + my $sgnf = substr($bits, 12, 52); + my $exp = substr($bits, 1, 11); + $bits = ''; + while ( $sgnf =~ /(.{4})/g ) { + $bits .= unpack('H',pack('B4', $1)); + } + $exp = ((unpack("C*",(pack("B8", (substr('00000'.$exp,0,8) )))) <<8 ) + + unpack("C*",(pack("B8", (substr('00000'.$exp,8,8) ))))) - 1023; + return Data::Float::hex_float( $sign . '0x1.' . $bits . 'p' . $exp ) + 0.0; + }; + *unpack_float = sub { Carp::croak("unpack_float is disable in less than Perl 5.10"); }; + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? -(~$v + 1) : $v; + }; + *unpack_int64 = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); }; + } + else { + *pack_double = sub { return pack 'Cd>', 0xcb, $_[0]; }; + *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_float = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); }; + *unpack_int16 = sub { return unpack 'n!', substr( $_[0], $_[1], 2 ); }; + *unpack_int32 = sub { return unpack 'N!', substr( $_[0], $_[1], 4 ); }; + *unpack_int64 = sub { return unpack 'Q>', substr( $_[0], $_[1], 8 ); }; + } + # for 5.8 etc. + unless ( defined &utf8::is_utf8 ) { + require Encode; + *utf8::is_utf8 = *Encode::is_utf8; + } +} + + +# +# PACK +# + +{ + my $max_depth; + +sub pack { + Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; + $max_depth = defined $_[2] ? $_[2] : 512; # init + return _pack( $_[1] ); +} + + +sub _pack { + my ( $value ) = @_; + + return pack( 'C', 0xc0 ) if ( not defined $value ); + + my $b_obj = B::svref_2object( ref $value ? $value : \$value ); + + if ( $b_obj->isa('B::AV') ) { + my $num = @$value; + my $header = + $num < 16 ? pack( 'C', 0x90 + $num ) + : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xdc, $num ) + : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdd, $num ) + : die "" # don't arrivie here + ; + if ( --$max_depth <= 0 ) { + Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); + } + return join( '', $header, map { _pack( $_ ) } @$value ); + } + + elsif ( $b_obj->isa('B::HV') ) { + my $num = keys %$value; + my $header = + $num < 16 ? pack( 'C', 0x80 + $num ) + : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xde, $num ) + : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdf, $num ) + : die "" # don't arrivie here + ; + if ( --$max_depth <= 0 ) { + Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); + } + return join( '', $header, map { _pack( $_ ) } %$value ); + } + + elsif ( blessed( $value ) eq 'Data::MessagePack::Boolean' ) { + return pack( 'C', $$value ? 0xc3 : 0xc2 ); + } + + my $flags = $b_obj->FLAGS; + + if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) { + + if ($value >= 0) { + return $value <= 127 ? pack 'C', $value + : $value < 2 ** 8 ? pack 'CC', 0xcc, $value + : $value < 2 ** 16 ? pack 'Cn', 0xcd, $value + : $value < 2 ** 32 ? pack 'CN', 0xce, $value + : pack 'CQ>', 0xcf, $value; + } + else { + return -$value <= 32 ? pack 'C', $value + : -$value <= 2 ** 7 ? pack 'Cc', 0xd0, $value + : -$value <= 2 ** 15 ? pack 'Cn', 0xd1, $value + : -$value <= 2 ** 31 ? pack 'CN', 0xd2, $value + : pack 'Cq>', 0xd3, $value; + } + + } + + elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule + + if ( $PreferInteger ) { + if ( $value =~ /^-?[0-9]+$/ ) { # ok? + my $value2 = 0 + $value; + if ( 0 + $value != B::svref_2object( \$value2 )->int_value ) { + local $PreferInteger; # avoid for PV => NV + return _pack( "$value" ); + } + return _pack( $value + 0 ); + } + } + + utf8::encode( $value ) if utf8::is_utf8( $value ); + + my $num = length $value; + my $header = + $num < 32 ? pack( 'C', 0xa0 + $num ) + : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xda, $num ) + : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdb, $num ) + : die "" # don't arrivie here + ; + + return $header . $value; + + } + + elsif ( $flags & ( B::SVf_NOK | B::SVp_NOK ) ) { # double only + return pack_double( $value ); + } + + else { + die "???"; + } + +} + +} # PACK + + +# +# UNPACK +# + +{ + my $p; # position variables for speed. + +sub unpack { + $p = 0; # init + _unpack( $_[1] ); +} + + +sub _unpack { + my ( $value ) = @_; + my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header + + die "invalid data" unless defined $byte; + + if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) { + my $num; + if ( $byte == 0xdc ) { # array 16 + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2; + } + elsif ( $byte == 0xdd ) { # array 32 + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4; + } + else { # fix array + $num = $byte & ~0x90; + } + my @array; + push @array, _unpack( $value ) while $num-- > 0; + return \@array; + } + + elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) { + my $num; + if ( $byte == 0xde ) { # map 16 + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2; + } + elsif ( $byte == 0xdf ) { # map 32 + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4; + } + else { # fix map + $num = $byte & ~0x80; + } + my %map; + for ( 0 .. $num - 1 ) { + my $key = _unpack( $value ); + my $val = _unpack( $value ); + $map{ $key } = $val; + } + return \%map; + } + + elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum + return $byte; + } + elsif ( $byte == 0xcc ) { # uint8 + unpack( 'C', substr( $value, $p++, 1 ) ); + } + elsif ( $byte == 0xcd ) { # uint16 + $p += 2; + return unpack 'n', substr( $value, $p - 2, 2 ); + } + elsif ( $byte == 0xce ) { # unit32 + $p += 4; + return unpack 'N', substr( $value, $p - 4, 4 ); + } + elsif ( $byte == 0xcf ) { # unit64 + $p += 8; + return unpack 'Q>', substr( $value, $p - 8, 8 ); + } + elsif ( $byte == 0xd3 ) { # int64 + $p += 8; + return unpack_int64( $value, $p - 8 ); + return unpack 'q>', substr( $value, $p - 8, 8 ); + } + elsif ( $byte == 0xd2 ) { # int32 + $p += 4; + return unpack_int32( $value, $p - 4 ); + } + elsif ( $byte == 0xd1 ) { # int16 + $p += 2; + return unpack_int16( $value, $p - 2 ); + } + elsif ( $byte == 0xd0 ) { # int8 + return unpack 'c', substr( $value, $p++, 1 ); # c / C + } + elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum + return $byte - 256; + } + + elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { # raw + my $num; + if ( $byte == 0xda ) { + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2 + $num; + } + elsif ( $byte == 0xdb ) { + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4 + $num; + } + else { # fix raw + $num = $byte & ~0xa0; + $p += $num; + } + return substr( $value, $p - $num, $num ); + } + + elsif ( $byte == 0xc0 ) { # nil + return undef; + } + elsif ( $byte == 0xc2 ) { # boolean + return false; + } + elsif ( $byte == 0xc3 ) { # boolean + return true; + } + + elsif ( $byte == 0xcb ) { # double + $p += 8; + return unpack_double( $value, $p - 8 ); + } + + elsif ( $byte == 0xca ) { # float + $p += 4; + return unpack_float( $value, $p - 4 ); + } + + else { + die "???"; + } + +} + + +} # UNPACK + + +# +# Data::MessagePack::Unpacker +# + +package Data::MessagePack::PP::Unpacker; + +use strict; + +sub new { + bless { stack => [] }, shift; +} + + +sub execute_limit { + execute( @_ ); +} + + +{ + my $p; + #my $r; # remained data. + +sub execute { + my ( $self, $data, $offset, $limit ) = @_; + #my $value = ( defined $self->{ remain } ? $self->{ remain } : '' ) . substr( $data, $offset, $limit ); + my $value = substr( $data, $offset, $limit ? $limit : length $data ); + my $len = length $value; + + $p = 0; + #$r = 0; + + while ( $len > $p ) { + _count( $self, $value ) or last; + + if ( @{ $self->{stack} } > 0 ) { + $self->{stack}->[-1]; + pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0; + } + } + + if ( $len == $p ) { + $self->{ data } .= substr( $value, 0, $p ); + $self->{ remain } = undef; + } + else { # I thought this feature is needed. but XS version can't do so + #$self->{ remain } = substr( $value, 0, $p + $r ); + } + + return $p; +} + + +sub _count { + my ( $self, $value ) = @_; + my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header + + if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) { + my $num; + if ( $byte == 0xdc ) { # array 16 + # I thought this feature is needed. but XS version can't do so. So commented out. + #my $len = length substr( $value, $p, 2 ); + #if ( $len != 2 ) { + # $r = $len; + # return 0; + #} + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2; + } + elsif ( $byte == 0xdd ) { # array 32 + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4; + } + else { # fix array + $num = $byte & ~0x90; + } + + push @{ $self->{stack} }, $num + 1; + + return 1; + } + + elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) { + my $num; + if ( $byte == 0xde ) { # map 16 + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2; + } + elsif ( $byte == 0xdf ) { # map 32 + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4; + } + else { # fix map + $num = $byte & ~0x80; + } + + push @{ $self->{stack} }, $num * 2 + 1; # a pair + + return 1; + } + + elsif ( $byte == 0xc0 or $byte == 0xc2 or $byte == 0xc3 ) { # nil, false, true + return 1; + } + + elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum + return 1; + } + + elsif ( $byte >= 0xcc and $byte <= 0xcf ) { # uint + $p += $byte == 0xcc ? 1 + : $byte == 0xcd ? 2 + : $byte == 0xce ? 4 + : $byte == 0xcf ? 8 + : die; + return 1; + } + + elsif ( $byte >= 0xd0 and $byte <= 0xd3 ) { # int + $p += $byte == 0xd0 ? 1 + : $byte == 0xd1 ? 2 + : $byte == 0xd2 ? 4 + : $byte == 0xd3 ? 8 + : die; + return 1; + } + + elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum + return 1; + } + + elsif ( $byte >= 0xca and $byte <= 0xcb ) { # float, double + $p += $byte == 0xca ? 4 : 8; + return 1; + } + + elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { + my $num; + if ( $byte == 0xda ) { + $num = unpack 'n', substr( $value, $p, 2 ); + $p += 2; + } + elsif ( $byte == 0xdb ) { + $num = unpack 'N', substr( $value, $p, 4 ); + $p += 4; + } + else { # fix raw + $num = $byte & ~0xa0; + } + $p += $num; + return 1; + } + + else { + die "???"; + } + + return 0; +} + +} # execute + + +sub data { + my $data = Data::MessagePack->unpack( $_[0]->{ data } ); + $_[0]->reset; + return $data; +} + + +sub is_finished { + my ( $self ) = @_; + ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1; +} + + +sub reset { + $_[0]->{ stack } = []; + $_[0]->{ data } = undef; + $_[0]->{ remain } = undef; +} + +1; +__END__ + +=pod + +=head1 NAME + +Data::MessagePack::PP - the pure perl version of Data::MessagePack + +=head1 LIMITATION + +Currently this module works completely in Perl 5.10 or later. +In Perl 5.8.x, it requires L and cannot unpack int64 and float (pack int64 too). + + +=head1 SEE ALSO + +L, +L, +L + +=head1 AUTHOR + +makamaka + +=head1 COPYRIGHT AND LICENSE + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut From a0705a6c67e852154e92bb16876ac9e950a8f044 Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 1 Sep 2010 11:59:01 +0900 Subject: [PATCH 100/259] added PP backend switch into Data::MessagePack --- perl/Changes | 6 ++++++ perl/lib/Data/MessagePack.pm | 15 +++++++++++++-- perl/lib/Data/MessagePack/PP.pm | 33 +++++++++++++++------------------ 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/perl/Changes b/perl/Changes index 189990a8..a8a4298c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,9 @@ + +0.1x + + - added PP version. + (makamaka) + 0.15 - better argument validation. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 276353a2..a3f8264e 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -1,7 +1,6 @@ package Data::MessagePack; use strict; use warnings; -use XSLoader; use 5.008001; our $VERSION = '0.15'; @@ -12,7 +11,19 @@ our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; sub true () { $true } sub false () { $false } -XSLoader::load(__PACKAGE__, $VERSION); +if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate + if ( $ENV{ PERL_DATA_MESSAGEPACK } !~ /\b pp \b/xms ) { + eval { + require XSLoader; + XSLoader::load(__PACKAGE__, $VERSION); + }; + die $@ if $@ && $ENV{ PERL_DATA_MESSAGEPACK } =~ /\b xs \b/xms; # force XS + } + if ( !__PACKAGE__->can('pack') ) { + print "PP\n"; + require 'Data/MessagePack/PP.pm'; + } +} 1; __END__ diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index f4f1060f..ecb97b46 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -2,21 +2,9 @@ package Data::MessagePack::PP; use 5.008000; use strict; -use B (); -use Scalar::Util qw( blessed ); use Carp (); -our $VERSION = '0.03'; - - -# copied from Data::MessagePack -our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; -our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; - -sub true () { $true } -sub false () { $false } - -our $PreferInteger; +our $VERSION = '0.15'; # See also # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec @@ -24,6 +12,13 @@ our $PreferInteger; # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack +package + Data::MessagePack; + +use Scalar::Util qw( blessed ); +use strict; +use B (); + BEGIN { # for pack and unpack compatibility if ( $] < 5.010 ) { @@ -160,11 +155,11 @@ sub _pack { elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule - if ( $PreferInteger ) { + if ( $Data::MessagePack::PreferInteger ) { if ( $value =~ /^-?[0-9]+$/ ) { # ok? my $value2 = 0 + $value; if ( 0 + $value != B::svref_2object( \$value2 )->int_value ) { - local $PreferInteger; # avoid for PV => NV + local $Data::MessagePack::PreferInteger; # avoid for PV => NV return _pack( "$value" ); } return _pack( $value + 0 ); @@ -346,7 +341,8 @@ sub _unpack { # Data::MessagePack::Unpacker # -package Data::MessagePack::PP::Unpacker; +package + Data::MessagePack::Unpacker; use strict; @@ -530,7 +526,7 @@ __END__ =head1 NAME -Data::MessagePack::PP - the pure perl version of Data::MessagePack +Data::MessagePack::PP - Pure Perl version of Data::MessagePack =head1 LIMITATION @@ -540,9 +536,10 @@ In Perl 5.8.x, it requires L and cannot unpack int64 and float (pac =head1 SEE ALSO +L, L, +L, L, -L =head1 AUTHOR From af83a624743735e1f4404bcd3942e98eee36ce2a Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 1 Sep 2010 16:04:25 +0900 Subject: [PATCH 101/259] modified some codes for test warnings --- perl/lib/Data/MessagePack.pm | 5 ++- perl/lib/Data/MessagePack/PP.pm | 74 +++++++++++++++++---------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index a3f8264e..f8d16254 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -12,12 +12,13 @@ sub true () { $true } sub false () { $false } if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate - if ( $ENV{ PERL_DATA_MESSAGEPACK } !~ /\b pp \b/xms ) { + my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; + if ( $backend !~ /\b pp \b/xms ) { eval { require XSLoader; XSLoader::load(__PACKAGE__, $VERSION); }; - die $@ if $@ && $ENV{ PERL_DATA_MESSAGEPACK } =~ /\b xs \b/xms; # force XS + die $@ if $@ && $backend =~ /\b xs \b/xms; # force XS } if ( !__PACKAGE__->can('pack') ) { print "PP\n"; diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index ecb97b46..1e05bab0 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -84,6 +84,8 @@ BEGIN { # { + no warnings 'recursion'; + my $max_depth; sub pack { @@ -96,16 +98,16 @@ sub pack { sub _pack { my ( $value ) = @_; - return pack( 'C', 0xc0 ) if ( not defined $value ); + return CORE::pack( 'C', 0xc0 ) if ( not defined $value ); my $b_obj = B::svref_2object( ref $value ? $value : \$value ); if ( $b_obj->isa('B::AV') ) { my $num = @$value; my $header = - $num < 16 ? pack( 'C', 0x90 + $num ) - : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xdc, $num ) - : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdd, $num ) + $num < 16 ? CORE::pack( 'C', 0x90 + $num ) + : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc, $num ) + : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) : die "" # don't arrivie here ; if ( --$max_depth <= 0 ) { @@ -117,9 +119,9 @@ sub _pack { elsif ( $b_obj->isa('B::HV') ) { my $num = keys %$value; my $header = - $num < 16 ? pack( 'C', 0x80 + $num ) - : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xde, $num ) - : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdf, $num ) + $num < 16 ? CORE::pack( 'C', 0x80 + $num ) + : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde, $num ) + : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) : die "" # don't arrivie here ; if ( --$max_depth <= 0 ) { @@ -128,8 +130,8 @@ sub _pack { return join( '', $header, map { _pack( $_ ) } %$value ); } - elsif ( blessed( $value ) eq 'Data::MessagePack::Boolean' ) { - return pack( 'C', $$value ? 0xc3 : 0xc2 ); + elsif ( blessed( $value ) and blessed( $value ) eq 'Data::MessagePack::Boolean' ) { + return CORE::pack( 'C', $$value ? 0xc3 : 0xc2 ); } my $flags = $b_obj->FLAGS; @@ -137,18 +139,18 @@ sub _pack { if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) { if ($value >= 0) { - return $value <= 127 ? pack 'C', $value - : $value < 2 ** 8 ? pack 'CC', 0xcc, $value - : $value < 2 ** 16 ? pack 'Cn', 0xcd, $value - : $value < 2 ** 32 ? pack 'CN', 0xce, $value - : pack 'CQ>', 0xcf, $value; + return $value <= 127 ? CORE::pack 'C', $value + : $value < 2 ** 8 ? CORE::pack 'CC', 0xcc, $value + : $value < 2 ** 16 ? CORE::pack 'Cn', 0xcd, $value + : $value < 2 ** 32 ? CORE::pack 'CN', 0xce, $value + : CORE::pack 'CQ>', 0xcf, $value; } else { - return -$value <= 32 ? pack 'C', $value - : -$value <= 2 ** 7 ? pack 'Cc', 0xd0, $value - : -$value <= 2 ** 15 ? pack 'Cn', 0xd1, $value - : -$value <= 2 ** 31 ? pack 'CN', 0xd2, $value - : pack 'Cq>', 0xd3, $value; + return -$value <= 32 ? CORE::pack 'C', ($value & 255) + : -$value <= 2 ** 7 ? CORE::pack 'Cc', 0xd0, $value + : -$value <= 2 ** 15 ? CORE::pack 'Cn', 0xd1, $value + : -$value <= 2 ** 31 ? CORE::pack 'CN', 0xd2, $value + : CORE::pack 'Cq>', 0xd3, $value; } } @@ -170,9 +172,9 @@ sub _pack { my $num = length $value; my $header = - $num < 32 ? pack( 'C', 0xa0 + $num ) - : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xda, $num ) - : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdb, $num ) + $num < 32 ? CORE::pack( 'C', 0xa0 + $num ) + : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num ) + : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num ) : die "" # don't arrivie here ; @@ -198,6 +200,7 @@ sub _pack { # { + my $p; # position variables for speed. sub unpack { @@ -208,18 +211,18 @@ sub unpack { sub _unpack { my ( $value ) = @_; - my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header + my $byte = CORE::unpack( 'C', substr( $value, $p++, 1 ) ); # get header die "invalid data" unless defined $byte; if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) { my $num; if ( $byte == 0xdc ) { # array 16 - $num = unpack 'n', substr( $value, $p, 2 ); + $num = CORE::unpack 'n', substr( $value, $p, 2 ); $p += 2; } elsif ( $byte == 0xdd ) { # array 32 - $num = unpack 'N', substr( $value, $p, 4 ); + $num = CORE::unpack 'N', substr( $value, $p, 4 ); $p += 4; } else { # fix array @@ -233,11 +236,11 @@ sub _unpack { elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) { my $num; if ( $byte == 0xde ) { # map 16 - $num = unpack 'n', substr( $value, $p, 2 ); + $num = CORE::unpack 'n', substr( $value, $p, 2 ); $p += 2; } elsif ( $byte == 0xdf ) { # map 32 - $num = unpack 'N', substr( $value, $p, 4 ); + $num = CORE::unpack 'N', substr( $value, $p, 4 ); $p += 4; } else { # fix map @@ -245,6 +248,7 @@ sub _unpack { } my %map; for ( 0 .. $num - 1 ) { + no warnings; # for undef key case my $key = _unpack( $value ); my $val = _unpack( $value ); $map{ $key } = $val; @@ -256,24 +260,23 @@ sub _unpack { return $byte; } elsif ( $byte == 0xcc ) { # uint8 - unpack( 'C', substr( $value, $p++, 1 ) ); + CORE::unpack( 'C', substr( $value, $p++, 1 ) ); } elsif ( $byte == 0xcd ) { # uint16 $p += 2; - return unpack 'n', substr( $value, $p - 2, 2 ); + return CORE::unpack 'n', substr( $value, $p - 2, 2 ); } elsif ( $byte == 0xce ) { # unit32 $p += 4; - return unpack 'N', substr( $value, $p - 4, 4 ); + return CORE::unpack 'N', substr( $value, $p - 4, 4 ); } elsif ( $byte == 0xcf ) { # unit64 $p += 8; - return unpack 'Q>', substr( $value, $p - 8, 8 ); + return CORE::unpack 'Q>', substr( $value, $p - 8, 8 ); } elsif ( $byte == 0xd3 ) { # int64 $p += 8; return unpack_int64( $value, $p - 8 ); - return unpack 'q>', substr( $value, $p - 8, 8 ); } elsif ( $byte == 0xd2 ) { # int32 $p += 4; @@ -284,7 +287,7 @@ sub _unpack { return unpack_int16( $value, $p - 2 ); } elsif ( $byte == 0xd0 ) { # int8 - return unpack 'c', substr( $value, $p++, 1 ); # c / C + return CORE::unpack 'c', substr( $value, $p++, 1 ); # c / C } elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum return $byte - 256; @@ -293,11 +296,11 @@ sub _unpack { elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { # raw my $num; if ( $byte == 0xda ) { - $num = unpack 'n', substr( $value, $p, 2 ); + $num = CORE::unpack 'n', substr( $value, $p, 2 ); $p += 2 + $num; } elsif ( $byte == 0xdb ) { - $num = unpack 'N', substr( $value, $p, 4 ); + $num = CORE::unpack 'N', substr( $value, $p, 4 ); $p += 4 + $num; } else { # fix raw @@ -373,7 +376,6 @@ sub execute { _count( $self, $value ) or last; if ( @{ $self->{stack} } > 0 ) { - $self->{stack}->[-1]; pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0; } } From 4a15d8b6d2b69bdc1de0b0a7f643b02e00100e66 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 01:29:57 +0900 Subject: [PATCH 102/259] python: Support Python3. --- python/msgpack/__init__.py | 2 +- python/msgpack/_msgpack.pyx | 37 +++++++++++++++++++------------------ python/msgpack/unpack.h | 2 +- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py index 797b29c2..9593714d 100644 --- a/python/msgpack/__init__.py +++ b/python/msgpack/__init__.py @@ -1,3 +1,3 @@ # coding: utf-8 -from _msgpack import * +from msgpack._msgpack import * diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 61ae36b2..6a0b1a5a 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -1,26 +1,24 @@ # coding: utf-8 -import cStringIO - cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject - cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef PyObject* Py_True cdef PyObject* Py_False - cdef char* PyString_AsString(object o) cdef long long PyLong_AsLongLong(object o) cdef unsigned long long PyLong_AsUnsignedLongLong(object o) - cdef int PyMapping_Check(object o) - cdef int PySequence_Check(object o) - cdef int PyLong_Check(object o) - cdef int PyInt_Check(object o) - cdef int PyFloat_Check(object o) - cdef int PyString_Check(object o) - cdef int PyUnicode_Check(object o) + cdef bint PyBool_Check(object o) + cdef bint PyMapping_Check(object o) + cdef bint PySequence_Check(object o) + cdef bint PyLong_Check(object o) + cdef bint PyInt_Check(object o) + cdef bint PyFloat_Check(object o) + cdef bint PyBytes_Check(object o) + cdef bint PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -81,10 +79,12 @@ cdef class Packer(object): if o is None: ret = msgpack_pack_nil(&self.pk) - elif o == Py_True: - ret = msgpack_pack_true(&self.pk) - elif o == Py_False: - ret = msgpack_pack_false(&self.pk) + #elif PyBool_Check(o): + elif isinstance(o, bool): + if o: + ret = msgpack_pack_true(&self.pk) + else: + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) @@ -98,7 +98,7 @@ cdef class Packer(object): elif PyFloat_Check(o): fval = o ret = msgpack_pack_double(&self.pk, fval) - elif PyString_Check(o): + elif PyBytes_Check(o): rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: @@ -133,7 +133,7 @@ cdef class Packer(object): ret = self.__pack(obj) if ret: raise TypeError - buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 return buf @@ -262,10 +262,11 @@ cdef class Unpacker(object): cdef char* buf = self.buf cdef Py_ssize_t tail = self.buf_tail cdef Py_ssize_t l + cdef bytes b for b in self.waiting_bytes: l = len(b) - memcpy(buf + tail, PyString_AsString(b), l) + memcpy(buf + tail, (b), l) tail += l self.buf_tail = tail del self.waiting_bytes[:] diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 61a3786c..9eb8ce77 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -175,7 +175,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - py = PyString_FromStringAndSize(p, l); + py = PyBytes_FromStringAndSize(p, l); if (!py) return -1; *o = py; From 2146f5f623ce4ee2afda660c77bce7852d4a1530 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 02:02:47 +0900 Subject: [PATCH 103/259] python: Fix Unpacker.feed doesn't accept bytes on Python3. --- python/msgpack/_msgpack.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 6a0b1a5a..85d717e6 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -253,9 +253,7 @@ cdef class Unpacker(object): template_init(&self.ctx) self.ctx.user.use_list = use_list - def feed(self, next_bytes): - if not isinstance(next_bytes, str): - raise ValueError, "Argument must be bytes object" + def feed(self, bytes next_bytes): self.waiting_bytes.append(next_bytes) cdef append_buffer(self): From 8d0d2bd3fca404906aef8982a99a81b2acdc50a0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 02:16:28 +0900 Subject: [PATCH 104/259] python: Add test for python3 and fix found problems. --- python/msgpack/_msgpack.pyx | 20 ++++--- python/test3/test_case.py | 102 +++++++++++++++++++++++++++++++++ python/test3/test_except.py | 14 +++++ python/test3/test_format.py | 75 ++++++++++++++++++++++++ python/test3/test_pack.py | 28 +++++++++ python/test3/test_sequnpack.py | 36 ++++++++++++ 6 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 python/test3/test_case.py create mode 100644 python/test3/test_except.py create mode 100644 python/test3/test_format.py create mode 100644 python/test3/test_pack.py create mode 100644 python/test3/test_sequnpack.py diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 85d717e6..c887127b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -12,7 +12,7 @@ cdef extern from "Python.h": cdef unsigned long long PyLong_AsUnsignedLongLong(object o) cdef bint PyBool_Check(object o) - cdef bint PyMapping_Check(object o) + cdef bint PyDict_Check(object o) cdef bint PySequence_Check(object o) cdef bint PyLong_Check(object o) cdef bint PyInt_Check(object o) @@ -69,13 +69,14 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o) except -1: + cdef int _pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval cdef int ret + cdef dict d if o is None: ret = msgpack_pack_nil(&self.pk) @@ -109,19 +110,20 @@ cdef class Packer(object): ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif PyMapping_Check(o): - ret = msgpack_pack_map(&self.pk, len(o)) + elif PyDict_Check(o): + d = o + ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: - for k,v in o.iteritems(): - ret = self.__pack(k) + for k,v in d.items(): + ret = self._pack(k) if ret != 0: break - ret = self.__pack(v) + ret = self._pack(v) if ret != 0: break elif PySequence_Check(o): ret = msgpack_pack_array(&self.pk, len(o)) if ret == 0: for v in o: - ret = self.__pack(v) + ret = self._pack(v) if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. @@ -130,7 +132,7 @@ cdef class Packer(object): def pack(self, object obj): cdef int ret - ret = self.__pack(obj) + ret = self._pack(obj) if ret: raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/python/test3/test_case.py b/python/test3/test_case.py new file mode 100644 index 00000000..53dfcaf0 --- /dev/null +++ b/python/test3/test_case.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, b" " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, (None,) * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, b'\xc0'), + (False, b'\xc2'), + (True, b'\xc3'), + (0, b'\x00'), + (127, b'\x7f'), + (128, b'\xcc\x80'), + (256, b'\xcd\x01\x00'), + (-1, b'\xff'), + (-33, b'\xd0\xdf'), + (-129, b'\xd1\xff\x7f'), + ({1:1}, b'\x81\x01\x01'), + (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ((), b'\x90'), + (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, b'\x80'), + (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/python/test3/test_except.py b/python/test3/test_except.py new file mode 100644 index 00000000..574728fb --- /dev/null +++ b/python/test3/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() diff --git a/python/test3/test_format.py b/python/test3/test_format.py new file mode 100644 index 00000000..022e6801 --- /dev/null +++ b/python/test3/test_format.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import unpacks + +def check(src, should): + assert_equal(unpacks(src), should) + +def testSimpleValue(): + check(b"\x93\xc0\xc2\xc3", + (None, False, True,)) + +def testFixnum(): + check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + ((0,64,127,), (-32,-16,-1,),) + ) + +def testFixArray(): + check(b"\x92\x90\x91\x91\xc0", + ((),((None,),),), + ) + +def testFixRaw(): + check(b"\x94\xa0\xa1a\xa2bc\xa3def", + (b"", b"a", b"bc", b"def",), + ) + +def testFixMap(): + check( + b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) + +def testUnsignedInt(): + check( + b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + b"\xce\xff\xff\xff\xff", + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), + ) + +def testSignedInt(): + check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + b"\xd2\xff\xff\xff\xff", + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) + +def testRaw(): + check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + (b"", b"a", b"ab", b"", b"a", b"ab")) + +def testArray(): + check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + b"\xc2\xc3", + ((), (None,), (False,True), (), (None,), (False,True)) + ) + +def testMap(): + check( + b"\x96" + b"\xde\x00\x00" + b"\xde\x00\x01\xc0\xc2" + b"\xde\x00\x02\xc0\xc2\xc3\xc2" + b"\xdf\x00\x00\x00\x00" + b"\xdf\x00\x00\x00\x01\xc0\xc2" + b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) + +if __name__ == '__main__': + main() diff --git a/python/test3/test_pack.py b/python/test3/test_pack.py new file mode 100644 index 00000000..c861704b --- /dev/null +++ b/python/test3/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + b"", b"a", b"a"*31, b"a"*32, + None, True, False, + (), ((),), ((), None,), + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() diff --git a/python/test3/test_sequnpack.py b/python/test3/test_sequnpack.py new file mode 100644 index 00000000..5fd377c4 --- /dev/null +++ b/python/test3/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + + + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'b') + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') + try: + o = unpacker.unpack() + print(("Oops!", o)) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == e + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From 8fa64e3ab2350faadfe40e0828bafafadac0990a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 09:54:38 +0900 Subject: [PATCH 105/259] Add msgpack.version as version tuple. --- python/.gitignore | 1 + python/Makefile | 7 ++++++- python/msgpack/__init__.py | 1 + python/setup.py | 8 ++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/.gitignore b/python/.gitignore index 430c633e..8531de3f 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -3,3 +3,4 @@ build/* dist/* *.pyc *.pyo +msgpack/__version__.py diff --git a/python/Makefile b/python/Makefile index e06794da..245c09c2 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,12 @@ +.PHONY: test all python3 + all: python setup.py build_ext -i -f python setup.py build sdist -.PHONY: test +python3: + python3 setup.py build_ext -i -f + python3 setup.py build sdist + test: nosetests test diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py index 86786a27..cdf045fb 100644 --- a/python/msgpack/__init__.py +++ b/python/msgpack/__init__.py @@ -1,4 +1,5 @@ # coding: utf-8 +from msgpack.__version__ import * from msgpack._msgpack import * # alias for compatibility to simplejson/marshal/pickle. diff --git a/python/setup.py b/python/setup.py index 64e71ede..c79c1487 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding: utf-8 +version = (0, 1, 5, 'dev') import os from glob import glob @@ -14,7 +15,10 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.1.4' +# make msgpack/__verison__.py +f = open('msgpack/__version__.py', 'w') +f.write("version = %r\n" % (version,)) +f.close() # take care of extension modules. if have_cython: @@ -53,7 +57,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/) setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', - version=version, + version=''.join(str(x) for x in version), cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], From 138d232149e82daabb13d5b6f750aef831ce4398 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 09:58:50 +0900 Subject: [PATCH 106/259] python: vesion 0.1.5 --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index c79c1487..67ff74c4 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 5, 'dev') +version = (0, 1, 5, 'final') import os from glob import glob From a62aefe74bf4017ea90c3dadc86230d12eea5b43 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 10:10:34 +0900 Subject: [PATCH 107/259] python: Release 0.1.6 - Fix wrong version string. --- python/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 67ff74c4..d079e3e3 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 5, 'final') +version = (0, 1, 6, 'final') import os from glob import glob @@ -19,6 +19,9 @@ except ImportError: f = open('msgpack/__version__.py', 'w') f.write("version = %r\n" % (version,)) f.close() +version_str = '.'.join(str(x) for x in version[:3]) +if len(version) > 3 and version[3] != 'final': + version_str += version[3] # take care of extension modules. if have_cython: @@ -57,7 +60,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/) setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', - version=''.join(str(x) for x in version), + version=version_str, cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], From bf0cb4058634cb28450036c296d18185d7d8867a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 10:13:49 +0900 Subject: [PATCH 108/259] python: Add python3 category. --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index d079e3e3..ac7ece5c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -69,6 +69,7 @@ setup(name='msgpack-python', url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ + 'Programming Language :: Python :: 3', 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', From 8fc86ce7fa588657ce841a9cf30ea868c461c4e1 Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 14:33:59 +0900 Subject: [PATCH 109/259] removed commented out codes --- perl/lib/Data/MessagePack.pm | 1 - perl/lib/Data/MessagePack/PP.pm | 12 ------------ perl/{ => xs-src}/MessagePack.c | 0 perl/{ => xs-src}/pack.c | 0 perl/{ => xs-src}/unpack.c | 0 5 files changed, 13 deletions(-) rename perl/{ => xs-src}/MessagePack.c (100%) rename perl/{ => xs-src}/pack.c (100%) rename perl/{ => xs-src}/unpack.c (100%) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index f8d16254..785f275d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -21,7 +21,6 @@ if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate die $@ if $@ && $backend =~ /\b xs \b/xms; # force XS } if ( !__PACKAGE__->can('pack') ) { - print "PP\n"; require 'Data/MessagePack/PP.pm'; } } diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 1e05bab0..540b416d 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -361,16 +361,13 @@ sub execute_limit { { my $p; - #my $r; # remained data. sub execute { my ( $self, $data, $offset, $limit ) = @_; - #my $value = ( defined $self->{ remain } ? $self->{ remain } : '' ) . substr( $data, $offset, $limit ); my $value = substr( $data, $offset, $limit ? $limit : length $data ); my $len = length $value; $p = 0; - #$r = 0; while ( $len > $p ) { _count( $self, $value ) or last; @@ -384,9 +381,6 @@ sub execute { $self->{ data } .= substr( $value, 0, $p ); $self->{ remain } = undef; } - else { # I thought this feature is needed. but XS version can't do so - #$self->{ remain } = substr( $value, 0, $p + $r ); - } return $p; } @@ -399,12 +393,6 @@ sub _count { if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) { my $num; if ( $byte == 0xdc ) { # array 16 - # I thought this feature is needed. but XS version can't do so. So commented out. - #my $len = length substr( $value, $p, 2 ); - #if ( $len != 2 ) { - # $r = $len; - # return 0; - #} $num = unpack 'n', substr( $value, $p, 2 ); $p += 2; } diff --git a/perl/MessagePack.c b/perl/xs-src/MessagePack.c similarity index 100% rename from perl/MessagePack.c rename to perl/xs-src/MessagePack.c diff --git a/perl/pack.c b/perl/xs-src/pack.c similarity index 100% rename from perl/pack.c rename to perl/xs-src/pack.c diff --git a/perl/unpack.c b/perl/xs-src/unpack.c similarity index 100% rename from perl/unpack.c rename to perl/xs-src/unpack.c From 918dbd1926589a9b70f34037e35e98f3194302fc Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 14:37:22 +0900 Subject: [PATCH 110/259] made Makefile.PL XS/PP configurable --- perl/Makefile.PL | 69 +++++++++++++++++++++++++++++++++++++++------ perl/t/00_compile.t | 2 +- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index e9f9618a..7440a46d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -5,15 +5,29 @@ name 'Data-MessagePack'; all_from 'lib/Data/MessagePack.pm'; readme_from('lib/Data/MessagePack.pm'); -perl_version '5.008005'; +perl_version '5.008000'; license 'perl'; -can_cc or die "This module requires a C compiler"; tests 't/*.t'; recursive_author_tests('xt'); -use_ppport 3.19; -requires_c99(); # msgpack C library requires C99. +my $use_xs = want_xs(); + +if ( $] >= 5.008005 and $use_xs ) { + can_cc or die "This module requires a C compiler"; + use_ppport 3.19; + requires_c99(); # msgpack C library requires C99. + cc_src_paths('xs-src'); + if ($ENV{DEBUG}) { + cc_append_to_ccflags '-g'; + } + # for author's test_pp + requires 'Data::Float' => 0 if ( $Module::Install::AUTHOR and $] < 5.010 ); +} +else { # for Data::MessagePack::PP + print "configure PP version\n"; + requires 'Data::Float' => 0 if ( $] < 5.010 ); +} clean_files qw{ *.stackdump @@ -23,10 +37,6 @@ clean_files qw{ cover_db }; -if ($ENV{DEBUG}) { - cc_append_to_ccflags '-g'; -} - # copy modules if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { mkdir 'msgpack' unless -d 'msgpack'; @@ -39,7 +49,50 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { requires 'Test::More' => 0.94; # done_testing test_requires('Test::Requires'); +test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' ); + +if($Module::Install::AUTHOR) { + postamble qq{test :: test_pp\n\n}; +} + auto_set_repository(); auto_include; WriteAll; +# copied from Makefile.PL in Text::Xslate. +sub test_with_env { + my($name, %env) = @_; + + my $dir = '.testenv'; + if(not -e $dir) { + mkdir $dir or die "Cannot mkdir '.testenv': $!"; + } + clean_files($dir); + + { + open my $out, '>', "$dir/$name.pl" + or die "Cannot open '$dir/$name.pl' for writing: $!"; + print $out "# This file sets the env for 'make $name', \n"; + print $out "# generated by $0 at ", scalar(localtime), ".\n"; + print $out "# DO NOT EDIT THIS FILE DIRECTLY.\n"; + print $out "\n"; + + while(my($name, $value) = each %env) { + printf $out '$ENV{q{%s}} = q{%s};'."\n", $name, $value; + } + } + + # repeat testing for pure Perl mode + # see also ExtUtils::MM_Any::test_via_harness() + + my $t = q{$(FULLPERLRUN) -MExtUtils::Command::MM -e} + .q{ "do q[%s]; test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')"} + .q{ $(TEST_FILES)}; + + postamble qq{$name :: pure_all\n} + . qq{\t} . q{$(NOECHO) $(ECHO) TESTING: } . $name . qq{\n} + . qq{\t} . sprintf($t, "$dir/$name.pl") . qq{\n\n} + + . qq{testall :: $name\n\n}; + return; +} diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t index 66fe8f0e..f91b29e7 100644 --- a/perl/t/00_compile.t +++ b/perl/t/00_compile.t @@ -3,4 +3,4 @@ use warnings; use Test::More tests => 1; use_ok 'Data::MessagePack'; - +diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' ); From 8f43e033a49aaf1bacb8fb887a0f7b7a538c4031 Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 23:45:05 +0900 Subject: [PATCH 111/259] removed dependency on Data::Float --- perl/Makefile.PL | 5 +-- perl/lib/Data/MessagePack/PP.pm | 70 +++++++++++++-------------------- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 7440a46d..b7864854 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -21,12 +21,9 @@ if ( $] >= 5.008005 and $use_xs ) { if ($ENV{DEBUG}) { cc_append_to_ccflags '-g'; } - # for author's test_pp - requires 'Data::Float' => 0 if ( $Module::Install::AUTHOR and $] < 5.010 ); } -else { # for Data::MessagePack::PP +else { print "configure PP version\n"; - requires 'Data::Float' => 0 if ( $] < 5.010 ); } clean_files qw{ diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 540b416d..86583733 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -22,37 +22,19 @@ use B (); BEGIN { # for pack and unpack compatibility if ( $] < 5.010 ) { - require Data::Float; - *pack_double = sub { - my $float_hex = Data::Float::float_hex( $_[0] ); - my ( $sign, $sgnf, $exp ) = $float_hex =~ /^([-+])0x1\.([a-z0-9]+)p([-+][\d]+)$/; - my @bits; - - $sign = $sign eq '-' ? 1 : 0; - $exp = sprintf( '%011b', 1023 + $exp ); - - my $bit = $sign . $exp . join( '', map { unpack('B4', pack('H', $_) ) } split //, $sgnf ); - - while ( $bit =~ /(.{8})/g ) { - push @bits, $1; - } - - return pack( 'C*', 0xcb, map { unpack( 'C', pack("B*", $_ ) ) } @bits ); - }; - *unpack_double = sub { - my $bits = join('', map { sprintf('%08b', $_) } unpack( 'C*', substr( $_[0], $_[1], 8 ) ) ); - my $sign = substr($bits, 0, 1) ? '-' : '+'; - my $sgnf = substr($bits, 12, 52); - my $exp = substr($bits, 1, 11); - $bits = ''; - while ( $sgnf =~ /(.{4})/g ) { - $bits .= unpack('H',pack('B4', $1)); - } - $exp = ((unpack("C*",(pack("B8", (substr('00000'.$exp,0,8) )))) <<8 ) - + unpack("C*",(pack("B8", (substr('00000'.$exp,8,8) ))))) - 1023; - return Data::Float::hex_float( $sign . '0x1.' . $bits . 'p' . $exp ) + 0.0; - }; - *unpack_float = sub { Carp::croak("unpack_float is disable in less than Perl 5.10"); }; + my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE + *pack_double = $bo_is_le ? sub { + my @v = unpack( 'V2', pack( 'd', $_[0] ) ); + return pack 'CN2', 0xcb, @v[1,0]; + } : sub { pack 'Cd', 0xcb, $_[0]; }; + *unpack_float = $bo_is_le ? sub { + my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) ); + return unpack( 'f', pack( 'n2', @v[1,0] ) ); + } : sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; + *unpack_double = $bo_is_le ? sub { + my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) ); + return unpack( 'd', pack( 'N2', @v[1,0] ) ); + } : sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; *unpack_int16 = sub { my $v = unpack 'n', substr( $_[0], $_[1], 2 ); return $v ? $v - 0x10000 : 0; @@ -62,14 +44,16 @@ BEGIN { return $v ? -(~$v + 1) : $v; }; *unpack_int64 = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); }; + *unpack_uint64 = sub { Carp::croak("unpack_uint64 is disable in less than Perl 5.10"); }; } else { *pack_double = sub { return pack 'Cd>', 0xcb, $_[0]; }; - *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; *unpack_float = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); }; - *unpack_int16 = sub { return unpack 'n!', substr( $_[0], $_[1], 2 ); }; - *unpack_int32 = sub { return unpack 'N!', substr( $_[0], $_[1], 4 ); }; - *unpack_int64 = sub { return unpack 'Q>', substr( $_[0], $_[1], 8 ); }; + *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_int16 = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); }; + *unpack_int32 = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); }; + *unpack_int64 = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } # for 5.8 etc. unless ( defined &utf8::is_utf8 ) { @@ -272,7 +256,7 @@ sub _unpack { } elsif ( $byte == 0xcf ) { # unit64 $p += 8; - return CORE::unpack 'Q>', substr( $value, $p - 8, 8 ); + return pack_uint64( $value, $p - 8 ); } elsif ( $byte == 0xd3 ) { # int64 $p += 8; @@ -516,25 +500,25 @@ __END__ =head1 NAME -Data::MessagePack::PP - Pure Perl version of Data::MessagePack - -=head1 LIMITATION - -Currently this module works completely in Perl 5.10 or later. -In Perl 5.8.x, it requires L and cannot unpack int64 and float (pack int64 too). +Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack =head1 SEE ALSO L, L, -L, L, =head1 AUTHOR makamaka +=head1 LIMITATION + +Currently this module works completely in Perl 5.10 or later. +In Perl 5.8.x, it cannot C uint64 and int64. + + =head1 COPYRIGHT AND LICENSE This library is free software; you can redistribute it and/or modify From 4cc6c3e535e1181dcd9810fd862b80954246f9b7 Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 23:48:57 +0900 Subject: [PATCH 112/259] modified t/05_preferred_int.t for Win32 --- perl/t/05_preferred_int.t | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 9860711b..67d11aaf 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -16,6 +16,7 @@ sub pis ($$) { # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0])); } +my $is_win = $^O eq 'MSWin32'; my @dat = ( '', 'a0', '0', '00', @@ -29,12 +30,16 @@ my @dat = ( ''.0xFFFFFF => 'ce 00 ff ff ff', ''.0xFFFFFFFF => 'ce ff ff ff ff', ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', - ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => $is_win ? + 'b5 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 30 33 34' + : 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', '-'.0x8000000 => 'd2 f8 00 00 00', '-'.0x80000000 => 'd2 80 00 00 00', '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38', '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38', - '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35', + '-'.0x800000000000000000000000000000 => $is_win ? + 'b6 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 30 33 35' + : 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35', {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); From cdc09a7d30e3390dba17db64df121a2dc34c8f04 Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 23:52:36 +0900 Subject: [PATCH 113/259] Changes --- perl/Changes | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index a8a4298c..a9bb2dbb 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,7 +1,10 @@ 0.1x - - added PP version. + - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS). + - made Makefile.PL PP configurable. + - test_pp in author's test + - modified t/05_preferred_int.t for Win32 (makamaka) 0.15 From 2b75d54ce14521b70b63d7aa808a005ac8dafdfa Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 23:56:55 +0900 Subject: [PATCH 114/259] modified pod --- perl/lib/Data/MessagePack/PP.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 86583733..270db343 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -509,16 +509,16 @@ L, L, L, -=head1 AUTHOR - -makamaka - =head1 LIMITATION Currently this module works completely in Perl 5.10 or later. In Perl 5.8.x, it cannot C uint64 and int64. +=head1 AUTHOR + +makamaka + =head1 COPYRIGHT AND LICENSE This library is free software; you can redistribute it and/or modify From f91728561fe9c374edb93262e7c9a7c1d819d284 Mon Sep 17 00:00:00 2001 From: makamaka Date: Thu, 2 Sep 2010 23:58:40 +0900 Subject: [PATCH 115/259] ouch, modified pod --- perl/lib/Data/MessagePack/PP.pm | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 270db343..5d956078 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -502,6 +502,11 @@ __END__ Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack +=head1 LIMITATION + +Currently this module works completely in Perl 5.10 or later. +In Perl 5.8.x, it cannot C uint64 and int64. + =head1 SEE ALSO @@ -509,12 +514,6 @@ L, L, L, -=head1 LIMITATION - -Currently this module works completely in Perl 5.10 or later. -In Perl 5.8.x, it cannot C uint64 and int64. - - =head1 AUTHOR makamaka From b97baf4d4713580e89e0dca3bad350339618923e Mon Sep 17 00:00:00 2001 From: makamaka Date: Fri, 3 Sep 2010 12:53:56 +0900 Subject: [PATCH 116/259] added some comments in Data::MessagePack::PP --- perl/lib/Data/MessagePack/PP.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 5d956078..e01b7972 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -22,7 +22,10 @@ use B (); BEGIN { # for pack and unpack compatibility if ( $] < 5.010 ) { + # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); + # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE + # In really, since 5.9.2 '>' is introduced. *pack_double = $bo_is_le ? sub { my @v = unpack( 'V2', pack( 'd', $_[0] ) ); return pack 'CN2', 0xcb, @v[1,0]; From 1fe4109a42d717aea41ea7ffd7a3193208711e77 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 3 Sep 2010 14:50:01 +0900 Subject: [PATCH 117/259] fixed tests on 64bit machines with -Duselongdouble #60625 --- perl/t/05_preferred_int.t | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 9860711b..fe14ef6c 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -12,7 +12,11 @@ sub packit { } sub pis ($$) { - is packit($_[0]), $_[1], 'dump ' . $_[1]; + if (ref $_[1]) { + like packit($_[0]), $_[1], 'dump ' . $_[1]; + } else { + is packit($_[0]), $_[1], 'dump ' . $_[1]; + } # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0])); } @@ -29,12 +33,12 @@ my @dat = ( ''.0xFFFFFF => 'ce 00 ff ff ff', ''.0xFFFFFFFF => 'ce ff ff ff ff', ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', - ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$}, '-'.0x8000000 => 'd2 f8 00 00 00', '-'.0x80000000 => 'd2 80 00 00 00', '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38', '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38', - '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35', + '-'.0x800000000000000000000000000000 => qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)}, {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); From adfadc542a98dcc7d838778797b512ccf8bd78f2 Mon Sep 17 00:00:00 2001 From: makamaka Date: Sat, 4 Sep 2010 14:35:24 +0900 Subject: [PATCH 118/259] enable PP to pack/unpack int64 in less than Perl 5.10 --- perl/lib/Data/MessagePack/PP.pm | 39 +++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index e01b7972..bd37ad76 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -26,6 +26,14 @@ BEGIN { # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE # In really, since 5.9.2 '>' is introduced. + *pack_uint64 = $bo_is_le ? sub { + my @v = unpack( 'V2', pack( 'Q', $_[0] ) ); + return pack 'CN2', 0xcf, @v[1,0]; + } : sub { pack 'CQ', 0xcf, $_[0]; }; + *pack_int64 = $bo_is_le ? sub { + my @v = unpack( 'V2', pack( 'q', $_[0] ) ); + return pack 'CN2', 0xd3, @v[1,0]; + } : sub { pack 'Cq', 0xd3, $_[0]; }; *pack_double = $bo_is_le ? sub { my @v = unpack( 'V2', pack( 'd', $_[0] ) ); return pack 'CN2', 0xcb, @v[1,0]; @@ -43,13 +51,22 @@ BEGIN { return $v ? $v - 0x10000 : 0; }; *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? -(~$v + 1) : $v; + return $v ? $v - 0x100000000 : 0; + }; + *unpack_int64 = sub { + my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); + return unpack( 'q', pack( 'N2', @v[1,0] ) ); + }; + *unpack_uint64 = sub { + my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); + return unpack( 'Q', pack( 'N2', @v[1,0] ) ); }; - *unpack_int64 = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); }; - *unpack_uint64 = sub { Carp::croak("unpack_uint64 is disable in less than Perl 5.10"); }; } else { + *pack_uint64 = sub { return pack 'CQ>', 0xcf, $_[0]; }; + *pack_int64 = sub { return pack 'Cq>', 0xd3, $_[0]; }; *pack_double = sub { return pack 'Cd>', 0xcb, $_[0]; }; *unpack_float = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; @@ -130,14 +147,14 @@ sub _pack { : $value < 2 ** 8 ? CORE::pack 'CC', 0xcc, $value : $value < 2 ** 16 ? CORE::pack 'Cn', 0xcd, $value : $value < 2 ** 32 ? CORE::pack 'CN', 0xce, $value - : CORE::pack 'CQ>', 0xcf, $value; + : pack_uint64( $value ); } else { return -$value <= 32 ? CORE::pack 'C', ($value & 255) : -$value <= 2 ** 7 ? CORE::pack 'Cc', 0xd0, $value : -$value <= 2 ** 15 ? CORE::pack 'Cn', 0xd1, $value : -$value <= 2 ** 31 ? CORE::pack 'CN', 0xd2, $value - : CORE::pack 'Cq>', 0xd3, $value; + : pack_int64( $value ); } } @@ -147,7 +164,9 @@ sub _pack { if ( $Data::MessagePack::PreferInteger ) { if ( $value =~ /^-?[0-9]+$/ ) { # ok? my $value2 = 0 + $value; - if ( 0 + $value != B::svref_2object( \$value2 )->int_value ) { + if ( $value > 0xFFFFFFFF or $value < '-'.0x80000000 or # <- needless but for XS compat + 0 + $value != B::svref_2object( \$value2 )->int_value + ) { local $Data::MessagePack::PreferInteger; # avoid for PV => NV return _pack( "$value" ); } @@ -259,7 +278,7 @@ sub _unpack { } elsif ( $byte == 0xcf ) { # unit64 $p += 8; - return pack_uint64( $value, $p - 8 ); + return unpack_uint64( $value, $p - 8 ); } elsif ( $byte == 0xd3 ) { # int64 $p += 8; @@ -505,11 +524,9 @@ __END__ Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack -=head1 LIMITATION - -Currently this module works completely in Perl 5.10 or later. -In Perl 5.8.x, it cannot C uint64 and int64. +=head1 DESCRIPTION +This module is used by L internally. =head1 SEE ALSO From 25531d83936a1253a9dc5ee1b0f4f771d301317d Mon Sep 17 00:00:00 2001 From: makamaka Date: Sat, 4 Sep 2010 19:54:12 +0900 Subject: [PATCH 119/259] modified t/05_preferred_int.t for Win32 --- perl/t/05_preferred_int.t | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 9eb223ad..084df31c 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -34,12 +34,16 @@ my @dat = ( ''.0xFFFFFF => 'ce 00 ff ff ff', ''.0xFFFFFFFF => 'ce ff ff ff ff', ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', - ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$}, + ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => $is_win ? + qr{^(b5 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 30 33 34|b8 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 30 33 34)$} + : qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$}, '-'.0x8000000 => 'd2 f8 00 00 00', '-'.0x80000000 => 'd2 80 00 00 00', '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38', '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38', - '-'.0x800000000000000000000000000000 => qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)}, + '-'.0x800000000000000000000000000000 => $is_win ? + qr{^(b6 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 30 33 35|b9 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 30 33 35)} + : qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)}, {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); From 84123f544524d6ffd118b91cd7e053d0a6d8bbe4 Mon Sep 17 00:00:00 2001 From: makamaka Date: Sat, 4 Sep 2010 20:02:46 +0900 Subject: [PATCH 120/259] fallback PP configuration with c99 unspport compiler --- perl/Makefile.PL | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index b7864854..783e658d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -11,19 +11,31 @@ license 'perl'; tests 't/*.t'; recursive_author_tests('xt'); -my $use_xs = want_xs(); -if ( $] >= 5.008005 and $use_xs ) { - can_cc or die "This module requires a C compiler"; - use_ppport 3.19; - requires_c99(); # msgpack C library requires C99. - cc_src_paths('xs-src'); - if ($ENV{DEBUG}) { - cc_append_to_ccflags '-g'; +if ( $] >= 5.008005 and want_xs() ) { + can_cc or die "This module requires a C compiler. Please retry with --pp"; + + my $has_c99 = c99_available(); # msgpack C library requires C99. + + if ( $has_c99 ) { + use_ppport 3.19; + cc_src_paths('xs-src'); + if ($ENV{DEBUG}) { + cc_append_to_ccflags '-g'; + } + } + else { + print < Date: Sun, 5 Sep 2010 01:54:44 +0900 Subject: [PATCH 121/259] modified begin process about byte order --- perl/lib/Data/MessagePack/PP.pm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index bd37ad76..9e322991 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -25,19 +25,19 @@ BEGIN { # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE - # In really, since 5.9.2 '>' is introduced. + # In really, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? *pack_uint64 = $bo_is_le ? sub { my @v = unpack( 'V2', pack( 'Q', $_[0] ) ); return pack 'CN2', 0xcf, @v[1,0]; - } : sub { pack 'CQ', 0xcf, $_[0]; }; + } : sub { pack 'CQ', 0xcf, $_[0]; }; *pack_int64 = $bo_is_le ? sub { my @v = unpack( 'V2', pack( 'q', $_[0] ) ); return pack 'CN2', 0xd3, @v[1,0]; - } : sub { pack 'Cq', 0xd3, $_[0]; }; + } : sub { pack 'Cq', 0xd3, $_[0]; }; *pack_double = $bo_is_le ? sub { my @v = unpack( 'V2', pack( 'd', $_[0] ) ); return pack 'CN2', 0xcb, @v[1,0]; - } : sub { pack 'Cd', 0xcb, $_[0]; }; + } : sub { pack 'Cd', 0xcb, $_[0]; }; *unpack_float = $bo_is_le ? sub { my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) ); return unpack( 'f', pack( 'n2', @v[1,0] ) ); @@ -55,14 +55,14 @@ BEGIN { my $v = unpack 'N', substr( $_[0], $_[1], 4 ); return $v ? $v - 0x100000000 : 0; }; - *unpack_int64 = sub { + *unpack_int64 = $bo_is_le ? sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); - }; - *unpack_uint64 = sub { + } : sub { pack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $bo_is_le ? sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'Q', pack( 'N2', @v[1,0] ) ); - }; + } : sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; } else { *pack_uint64 = sub { return pack 'CQ>', 0xcf, $_[0]; }; From b9bca2a19fc6519296bcda2c6af5f82cc744e005 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 5 Sep 2010 16:17:19 +0900 Subject: [PATCH 122/259] bump to 0.16 --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 189990a8..448130b4 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.16 + + - tests on 64bit machines with -Duselongdouble + (reported by andk) + 0.15 - better argument validation. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 276353a2..94f28c19 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.15'; +our $VERSION = '0.16'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 80db9971b5a579a1388e0e110baa4a8ec3d1ea7c Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Mon, 6 Sep 2010 01:32:00 +0900 Subject: [PATCH 123/259] pure haskell implementation. --- haskell/LICENSE | 2 +- haskell/cbits/msgpack.c | 137 ---- haskell/msgpack.cabal | 55 +- haskell/src/Data/MessagePack.hs | 130 ++-- haskell/src/Data/MessagePack/Base.hsc | 584 ------------------ haskell/src/Data/MessagePack/Feed.hs | 62 -- haskell/src/Data/MessagePack/Monad.hs | 156 ----- .../Data/MessagePack/{Class.hs => Object.hs} | 38 +- haskell/src/Data/MessagePack/Packer.hs | 147 +++++ haskell/src/Data/MessagePack/Put.hs | 202 ++++++ haskell/src/Data/MessagePack/Stream.hs | 82 --- haskell/test/Monad.hs | 15 +- haskell/test/Stream.hs | 14 - haskell/test/Test.hs | 69 ++- 14 files changed, 543 insertions(+), 1150 deletions(-) delete mode 100644 haskell/cbits/msgpack.c delete mode 100644 haskell/src/Data/MessagePack/Base.hsc delete mode 100644 haskell/src/Data/MessagePack/Feed.hs delete mode 100644 haskell/src/Data/MessagePack/Monad.hs rename haskell/src/Data/MessagePack/{Class.hs => Object.hs} (77%) create mode 100644 haskell/src/Data/MessagePack/Packer.hs create mode 100644 haskell/src/Data/MessagePack/Put.hs delete mode 100644 haskell/src/Data/MessagePack/Stream.hs delete mode 100644 haskell/test/Stream.hs diff --git a/haskell/LICENSE b/haskell/LICENSE index 2de30f66..3cb4d8c8 100644 --- a/haskell/LICENSE +++ b/haskell/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009, Hideyuki Tanaka +Copyright (c) 2009-2010, Hideyuki Tanaka All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/haskell/cbits/msgpack.c b/haskell/cbits/msgpack.c deleted file mode 100644 index be445925..00000000 --- a/haskell/cbits/msgpack.c +++ /dev/null @@ -1,137 +0,0 @@ -#include - -void msgpack_sbuffer_init_wrap(msgpack_sbuffer* sbuf) -{ - msgpack_sbuffer_init(sbuf); -} - -void msgpack_sbuffer_destroy_wrap(msgpack_sbuffer* sbuf) -{ - msgpack_sbuffer_destroy(sbuf); -} - -int msgpack_sbuffer_write_wrap(void* data, const char* buf, unsigned int len) -{ - return msgpack_sbuffer_write(data, buf, len); -} - -msgpack_packer* msgpack_packer_new_wrap(void *data, msgpack_packer_write callback) -{ - return msgpack_packer_new(data, callback); -} - -void msgpack_packer_free_wrap(msgpack_packer* pk) -{ - msgpack_packer_free(pk); -} - -int msgpack_pack_uint8_wrap(msgpack_packer* pk, uint8_t d) -{ - return msgpack_pack_uint8(pk, d); -} - -int msgpack_pack_uint16_wrap(msgpack_packer* pk, uint16_t d) -{ - return msgpack_pack_uint16(pk, d); -} - -int msgpack_pack_uint32_wrap(msgpack_packer* pk, uint32_t d) -{ - return msgpack_pack_uint32(pk, d); -} - -int msgpack_pack_uint64_wrap(msgpack_packer* pk, uint64_t d) -{ - return msgpack_pack_uint64(pk, d); -} - -int msgpack_pack_int8_wrap(msgpack_packer* pk, int8_t d) -{ - return msgpack_pack_int8(pk, d); -} - -int msgpack_pack_int16_wrap(msgpack_packer* pk, int16_t d) -{ - return msgpack_pack_int16(pk, d); -} - -int msgpack_pack_int32_wrap(msgpack_packer* pk, int32_t d) -{ - return msgpack_pack_int32(pk, d); -} - -int msgpack_pack_int64_wrap(msgpack_packer* pk, int64_t d) -{ - return msgpack_pack_int64(pk, d); -} - -int msgpack_pack_double_wrap(msgpack_packer* pk, double d) -{ - return msgpack_pack_double(pk, d); -} - -int msgpack_pack_nil_wrap(msgpack_packer* pk) -{ - return msgpack_pack_nil(pk); -} - -int msgpack_pack_true_wrap(msgpack_packer* pk) -{ - return msgpack_pack_true(pk); -} - -int msgpack_pack_false_wrap(msgpack_packer* pk) -{ - return msgpack_pack_false(pk); -} - -int msgpack_pack_array_wrap(msgpack_packer* pk, unsigned int n) -{ - return msgpack_pack_array(pk, n); -} - -int msgpack_pack_map_wrap(msgpack_packer* pk, unsigned int n) -{ - return msgpack_pack_map(pk, n); -} - -int msgpack_pack_raw_wrap(msgpack_packer* pk, size_t l) -{ - return msgpack_pack_raw(pk, l); -} - -int msgpack_pack_raw_body_wrap(msgpack_packer* pk, const void *b, size_t l) -{ - return msgpack_pack_raw_body(pk, b, l); -} - -bool msgpack_unpacker_reserve_buffer_wrap(msgpack_unpacker *mpac, size_t size) -{ - return msgpack_unpacker_reserve_buffer(mpac, size); -} - -char *msgpack_unpacker_buffer_wrap(msgpack_unpacker *mpac) -{ - return msgpack_unpacker_buffer(mpac); -} - -size_t msgpack_unpacker_buffer_capacity_wrap(const msgpack_unpacker *mpac) -{ - return msgpack_unpacker_buffer_capacity(mpac); -} - -void msgpack_unpacker_buffer_consumed_wrap(msgpack_unpacker *mpac, size_t size) -{ - msgpack_unpacker_buffer_consumed(mpac, size); -} - -void msgpack_unpacker_data_wrap(msgpack_unpacker *mpac, msgpack_object *obj) -{ - *obj=msgpack_unpacker_data(mpac); -} - -size_t msgpack_unpacker_message_size_wrap(const msgpack_unpacker *mpac) -{ - return msgpack_unpacker_message_size(mpac); -} - diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 82cdb525..8346c1f8 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,32 +1,35 @@ -Name: msgpack -Version: 0.2.2 -License: BSD3 -License-File: LICENSE -Author: Hideyuki Tanaka -Maintainer: Hideyuki Tanaka -Category: Data -Synopsis: A Haskell binding to MessagePack +Name: msgpack +Version: 0.3.0 +Synopsis: A Haskell binding to MessagePack Description: A Haskell binding to MessagePack -Homepage: http://github.com/tanakh/hsmsgpack -Stability: Experimental -Tested-with: GHC==6.10.4 -Cabal-Version: >=1.2 -Build-Type: Simple -library - build-depends: base>=4 && <5, mtl, bytestring - ghc-options: -O2 -Wall - hs-source-dirs: src - extra-libraries: msgpackc +License: BSD3 +License-File: LICENSE +Category: Data +Author: Hideyuki Tanaka +Maintainer: Hideyuki Tanaka +Homepage: http://github.com/tanakh/hsmsgpack +Stability: Experimental +Tested-with: GHC == 6.12.3 +Cabal-Version: >= 1.2 +Build-Type: Simple + +Library + Build-depends: base >=4 && <5, + transformers >= 0.2.1 && < 0.2.2, + MonadCatchIO-transformers >= 0.2.2 && < 0.2.3, + bytestring >= 0.9 && < 0.10, + vector >= 0.6.0 && < 0.6.1, + iteratee >= 0.4 && < 0.5, + attoparsec >= 0.8.1 && < 0.8.2, + binary >= 0.5.0 && < 0.5.1, + data-binary-ieee754 >= 0.4 && < 0.5 + Ghc-options: -Wall -O2 + Hs-source-dirs: src Exposed-modules: Data.MessagePack - Data.MessagePack.Base - Data.MessagePack.Class - Data.MessagePack.Feed - Data.MessagePack.Monad - Data.MessagePack.Stream - - C-Sources: - cbits/msgpack.c + Data.MessagePack.Object + Data.MessagePack.Put + Data.MessagePack.Parser diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs index 2949e603..010eaab0 100644 --- a/haskell/src/Data/MessagePack.hs +++ b/haskell/src/Data/MessagePack.hs @@ -1,7 +1,7 @@ -------------------------------------------------------------------- -- | -- Module : Data.MessagePack --- Copyright : (c) Hideyuki Tanaka, 2009 +-- Copyright : (c) Hideyuki Tanaka, 2009-2010 -- License : BSD3 -- -- Maintainer: tanaka.hideyuki@gmail.com @@ -13,51 +13,105 @@ -------------------------------------------------------------------- module Data.MessagePack( - module Data.MessagePack.Base, - module Data.MessagePack.Class, - module Data.MessagePack.Feed, - module Data.MessagePack.Monad, - module Data.MessagePack.Stream, + module Data.MessagePack.Object, + module Data.MessagePack.Put, + module Data.MessagePack.Parser, - -- * Pack and Unpack - packb, - unpackb, + -- * Simple functions of Pack and Unpack + pack, + unpack, + + -- * Pack functions + packToString, + packToHandle, + packToFile, + + -- * Unpack functions + unpackFromString, + unpackFromHandle, + unpackFromFile, - -- * Pure version of Pack and Unpack - packb', - unpackb', ) where -import Data.ByteString (ByteString) -import System.IO.Unsafe +import qualified Control.Monad.CatchIO as CIO +import Control.Monad.IO.Class +import qualified Data.Attoparsec as A +import Data.Binary.Put +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as L +import Data.Functor.Identity +import qualified Data.Iteratee as I +import qualified Data.Iteratee.IO as I +import System.IO -import Data.MessagePack.Base -import Data.MessagePack.Class -import Data.MessagePack.Feed -import Data.MessagePack.Monad -import Data.MessagePack.Stream +import Data.MessagePack.Object +import Data.MessagePack.Put +import Data.MessagePack.Parser + +bufferSize :: Int +bufferSize = 4 * 1024 + +class IsByteString s where + toBS :: s -> B.ByteString + +instance IsByteString B.ByteString where + toBS = id + +instance IsByteString L.ByteString where + toBS = B.concat . L.toChunks -- | Pack Haskell data to MessagePack string. -packb :: OBJECT a => a -> IO ByteString -packb dat = do - sb <- newSimpleBuffer - pc <- newPacker sb - pack pc dat - simpleBufferData sb +pack :: ObjectPut a => a -> L.ByteString +pack = packToString . put -- | Unpack MessagePack string to Haskell data. -unpackb :: OBJECT a => ByteString -> IO (Result a) -unpackb bs = do - withZone $ \z -> do - r <- unpackObject z bs - return $ case r of - Left err -> Left (show err) - Right (_, dat) -> fromObject dat +unpack :: (ObjectGet a, IsByteString s) => s -> a +unpack bs = + runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) (parserToIteratee get) --- | Pure version of 'packb'. -packb' :: OBJECT a => a -> ByteString -packb' dat = unsafePerformIO $ packb dat +-- TODO: tryUnpack --- | Pure version of 'unpackb'. -unpackb' :: OBJECT a => ByteString -> Result a -unpackb' bs = unsafePerformIO $ unpackb bs +-- | Pack to ByteString. +packToString :: Put -> L.ByteString +packToString = runPut + +-- | Pack to Handle +packToHandle :: Handle -> Put -> IO () +packToHandle h = L.hPutStr h . packToString + +-- | Pack to File +packToFile :: FilePath -> Put -> IO () +packToFile path = L.writeFile path . packToString + +-- | Unpack from ByteString +unpackFromString :: (Monad m, IsByteString s) => s -> A.Parser a -> m a +unpackFromString bs = + I.run . I.joinIM . I.enumPure1Chunk (toBS bs) . parserToIteratee + +-- | Unpack from Handle +unpackFromHandle :: CIO.MonadCatchIO m => Handle -> A.Parser a -> m a +unpackFromHandle h = + I.run . I.joinIM . I.enumHandle bufferSize h . parserToIteratee + +-- | Unpack from File +unpackFromFile :: CIO.MonadCatchIO m => FilePath -> A.Parser a -> m a +unpackFromFile path p = + CIO.bracket + (liftIO $ openBinaryFile path ReadMode) + (liftIO . hClose) + (flip unpackFromHandle p) + +parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a +parserToIteratee p = I.icont (itr (A.parse p)) Nothing + where + itr pcont s = case s of + I.EOF _ -> + I.throwErr (I.setEOF s) + I.Chunk bs -> + case pcont bs of + A.Fail _ _ msg -> + I.throwErr (I.iterStrExc msg) + A.Partial cont -> + I.icont (itr cont) Nothing + A.Done remain ret -> + I.idone ret (I.Chunk remain) diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc deleted file mode 100644 index b6cdc287..00000000 --- a/haskell/src/Data/MessagePack/Base.hsc +++ /dev/null @@ -1,584 +0,0 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE ForeignFunctionInterface #-} - --------------------------------------------------------------------- --- | --- Module : Data.MessagePack.Base --- Copyright : (c) Hideyuki Tanaka, 2009 --- License : BSD3 --- --- Maintainer: tanaka.hideyuki@gmail.com --- Stability : experimental --- Portability: portable --- --- Low Level Interface to MessagePack C API --- --------------------------------------------------------------------- - -module Data.MessagePack.Base( - -- * Simple Buffer - SimpleBuffer, - newSimpleBuffer, - simpleBufferData, - - -- * Serializer - Packer, - newPacker, - - packU8, - packU16, - packU32, - packU64, - packS8, - packS16, - packS32, - packS64, - - packTrue, - packFalse, - - packInt, - packDouble, - packNil, - packBool, - - packArray, - packMap, - packRAW, - packRAWBody, - packRAW', - - -- * Stream Deserializer - Unpacker, - defaultInitialBufferSize, - newUnpacker, - unpackerReserveBuffer, - unpackerBuffer, - unpackerBufferCapacity, - unpackerBufferConsumed, - unpackerFeed, - unpackerExecute, - unpackerData, - unpackerReleaseZone, - unpackerResetZone, - unpackerReset, - unpackerMessageSize, - - -- * MessagePack Object - Object(..), - packObject, - - UnpackReturn(..), - unpackObject, - - -- * Memory Zone - Zone, - newZone, - freeZone, - withZone, - ) where - -import Control.Exception -import Control.Monad -import Data.ByteString (ByteString) -import qualified Data.ByteString as BS hiding (pack, unpack) -import Data.Int -import Data.Word -import Foreign.C -import Foreign.Concurrent -import Foreign.ForeignPtr hiding (newForeignPtr) -import Foreign.Marshal.Alloc -import Foreign.Marshal.Array -import Foreign.Ptr -import Foreign.Storable - -#include - -type SimpleBuffer = ForeignPtr () - -type WriteCallback = Ptr () -> CString -> CUInt -> IO CInt - --- | Create a new Simple Buffer. It will be deleted automatically. -newSimpleBuffer :: IO SimpleBuffer -newSimpleBuffer = do - ptr <- mallocBytes (#size msgpack_sbuffer) - fptr <- newForeignPtr ptr $ do - msgpack_sbuffer_destroy ptr - free ptr - withForeignPtr fptr $ \p -> - msgpack_sbuffer_init p - return fptr - --- | Get data of Simple Buffer. -simpleBufferData :: SimpleBuffer -> IO ByteString -simpleBufferData sb = - withForeignPtr sb $ \ptr -> do - size <- (#peek msgpack_sbuffer, size) ptr - dat <- (#peek msgpack_sbuffer, data) ptr - BS.packCStringLen (dat, fromIntegral (size :: CSize)) - -foreign import ccall "msgpack_sbuffer_init_wrap" msgpack_sbuffer_init :: - Ptr () -> IO () - -foreign import ccall "msgpack_sbuffer_destroy_wrap" msgpack_sbuffer_destroy :: - Ptr () -> IO () - -foreign import ccall "msgpack_sbuffer_write_wrap" msgpack_sbuffer_write :: - WriteCallback - -type Packer = ForeignPtr () - --- | Create new Packer. It will be deleted automatically. -newPacker :: SimpleBuffer -> IO Packer -newPacker sbuf = do - cb <- wrap_callback msgpack_sbuffer_write - ptr <- withForeignPtr sbuf $ \ptr -> - msgpack_packer_new ptr cb - fptr <- newForeignPtr ptr $ do - msgpack_packer_free ptr - return fptr - -foreign import ccall "msgpack_packer_new_wrap" msgpack_packer_new :: - Ptr () -> FunPtr WriteCallback -> IO (Ptr ()) - -foreign import ccall "msgpack_packer_free_wrap" msgpack_packer_free :: - Ptr () -> IO () - -foreign import ccall "wrapper" wrap_callback :: - WriteCallback -> IO (FunPtr WriteCallback) - -packU8 :: Packer -> Word8 -> IO Int -packU8 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_uint8 ptr n - -foreign import ccall "msgpack_pack_uint8_wrap" msgpack_pack_uint8 :: - Ptr () -> Word8 -> IO CInt - -packU16 :: Packer -> Word16 -> IO Int -packU16 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_uint16 ptr n - -foreign import ccall "msgpack_pack_uint16_wrap" msgpack_pack_uint16 :: - Ptr () -> Word16 -> IO CInt - -packU32 :: Packer -> Word32 -> IO Int -packU32 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_uint32 ptr n - -foreign import ccall "msgpack_pack_uint32_wrap" msgpack_pack_uint32 :: - Ptr () -> Word32 -> IO CInt - -packU64 :: Packer -> Word64 -> IO Int -packU64 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_uint64 ptr n - -foreign import ccall "msgpack_pack_uint64_wrap" msgpack_pack_uint64 :: - Ptr () -> Word64 -> IO CInt - -packS8 :: Packer -> Int8 -> IO Int -packS8 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_int8 ptr n - -foreign import ccall "msgpack_pack_int8_wrap" msgpack_pack_int8 :: - Ptr () -> Int8 -> IO CInt - -packS16 :: Packer -> Int16 -> IO Int -packS16 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_int16 ptr n - -foreign import ccall "msgpack_pack_int16_wrap" msgpack_pack_int16 :: - Ptr () -> Int16 -> IO CInt - -packS32 :: Packer -> Int32 -> IO Int -packS32 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_int32 ptr n - -foreign import ccall "msgpack_pack_int32_wrap" msgpack_pack_int32 :: - Ptr () -> Int32 -> IO CInt - -packS64 :: Packer -> Int64 -> IO Int -packS64 pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_int64 ptr n - -foreign import ccall "msgpack_pack_int64_wrap" msgpack_pack_int64 :: - Ptr () -> Int64 -> IO CInt - --- | Pack an integral data. -packInt :: Integral a => Packer -> a -> IO Int -packInt pc n = packS64 pc $ fromIntegral n - --- | Pack a double data. -packDouble :: Packer -> Double -> IO Int -packDouble pc d = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_double ptr (realToFrac d) - -foreign import ccall "msgpack_pack_double_wrap" msgpack_pack_double :: - Ptr () -> CDouble -> IO CInt - --- | Pack a nil. -packNil :: Packer -> IO Int -packNil pc = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_nil ptr - -foreign import ccall "msgpack_pack_nil_wrap" msgpack_pack_nil :: - Ptr () -> IO CInt - -packTrue :: Packer -> IO Int -packTrue pc = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_true ptr - -foreign import ccall "msgpack_pack_true_wrap" msgpack_pack_true :: - Ptr () -> IO CInt - -packFalse :: Packer -> IO Int -packFalse pc = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_false ptr - -foreign import ccall "msgpack_pack_false_wrap" msgpack_pack_false :: - Ptr () -> IO CInt - --- | Pack a bool data. -packBool :: Packer -> Bool -> IO Int -packBool pc True = packTrue pc -packBool pc False = packFalse pc - --- | 'packArray' @p n@ starts packing an array. --- Next @n@ data will consist this array. -packArray :: Packer -> Int -> IO Int -packArray pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_array ptr (fromIntegral n) - -foreign import ccall "msgpack_pack_array_wrap" msgpack_pack_array :: - Ptr () -> CUInt -> IO CInt - --- | 'packMap' @p n@ starts packing a map. --- Next @n@ pairs of data (2*n data) will consist this map. -packMap :: Packer -> Int -> IO Int -packMap pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_map ptr (fromIntegral n) - -foreign import ccall "msgpack_pack_map_wrap" msgpack_pack_map :: - Ptr () -> CUInt -> IO CInt - --- | 'packRAW' @p n@ starts packing a byte sequence. --- Next total @n@ bytes of 'packRAWBody' call will consist this sequence. -packRAW :: Packer -> Int -> IO Int -packRAW pc n = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - msgpack_pack_raw ptr (fromIntegral n) - -foreign import ccall "msgpack_pack_raw_wrap" msgpack_pack_raw :: - Ptr () -> CSize -> IO CInt - --- | Pack a byte sequence. -packRAWBody :: Packer -> ByteString -> IO Int -packRAWBody pc bs = - liftM fromIntegral $ withForeignPtr pc $ \ptr -> - BS.useAsCStringLen bs $ \(str, len) -> - msgpack_pack_raw_body ptr (castPtr str) (fromIntegral len) - -foreign import ccall "msgpack_pack_raw_body_wrap" msgpack_pack_raw_body :: - Ptr () -> Ptr () -> CSize -> IO CInt - --- | Pack a single byte stream. It calls 'packRAW' and 'packRAWBody'. -packRAW' :: Packer -> ByteString -> IO Int -packRAW' pc bs = do - _ <- packRAW pc (BS.length bs) - packRAWBody pc bs - -type Unpacker = ForeignPtr () - -defaultInitialBufferSize :: Int -defaultInitialBufferSize = 32 * 1024 -- #const MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE - --- | 'newUnpacker' @initialBufferSize@ creates a new Unpacker. It will be deleted automatically. -newUnpacker :: Int -> IO Unpacker -newUnpacker initialBufferSize = do - ptr <- msgpack_unpacker_new (fromIntegral initialBufferSize) - fptr <- newForeignPtr ptr $ do - msgpack_unpacker_free ptr - return fptr - -foreign import ccall "msgpack_unpacker_new" msgpack_unpacker_new :: - CSize -> IO (Ptr ()) - -foreign import ccall "msgpack_unpacker_free" msgpack_unpacker_free :: - Ptr() -> IO () - --- | 'unpackerReserveBuffer' @up size@ reserves at least @size@ bytes of buffer. -unpackerReserveBuffer :: Unpacker -> Int -> IO Bool -unpackerReserveBuffer up size = - withForeignPtr up $ \ptr -> - liftM (/=0) $ msgpack_unpacker_reserve_buffer ptr (fromIntegral size) - -foreign import ccall "msgpack_unpacker_reserve_buffer_wrap" msgpack_unpacker_reserve_buffer :: - Ptr () -> CSize -> IO CChar - --- | Get a pointer of unpacker buffer. -unpackerBuffer :: Unpacker -> IO (Ptr CChar) -unpackerBuffer up = - withForeignPtr up $ \ptr -> - msgpack_unpacker_buffer ptr - -foreign import ccall "msgpack_unpacker_buffer_wrap" msgpack_unpacker_buffer :: - Ptr () -> IO (Ptr CChar) - --- | Get size of allocated buffer. -unpackerBufferCapacity :: Unpacker -> IO Int -unpackerBufferCapacity up = - withForeignPtr up $ \ptr -> - liftM fromIntegral $ msgpack_unpacker_buffer_capacity ptr - -foreign import ccall "msgpack_unpacker_buffer_capacity_wrap" msgpack_unpacker_buffer_capacity :: - Ptr () -> IO CSize - --- | 'unpackerBufferConsumed' @up size@ notices that writed @size@ bytes to buffer. -unpackerBufferConsumed :: Unpacker -> Int -> IO () -unpackerBufferConsumed up size = - withForeignPtr up $ \ptr -> - msgpack_unpacker_buffer_consumed ptr (fromIntegral size) - -foreign import ccall "msgpack_unpacker_buffer_consumed_wrap" msgpack_unpacker_buffer_consumed :: - Ptr () -> CSize -> IO () - --- | Write byte sequence to Unpacker. It is utility funciton, calls 'unpackerReserveBuffer', 'unpackerBuffer' and 'unpackerBufferConsumed'. -unpackerFeed :: Unpacker -> ByteString -> IO () -unpackerFeed up bs = - BS.useAsCStringLen bs $ \(str, len) -> do - True <- unpackerReserveBuffer up len - ptr <- unpackerBuffer up - copyArray ptr str len - unpackerBufferConsumed up len - --- | Execute deserializing. It returns 0 when buffer contains not enough bytes, returns 1 when succeeded, returns negative value when it failed. -unpackerExecute :: Unpacker -> IO Int -unpackerExecute up = - withForeignPtr up $ \ptr -> - liftM fromIntegral $ msgpack_unpacker_execute ptr - -foreign import ccall "msgpack_unpacker_execute" msgpack_unpacker_execute :: - Ptr () -> IO CInt - --- | Returns a deserialized object when 'unpackerExecute' returned 1. -unpackerData :: Unpacker -> IO Object -unpackerData up = - withForeignPtr up $ \ptr -> - allocaBytes (#size msgpack_object) $ \pobj -> do - msgpack_unpacker_data ptr pobj - peekObject pobj - -foreign import ccall "msgpack_unpacker_data_wrap" msgpack_unpacker_data :: - Ptr () -> Ptr () -> IO () - --- | Release memory zone. The returned zone must be freed by calling 'freeZone'. -unpackerReleaseZone :: Unpacker -> IO Zone -unpackerReleaseZone up = - withForeignPtr up $ \ptr -> - msgpack_unpacker_release_zone ptr - -foreign import ccall "msgpack_unpacker_release_zone" msgpack_unpacker_release_zone :: - Ptr () -> IO (Ptr ()) - --- | Free memory zone used by Unapcker. -unpackerResetZone :: Unpacker -> IO () -unpackerResetZone up = - withForeignPtr up $ \ptr -> - msgpack_unpacker_reset_zone ptr - -foreign import ccall "msgpack_unpacker_reset_zone" msgpack_unpacker_reset_zone :: - Ptr () -> IO () - --- | Reset Unpacker state except memory zone. -unpackerReset :: Unpacker -> IO () -unpackerReset up = - withForeignPtr up $ \ptr -> - msgpack_unpacker_reset ptr - -foreign import ccall "msgpack_unpacker_reset" msgpack_unpacker_reset :: - Ptr () -> IO () - --- | Returns number of bytes of sequence of deserializing object. -unpackerMessageSize :: Unpacker -> IO Int -unpackerMessageSize up = - withForeignPtr up $ \ptr -> - liftM fromIntegral $ msgpack_unpacker_message_size ptr - -foreign import ccall "msgpack_unpacker_message_size_wrap" msgpack_unpacker_message_size :: - Ptr () -> IO CSize - -type Zone = Ptr () - --- | Create a new memory zone. It must be freed manually. -newZone :: IO Zone -newZone = - msgpack_zone_new (#const MSGPACK_ZONE_CHUNK_SIZE) - --- | Free a memory zone. -freeZone :: Zone -> IO () -freeZone z = - msgpack_zone_free z - --- | Create a memory zone, then execute argument, then free memory zone. -withZone :: (Zone -> IO a) -> IO a -withZone z = - bracket newZone freeZone z - -foreign import ccall "msgpack_zone_new" msgpack_zone_new :: - CSize -> IO Zone - -foreign import ccall "msgpack_zone_free" msgpack_zone_free :: - Zone -> IO () - --- | Object Representation of MessagePack data. -data Object = - ObjectNil - | ObjectBool Bool - | ObjectInteger Int - | ObjectDouble Double - | ObjectRAW ByteString - | ObjectArray [Object] - | ObjectMap [(Object, Object)] - deriving (Show) - -peekObject :: Ptr a -> IO Object -peekObject ptr = do - typ <- (#peek msgpack_object, type) ptr - case (typ :: CInt) of - (#const MSGPACK_OBJECT_NIL) -> - return ObjectNil - (#const MSGPACK_OBJECT_BOOLEAN) -> - peekObjectBool ptr - (#const MSGPACK_OBJECT_POSITIVE_INTEGER) -> - peekObjectPositiveInteger ptr - (#const MSGPACK_OBJECT_NEGATIVE_INTEGER) -> - peekObjectNegativeInteger ptr - (#const MSGPACK_OBJECT_DOUBLE) -> - peekObjectDouble ptr - (#const MSGPACK_OBJECT_RAW) -> - peekObjectRAW ptr - (#const MSGPACK_OBJECT_ARRAY) -> - peekObjectArray ptr - (#const MSGPACK_OBJECT_MAP) -> - peekObjectMap ptr - _ -> - fail $ "peekObject: unknown object type (" ++ show typ ++ ")" - -peekObjectBool :: Ptr a -> IO Object -peekObjectBool ptr = do - b <- (#peek msgpack_object, via.boolean) ptr - return $ ObjectBool $ (b :: CUChar) /= 0 - -peekObjectPositiveInteger :: Ptr a -> IO Object -peekObjectPositiveInteger ptr = do - n <- (#peek msgpack_object, via.u64) ptr - return $ ObjectInteger $ fromIntegral (n :: Word64) - -peekObjectNegativeInteger :: Ptr a -> IO Object -peekObjectNegativeInteger ptr = do - n <- (#peek msgpack_object, via.i64) ptr - return $ ObjectInteger $ fromIntegral (n :: Int64) - -peekObjectDouble :: Ptr a -> IO Object -peekObjectDouble ptr = do - d <- (#peek msgpack_object, via.dec) ptr - return $ ObjectDouble $ realToFrac (d :: CDouble) - -peekObjectRAW :: Ptr a -> IO Object -peekObjectRAW ptr = do - size <- (#peek msgpack_object, via.raw.size) ptr - p <- (#peek msgpack_object, via.raw.ptr) ptr - bs <- BS.packCStringLen (p, fromIntegral (size :: Word32)) - return $ ObjectRAW bs - -peekObjectArray :: Ptr a -> IO Object -peekObjectArray ptr = do - csize <- (#peek msgpack_object, via.array.size) ptr - let size = fromIntegral (csize :: Word32) - p <- (#peek msgpack_object, via.array.ptr) ptr - objs <- mapM (\i -> peekObject $ p `plusPtr` - ((#size msgpack_object) * i)) - [0..size-1] - return $ ObjectArray objs - -peekObjectMap :: Ptr a -> IO Object -peekObjectMap ptr = do - csize <- (#peek msgpack_object, via.map.size) ptr - let size = fromIntegral (csize :: Word32) - p <- (#peek msgpack_object, via.map.ptr) ptr - dat <- mapM (\i -> peekObjectKV $ p `plusPtr` - ((#size msgpack_object_kv) * i)) - [0..size-1] - return $ ObjectMap dat - -peekObjectKV :: Ptr a -> IO (Object, Object) -peekObjectKV ptr = do - k <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, key) - v <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, val) - return (k, v) - --- | Pack a Object. -packObject :: Packer -> Object -> IO () -packObject pc ObjectNil = packNil pc >> return () - -packObject pc (ObjectBool b) = packBool pc b >> return () - -packObject pc (ObjectInteger n) = packInt pc n >> return () - -packObject pc (ObjectDouble d) = packDouble pc d >> return () - -packObject pc (ObjectRAW bs) = packRAW' pc bs >> return () - -packObject pc (ObjectArray ls) = do - _ <- packArray pc (length ls) - mapM_ (packObject pc) ls - -packObject pc (ObjectMap ls) = do - _ <- packMap pc (length ls) - mapM_ (\(a, b) -> packObject pc a >> packObject pc b) ls - -data UnpackReturn = - UnpackContinue -- ^ not enough bytes to unpack object - | UnpackParseError -- ^ got invalid bytes - | UnpackError -- ^ other error - deriving (Eq, Show) - --- | Unpack a single MessagePack object from byte sequence. -unpackObject :: Zone -> ByteString -> IO (Either UnpackReturn (Int, Object)) -unpackObject z dat = - allocaBytes (#size msgpack_object) $ \ptr -> - BS.useAsCStringLen dat $ \(str, len) -> - alloca $ \poff -> do - poke poff 0 - ret <- msgpack_unpack str (fromIntegral len) poff z ptr - case ret of - (#const MSGPACK_UNPACK_SUCCESS) -> do - off <- peek poff - obj <- peekObject ptr - return $ Right (fromIntegral off, obj) - (#const MSGPACK_UNPACK_EXTRA_BYTES) -> do - off <- peek poff - obj <- peekObject ptr - return $ Right (fromIntegral off, obj) - (#const MSGPACK_UNPACK_CONTINUE) -> - return $ Left UnpackContinue - (#const MSGPACK_UNPACK_PARSE_ERROR) -> - return $ Left UnpackParseError - _ -> - return $ Left UnpackError - -foreign import ccall "msgpack_unpack" msgpack_unpack :: - Ptr CChar -> CSize -> Ptr CSize -> Zone -> Ptr () -> IO CInt diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs deleted file mode 100644 index 4b486396..00000000 --- a/haskell/src/Data/MessagePack/Feed.hs +++ /dev/null @@ -1,62 +0,0 @@ --------------------------------------------------------------------- --- | --- Module : Data.MessagePack.Feed --- Copyright : (c) Hideyuki Tanaka, 2009 --- License : BSD3 --- --- Maintainer: tanaka.hideyuki@gmail.com --- Stability : experimental --- Portability: portable --- --- Feeders for Stream Deserializers --- --------------------------------------------------------------------- - -module Data.MessagePack.Feed( - -- * Feeder type - Feeder, - -- * Feeders - feederFromHandle, - feederFromFile, - feederFromString, - ) where - -import Data.ByteString (ByteString) -import qualified Data.ByteString as BS -import Data.IORef -import System.IO - --- | Feeder returns Just ByteString when bytes remains, otherwise Nothing. -type Feeder = IO (Maybe ByteString) - --- | Feeder from Handle -feederFromHandle :: Handle -> IO Feeder -feederFromHandle h = return $ do - bs <- BS.hGetNonBlocking h bufSize - if BS.length bs > 0 - then do return $ Just bs - else do - c <- BS.hGet h 1 - if BS.length c > 0 - then do return $ Just c - else do - hClose h - return Nothing - where - bufSize = 4096 - --- | Feeder from File -feederFromFile :: FilePath -> IO Feeder -feederFromFile path = - openFile path ReadMode >>= feederFromHandle - --- | Feeder from ByteString -feederFromString :: ByteString -> IO Feeder -feederFromString bs = do - r <- newIORef (Just bs) - return $ f r - where - f r = do - mb <- readIORef r - writeIORef r Nothing - return mb diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs deleted file mode 100644 index 15f21fe0..00000000 --- a/haskell/src/Data/MessagePack/Monad.hs +++ /dev/null @@ -1,156 +0,0 @@ --------------------------------------------------------------------- --- | --- Module : Data.MessagePack.Monad --- Copyright : (c) Hideyuki Tanaka, 2009 --- License : BSD3 --- --- Maintainer: tanaka.hideyuki@gmail.com --- Stability : experimental --- Portability: portable --- --- Monadic Stream Serializers and Deserializers --- --------------------------------------------------------------------- - -module Data.MessagePack.Monad( - -- * Classes - MonadPacker(..), - MonadUnpacker(..), - - -- * Packer and Unpacker type - PackerT(..), - UnpackerT(..), - - -- * Packers - packToString, - packToHandle, - packToFile, - - -- * Unpackers - unpackFrom, - unpackFromString, - unpackFromHandle, - unpackFromFile, - ) where - -import Control.Monad -import Control.Monad.Trans -import Data.ByteString (ByteString) -import qualified Data.ByteString as BS -import System.IO - -import Data.MessagePack.Base hiding (Unpacker) -import qualified Data.MessagePack.Base as Base -import Data.MessagePack.Class -import Data.MessagePack.Feed - -class Monad m => MonadPacker m where - -- | Serialize a object - put :: OBJECT a => a -> m () - -class Monad m => MonadUnpacker m where - -- | Deserialize a object - get :: OBJECT a => m a - --- | Serializer Type -newtype PackerT m r = PackerT { runPackerT :: Base.Packer -> m r } - -instance Monad m => Monad (PackerT m) where - a >>= b = - PackerT $ \pc -> do - r <- runPackerT a pc - runPackerT (b r) pc - - return r = - PackerT $ \_ -> return r - -instance MonadTrans PackerT where - lift m = PackerT $ \_ -> m - -instance MonadIO m => MonadIO (PackerT m) where - liftIO = lift . liftIO - -instance MonadIO m => MonadPacker (PackerT m) where - put v = PackerT $ \pc -> liftIO $ do - pack pc v - --- | Execute given serializer and returns byte sequence. -packToString :: MonadIO m => PackerT m r -> m ByteString -packToString m = do - sb <- liftIO $ newSimpleBuffer - pc <- liftIO $ newPacker sb - _ <- runPackerT m pc - liftIO $ simpleBufferData sb - --- | Execute given serializer and write byte sequence to Handle. -packToHandle :: MonadIO m => Handle -> PackerT m r -> m () -packToHandle h m = do - sb <- packToString m - liftIO $ BS.hPut h sb - liftIO $ hFlush h - --- | Execute given serializer and write byte sequence to file. -packToFile :: MonadIO m => FilePath -> PackerT m r -> m () -packToFile p m = do - sb <- packToString m - liftIO $ BS.writeFile p sb - --- | Deserializer type -newtype UnpackerT m r = UnpackerT { runUnpackerT :: Base.Unpacker -> Feeder -> m r } - -instance Monad m => Monad (UnpackerT m) where - a >>= b = - UnpackerT $ \up feed -> do - r <- runUnpackerT a up feed - runUnpackerT (b r) up feed - - return r = - UnpackerT $ \_ _ -> return r - -instance MonadTrans UnpackerT where - lift m = UnpackerT $ \_ _ -> m - -instance MonadIO m => MonadIO (UnpackerT m) where - liftIO = lift . liftIO - -instance MonadIO m => MonadUnpacker (UnpackerT m) where - get = UnpackerT $ \up feed -> liftIO $ do - executeOne up feed - obj <- unpackerData up - freeZone =<< unpackerReleaseZone up - unpackerReset up - let Right r = fromObject obj - return r - - where - executeOne up feed = do - resp <- unpackerExecute up - guard $ resp>=0 - when (resp==0) $ do - Just bs <- feed - unpackerFeed up bs - executeOne up feed - --- | Execute deserializer using given feeder. -unpackFrom :: MonadIO m => Feeder -> UnpackerT m r -> m r -unpackFrom f m = do - up <- liftIO $ newUnpacker defaultInitialBufferSize - runUnpackerT m up f - --- | Execute deserializer using given handle. -unpackFromHandle :: MonadIO m => Handle -> UnpackerT m r -> m r -unpackFromHandle h m = - flip unpackFrom m =<< liftIO (feederFromHandle h) - --- | Execute deserializer using given file content. -unpackFromFile :: MonadIO m => FilePath -> UnpackerT m r -> m r -unpackFromFile p m = do - h <- liftIO $ openFile p ReadMode - r <- flip unpackFrom m =<< liftIO (feederFromHandle h) - liftIO $ hClose h - return r - --- | Execute deserializer from given byte sequence. -unpackFromString :: MonadIO m => ByteString -> UnpackerT m r -> m r -unpackFromString bs m = do - flip unpackFrom m =<< liftIO (feederFromString bs) diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Object.hs similarity index 77% rename from haskell/src/Data/MessagePack/Class.hs rename to haskell/src/Data/MessagePack/Object.hs index 365acc5f..19a3aeba 100644 --- a/haskell/src/Data/MessagePack/Class.hs +++ b/haskell/src/Data/MessagePack/Object.hs @@ -1,38 +1,50 @@ {-# LANGUAGE TypeSynonymInstances #-} {-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE OverlappingInstances #-} -{-# LANGUAGE IncoherentInstances #-} -------------------------------------------------------------------- -- | --- Module : Data.MessagePack.Class --- Copyright : (c) Hideyuki Tanaka, 2009 +-- Module : Data.MessagePack.Object +-- Copyright : (c) Hideyuki Tanaka, 2009-2010 -- License : BSD3 -- -- Maintainer: tanaka.hideyuki@gmail.com -- Stability : experimental -- Portability: portable -- --- Serializing Haskell values to and from MessagePack Objects. +-- MessagePack object definition -- -------------------------------------------------------------------- -module Data.MessagePack.Class( +module Data.MessagePack.Object( + -- * MessagePack Object + Object(..), + -- * Serialization to and from Object OBJECT(..), Result, - pack, ) where -import Control.Monad.Error -import Data.ByteString.Char8 (ByteString) +import Control.Monad +import Control.Monad.Trans.Error () +import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as C8 -import Data.MessagePack.Base +-- | Object Representation of MessagePack data. +data Object = + ObjectNil + | ObjectBool Bool + | ObjectInteger Int + | ObjectDouble Double + | ObjectRAW B.ByteString + | ObjectArray [Object] + | ObjectMap [(Object, Object)] + deriving (Show) -- | The class of types serializable to and from MessagePack object class OBJECT a where + -- | Encode a value to MessagePack object toObject :: a -> Object + -- | Decode a value from MessagePack object fromObject :: Object -> Result a -- | A type for parser results @@ -65,7 +77,7 @@ instance OBJECT Double where fromObject (ObjectDouble d) = Right d fromObject _ = Left fromObjectError -instance OBJECT ByteString where +instance OBJECT B.ByteString where toObject = ObjectRAW fromObject (ObjectRAW bs) = Right bs fromObject _ = Left fromObjectError @@ -95,7 +107,3 @@ instance OBJECT a => OBJECT (Maybe a) where fromObject ObjectNil = return Nothing fromObject obj = liftM Just $ fromObject obj - --- | Pack a serializable Haskell value. -pack :: OBJECT a => Packer -> a -> IO () -pack pc = packObject pc . toObject diff --git a/haskell/src/Data/MessagePack/Packer.hs b/haskell/src/Data/MessagePack/Packer.hs new file mode 100644 index 00000000..9c10f5ed --- /dev/null +++ b/haskell/src/Data/MessagePack/Packer.hs @@ -0,0 +1,147 @@ +{-# Language FlexibleInstances #-} +{-# Language OverlappingInstances #-} + +module Data.MessagePack.Packer( + ObjectPut(..), + ) where + +import Data.Binary.Put +import Data.Binary.IEEE754 +import Data.Bits +import qualified Data.ByteString as B + +import Data.MessagePack.Object + +class ObjectPut a where + put :: a -> Put + +instance ObjectPut Object where + put = putObject + +instance ObjectPut Int where + put = putInteger + +instance ObjectPut () where + put _ = putNil + +instance ObjectPut Bool where + put = putBool + +instance ObjectPut Double where + put = putDouble + +instance ObjectPut B.ByteString where + put = putRAW + +instance ObjectPut a => ObjectPut [a] where + put = putArray + +instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where + put = putMap + +putObject :: Object -> Put +putObject obj = + case obj of + ObjectInteger n -> + putInteger n + ObjectNil -> + putNil + ObjectBool b -> + putBool b + ObjectDouble d -> + putDouble d + ObjectRAW raw -> + putRAW raw + ObjectArray arr -> + putArray arr + ObjectMap m -> + putMap m + +putInteger :: Int -> Put +putInteger n = + case n of + _ | n >= 0 && n <= 127 -> + putWord8 $ fromIntegral n + _ | n >= -32 && n <= -1 -> + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x100 -> do + putWord8 0xCC + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x10000 -> do + putWord8 0xCD + putWord16be $ fromIntegral n + _ | n >= 0 && n < 0x100000000 -> do + putWord8 0xCE + putWord32be $ fromIntegral n + _ | n >= 0 -> do + putWord8 0xCF + putWord64be $ fromIntegral n + _ | n >= -0x100 -> do + putWord8 0xD0 + putWord8 $ fromIntegral n + _ | n >= -0x10000 -> do + putWord8 0xD1 + putWord16be $ fromIntegral n + _ | n >= -0x100000000 -> do + putWord8 0xD2 + putWord32be $ fromIntegral n + _ -> do + putWord8 0xD3 + putWord64be $ fromIntegral n + +putNil :: Put +putNil = putWord8 0xC0 + +putBool :: Bool -> Put +putBool True = putWord8 0xC3 +putBool False = putWord8 0xC2 + +putDouble :: Double -> Put +putDouble d = do + putWord8 0xCB + putFloat64be d + +putRAW :: B.ByteString -> Put +putRAW bs = do + case len of + _ | len <= 31 -> do + putWord8 $ 0xA0 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDA + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDB + putWord32be $ fromIntegral len + putByteString bs + where + len = B.length bs + +putArray :: ObjectPut a => [a] -> Put +putArray arr = do + case len of + _ | len <= 15 -> + putWord8 $ 0x90 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDC + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDD + putWord32be $ fromIntegral len + mapM_ put arr + where + len = length arr + +putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put +putMap m = do + case len of + _ | len <= 15 -> + putWord8 $ 0x80 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDE + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDF + putWord16be $ fromIntegral len + mapM_ (\(k, v) -> put k >> put v) m + where + len = length m diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs new file mode 100644 index 00000000..8d0af2b2 --- /dev/null +++ b/haskell/src/Data/MessagePack/Put.hs @@ -0,0 +1,202 @@ +{-# Language FlexibleInstances #-} +{-# Language IncoherentInstances #-} +{-# Language OverlappingInstances #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Put +-- Copyright : (c) Hideyuki Tanaka, 2009-2010 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- MessagePack Serializer using @Data.Binary.Put@ +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Put( + -- * Serializable class + ObjectPut(..), + ) where + +import Data.Binary.Put +import Data.Binary.IEEE754 +import Data.Bits +import qualified Data.ByteString as B +import qualified Data.Vector as V + +import Data.MessagePack.Object + +-- | Serializable class +class ObjectPut a where + -- | Serialize a value + put :: a -> Put + +instance ObjectPut Object where + put = putObject + +instance ObjectPut Int where + put = putInteger + +instance ObjectPut () where + put _ = putNil + +instance ObjectPut Bool where + put = putBool + +instance ObjectPut Double where + put = putDouble + +instance ObjectPut B.ByteString where + put = putRAW + +instance ObjectPut a => ObjectPut [a] where + put = putArray + +instance ObjectPut a => ObjectPut (V.Vector a) where + put = putArrayVector + +instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where + put = putMap + +instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where + put = putMapVector + +putObject :: Object -> Put +putObject obj = + case obj of + ObjectInteger n -> + putInteger n + ObjectNil -> + putNil + ObjectBool b -> + putBool b + ObjectDouble d -> + putDouble d + ObjectRAW raw -> + putRAW raw + ObjectArray arr -> + putArray arr + ObjectMap m -> + putMap m + +putInteger :: Int -> Put +putInteger n = + case n of + _ | n >= 0 && n <= 127 -> + putWord8 $ fromIntegral n + _ | n >= -32 && n <= -1 -> + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x100 -> do + putWord8 0xCC + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x10000 -> do + putWord8 0xCD + putWord16be $ fromIntegral n + _ | n >= 0 && n < 0x100000000 -> do + putWord8 0xCE + putWord32be $ fromIntegral n + _ | n >= 0 -> do + putWord8 0xCF + putWord64be $ fromIntegral n + _ | n >= -0x80 -> do + putWord8 0xD0 + putWord8 $ fromIntegral n + _ | n >= -0x8000 -> do + putWord8 0xD1 + putWord16be $ fromIntegral n + _ | n >= -0x80000000 -> do + putWord8 0xD2 + putWord32be $ fromIntegral n + _ -> do + putWord8 0xD3 + putWord64be $ fromIntegral n + +putNil :: Put +putNil = putWord8 0xC0 + +putBool :: Bool -> Put +putBool True = putWord8 0xC3 +putBool False = putWord8 0xC2 + +putDouble :: Double -> Put +putDouble d = do + putWord8 0xCB + putFloat64be d + +putRAW :: B.ByteString -> Put +putRAW bs = do + case len of + _ | len <= 31 -> do + putWord8 $ 0xA0 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDA + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDB + putWord32be $ fromIntegral len + putByteString bs + where + len = B.length bs + +putArray :: ObjectPut a => [a] -> Put +putArray arr = do + case len of + _ | len <= 15 -> + putWord8 $ 0x90 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDC + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDD + putWord32be $ fromIntegral len + mapM_ put arr + where + len = length arr + +putArrayVector :: ObjectPut a => V.Vector a -> Put +putArrayVector arr = do + case len of + _ | len <= 15 -> + putWord8 $ 0x90 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDC + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDD + putWord32be $ fromIntegral len + V.mapM_ put arr + where + len = V.length arr + +putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put +putMap m = do + case len of + _ | len <= 15 -> + putWord8 $ 0x80 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDE + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDF + putWord32be $ fromIntegral len + mapM_ (\(k, v) -> put k >> put v) m + where + len = length m + +putMapVector :: (ObjectPut k, ObjectPut v) => V.Vector (k, v) -> Put +putMapVector m = do + case len of + _ | len <= 15 -> + putWord8 $ 0x80 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDE + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDF + putWord32be $ fromIntegral len + V.mapM_ (\(k, v) -> put k >> put v) m + where + len = V.length m diff --git a/haskell/src/Data/MessagePack/Stream.hs b/haskell/src/Data/MessagePack/Stream.hs deleted file mode 100644 index c56fe8d4..00000000 --- a/haskell/src/Data/MessagePack/Stream.hs +++ /dev/null @@ -1,82 +0,0 @@ --------------------------------------------------------------------- --- | --- Module : Data.MessagePack.Stream --- Copyright : (c) Hideyuki Tanaka, 2009 --- License : BSD3 --- --- Maintainer: tanaka.hideyuki@gmail.com --- Stability : experimental --- Portability: portable --- --- Lazy Stream Serializers and Deserializers --- --------------------------------------------------------------------- - -module Data.MessagePack.Stream( - unpackObjects, - unpackObjectsFromFile, - unpackObjectsFromHandle, - unpackObjectsFromString, - ) where - -import Data.ByteString (ByteString) -import System.IO -import System.IO.Unsafe - -import Data.MessagePack.Base -import Data.MessagePack.Feed - --- | Unpack objects using given feeder. -unpackObjects :: Feeder -> IO [Object] -unpackObjects feeder = do - up <- newUnpacker defaultInitialBufferSize - f up - where - f up = unsafeInterleaveIO $ do - mbo <- unpackOnce up - case mbo of - Just o -> do - os <- f up - return $ o:os - Nothing -> - return [] - - unpackOnce up = do - resp <- unpackerExecute up - case resp of - 0 -> do - r <- feedOnce up - if r - then unpackOnce up - else return Nothing - 1 -> do - obj <- unpackerData up - freeZone =<< unpackerReleaseZone up - unpackerReset up - return $ Just obj - _ -> - error $ "unpackerExecute fails: " ++ show resp - - feedOnce up = do - dat <- feeder - case dat of - Nothing -> - return False - Just bs -> do - unpackerFeed up bs - return True - --- | Unpack objects from file. -unpackObjectsFromFile :: FilePath -> IO [Object] -unpackObjectsFromFile fname = - unpackObjects =<< feederFromFile fname - --- | Unpack objects from handle. -unpackObjectsFromHandle :: Handle -> IO [Object] -unpackObjectsFromHandle h = - unpackObjects =<< feederFromHandle h - --- | Unpack oobjects from given byte sequence. -unpackObjectsFromString :: ByteString -> IO [Object] -unpackObjectsFromString bs = - unpackObjects =<< feederFromString bs diff --git a/haskell/test/Monad.hs b/haskell/test/Monad.hs index 4bee5c54..2ec40938 100644 --- a/haskell/test/Monad.hs +++ b/haskell/test/Monad.hs @@ -1,16 +1,21 @@ -import Control.Monad.Trans +{-# Language OverloadedStrings #-} + +import Control.Monad.IO.Class +import qualified Data.ByteString as B import Data.MessagePack main = do - sb <- packToString $ do + sb <- return $ packToString $ do put [1,2,3::Int] put (3.14 :: Double) - put "Hoge" + put ("Hoge" :: B.ByteString) print sb - unpackFromString sb $ do + r <- unpackFromString sb $ do arr <- get dbl <- get str <- get - liftIO $ print (arr :: [Int], dbl :: Double, str :: String) + return (arr :: [Int], dbl :: Double, str :: B.ByteString) + + print r diff --git a/haskell/test/Stream.hs b/haskell/test/Stream.hs deleted file mode 100644 index ce060dea..00000000 --- a/haskell/test/Stream.hs +++ /dev/null @@ -1,14 +0,0 @@ -import Control.Applicative -import qualified Data.ByteString as BS -import Data.MessagePack - -main = do - sb <- newSimpleBuffer - pc <- newPacker sb - pack pc [1,2,3::Int] - pack pc True - pack pc "hoge" - bs <- simpleBufferData sb - - os <- unpackObjectsFromString bs - mapM_ print os diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs index 4e713ba6..1bb551c1 100644 --- a/haskell/test/Test.hs +++ b/haskell/test/Test.hs @@ -1,36 +1,45 @@ +import Test.Framework +import Test.Framework.Providers.QuickCheck2 +import Test.QuickCheck + import Control.Monad +import qualified Data.ByteString.Char8 as B import Data.MessagePack -{- -main = do - sb <- newSimpleBuffer - pc <- newPacker sb - - pack pc [(1,2),(2,3),(3::Int,4::Int)] - pack pc [4,5,6::Int] - pack pc "hoge" - - bs <- simpleBufferData sb - print bs - - up <- newUnpacker defaultInitialBufferSize - - unpackerFeed up bs +mid :: (ObjectGet a, ObjectPut a) => a -> a +mid = unpack . pack - let f = do - res <- unpackerExecute up - when (res==1) $ do - obj <- unpackerData up - print obj - f - - f +prop_mid_int a = a == mid a + where types = a :: Int +prop_mid_nil a = a == mid a + where types = a :: () +prop_mid_bool a = a == mid a + where types = a :: Bool +prop_mid_double a = a == mid a + where types = a :: Double +prop_mid_string a = a == B.unpack (mid (B.pack a)) + where types = a :: String +prop_mid_array_int a = a == mid a + where types = a :: [Int] +prop_mid_array_string a = a == map B.unpack (mid (map B.pack a)) + where types = a :: [String] +prop_mid_map_int_double a = a == mid a + where types = a :: [(Int, Double)] +prop_mid_map_string_string a = a == map (\(x, y) -> (B.unpack x, B.unpack y)) (mid (map (\(x, y) -> (B.pack x, B.pack y)) a)) + where types = a :: [(String, String)] - return () --} +tests = + [ testGroup "simple" + [ testProperty "int" prop_mid_int + , testProperty "nil" prop_mid_nil + , testProperty "bool" prop_mid_bool + , testProperty "double" prop_mid_double + , testProperty "string" prop_mid_string + , testProperty "[int]" prop_mid_array_int + , testProperty "[string]" prop_mid_array_string + , testProperty "[(int, double)]" prop_mid_map_int_double + , testProperty "[(string, string)]" prop_mid_map_string_string + ] + ] -main = do - bs <- packb [(1,2),(2,3),(3::Int,4::Int)] - print bs - dat <- unpackb bs - print (dat :: Result [(Int, Int)]) +main = defaultMain tests From 0368a70dd70a91598507bc7baad8291adc1309fa Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 13:55:34 +0900 Subject: [PATCH 124/259] forgot to add file --- haskell/src/Data/MessagePack/Parser.hs | 259 +++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 haskell/src/Data/MessagePack/Parser.hs diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs new file mode 100644 index 00000000..d0cd0846 --- /dev/null +++ b/haskell/src/Data/MessagePack/Parser.hs @@ -0,0 +1,259 @@ +{-# Language FlexibleInstances #-} +{-# Language IncoherentInstances #-} +{-# Language OverlappingInstances #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Parser +-- Copyright : (c) Hideyuki Tanaka, 2009-2010 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- MessagePack Deserializer using @Data.Attoparsec@ +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Parser( + -- * MessagePack deserializer + ObjectGet(..), + ) where + +import Control.Monad +import qualified Data.Attoparsec as A +import Data.Binary.Get +import Data.Binary.IEEE754 +import Data.Bits +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as L +import Data.Int +import qualified Data.Vector as V +import Data.Word +import Text.Printf + +import Data.MessagePack.Object + +-- | Deserializable class +class ObjectGet a where + -- | Deserialize a value + get :: A.Parser a + +instance ObjectGet Int where + get = parseInt + +instance ObjectGet () where + get = parseNil + +instance ObjectGet Bool where + get = parseBool + +instance ObjectGet Double where + get = parseDouble + +instance ObjectGet B.ByteString where + get = parseRAW + +instance ObjectGet a => ObjectGet [a] where + get = parseArray + +instance ObjectGet a => ObjectGet (V.Vector a) where + get = parseArrayVector + +instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where + get = parseMap + +instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where + get = parseMapVector + +instance ObjectGet Object where + get = parseObject + +parseInt :: A.Parser Int +parseInt = do + c <- A.anyWord8 + case c of + _ | c .&. 0x80 == 0x00 -> + return $ fromIntegral c + _ | c .&. 0xE0 == 0xE0 -> + return $ fromIntegral (fromIntegral c :: Int8) + 0xCC -> + return . fromIntegral =<< A.anyWord8 + 0xCD -> + return . fromIntegral =<< parseUint16 + 0xCE -> + return . fromIntegral =<< parseUint32 + 0xCF -> + return . fromIntegral =<< parseUint64 + 0xD0 -> + return . fromIntegral =<< parseInt8 + 0xD1 -> + return . fromIntegral =<< parseInt16 + 0xD2 -> + return . fromIntegral =<< parseInt32 + 0xD3 -> + return . fromIntegral =<< parseInt64 + _ -> + fail $ printf "invlid integer tag: 0x%02X" c + +parseNil :: A.Parser () +parseNil = do + _ <- A.word8 0xC0 + return () + +parseBool :: A.Parser Bool +parseBool = do + c <- A.anyWord8 + case c of + 0xC3 -> + return True + 0xC2 -> + return False + _ -> + fail $ printf "invlid bool tag: 0x%02X" c + +parseDouble :: A.Parser Double +parseDouble = do + c <- A.anyWord8 + case c of + 0xCA -> + return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4 + 0xCB -> + return . runGet getFloat64be . toLBS =<< A.take 8 + _ -> + fail $ printf "invlid double tag: 0x%02X" c + +parseRAW :: A.Parser B.ByteString +parseRAW = do + c <- A.anyWord8 + case c of + _ | c .&. 0xE0 == 0xA0 -> + A.take . fromIntegral $ c .&. 0x1F + 0xDA -> + A.take . fromIntegral =<< parseUint16 + 0xDB -> + A.take . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid raw tag: 0x%02X" c + +parseArray :: ObjectGet a => A.Parser [a] +parseArray = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x90 -> + flip replicateM get . fromIntegral $ c .&. 0x0F + 0xDC -> + flip replicateM get . fromIntegral =<< parseUint16 + 0xDD -> + flip replicateM get . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid array tag: 0x%02X" c + +parseArrayVector :: ObjectGet a => A.Parser (V.Vector a) +parseArrayVector = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x90 -> + flip V.replicateM get . fromIntegral $ c .&. 0x0F + 0xDC -> + flip V.replicateM get . fromIntegral =<< parseUint16 + 0xDD -> + flip V.replicateM get . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid array tag: 0x%02X" c + +parseMap :: (ObjectGet k, ObjectGet v) => A.Parser [(k, v)] +parseMap = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x80 -> + flip replicateM parsePair . fromIntegral $ c .&. 0x0F + 0xDE -> + flip replicateM parsePair . fromIntegral =<< parseUint16 + 0xDF -> + flip replicateM parsePair . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid map tag: 0x%02X" c + +parseMapVector :: (ObjectGet k, ObjectGet v) => A.Parser (V.Vector (k, v)) +parseMapVector = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x80 -> + flip V.replicateM parsePair . fromIntegral $ c .&. 0x0F + 0xDE -> + flip V.replicateM parsePair . fromIntegral =<< parseUint16 + 0xDF -> + flip V.replicateM parsePair . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid map tag: 0x%02X" c + +parseObject :: A.Parser Object +parseObject = + A.choice + [ liftM ObjectInteger parseInt + , liftM (const ObjectNil) parseNil + , liftM ObjectBool parseBool + , liftM ObjectDouble parseDouble + , liftM ObjectRAW parseRAW + , liftM ObjectArray parseArray + , liftM ObjectMap parseMap + ] + +parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v) +parsePair = do + a <- get + b <- get + return (a, b) + +parseUint16 :: A.Parser Word16 +parseUint16 = do + b0 <- A.anyWord8 + b1 <- A.anyWord8 + return $ (fromIntegral b0 `shiftL` 8) .|. fromIntegral b1 + +parseUint32 :: A.Parser Word32 +parseUint32 = do + b0 <- A.anyWord8 + b1 <- A.anyWord8 + b2 <- A.anyWord8 + b3 <- A.anyWord8 + return $ (fromIntegral b0 `shiftL` 24) .|. + (fromIntegral b1 `shiftL` 16) .|. + (fromIntegral b2 `shiftL` 8) .|. + fromIntegral b3 + +parseUint64 :: A.Parser Word64 +parseUint64 = do + b0 <- A.anyWord8 + b1 <- A.anyWord8 + b2 <- A.anyWord8 + b3 <- A.anyWord8 + b4 <- A.anyWord8 + b5 <- A.anyWord8 + b6 <- A.anyWord8 + b7 <- A.anyWord8 + return $ (fromIntegral b0 `shiftL` 56) .|. + (fromIntegral b1 `shiftL` 48) .|. + (fromIntegral b2 `shiftL` 40) .|. + (fromIntegral b3 `shiftL` 32) .|. + (fromIntegral b4 `shiftL` 24) .|. + (fromIntegral b5 `shiftL` 16) .|. + (fromIntegral b6 `shiftL` 8) .|. + fromIntegral b7 + +parseInt8 :: A.Parser Int8 +parseInt8 = return . fromIntegral =<< A.anyWord8 + +parseInt16 :: A.Parser Int16 +parseInt16 = return . fromIntegral =<< parseUint16 + +parseInt32 :: A.Parser Int32 +parseInt32 = return . fromIntegral =<< parseUint32 + +parseInt64 :: A.Parser Int64 +parseInt64 = return . fromIntegral =<< parseUint64 + +toLBS :: B.ByteString -> L.ByteString +toLBS bs = L.fromChunks [bs] From 209d8d058c17b1ee92f13942a804eb2868191118 Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 13:57:47 +0900 Subject: [PATCH 125/259] forgot to remove file --- haskell/src/Data/MessagePack/Packer.hs | 147 ------------------------- 1 file changed, 147 deletions(-) delete mode 100644 haskell/src/Data/MessagePack/Packer.hs diff --git a/haskell/src/Data/MessagePack/Packer.hs b/haskell/src/Data/MessagePack/Packer.hs deleted file mode 100644 index 9c10f5ed..00000000 --- a/haskell/src/Data/MessagePack/Packer.hs +++ /dev/null @@ -1,147 +0,0 @@ -{-# Language FlexibleInstances #-} -{-# Language OverlappingInstances #-} - -module Data.MessagePack.Packer( - ObjectPut(..), - ) where - -import Data.Binary.Put -import Data.Binary.IEEE754 -import Data.Bits -import qualified Data.ByteString as B - -import Data.MessagePack.Object - -class ObjectPut a where - put :: a -> Put - -instance ObjectPut Object where - put = putObject - -instance ObjectPut Int where - put = putInteger - -instance ObjectPut () where - put _ = putNil - -instance ObjectPut Bool where - put = putBool - -instance ObjectPut Double where - put = putDouble - -instance ObjectPut B.ByteString where - put = putRAW - -instance ObjectPut a => ObjectPut [a] where - put = putArray - -instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where - put = putMap - -putObject :: Object -> Put -putObject obj = - case obj of - ObjectInteger n -> - putInteger n - ObjectNil -> - putNil - ObjectBool b -> - putBool b - ObjectDouble d -> - putDouble d - ObjectRAW raw -> - putRAW raw - ObjectArray arr -> - putArray arr - ObjectMap m -> - putMap m - -putInteger :: Int -> Put -putInteger n = - case n of - _ | n >= 0 && n <= 127 -> - putWord8 $ fromIntegral n - _ | n >= -32 && n <= -1 -> - putWord8 $ fromIntegral n - _ | n >= 0 && n < 0x100 -> do - putWord8 0xCC - putWord8 $ fromIntegral n - _ | n >= 0 && n < 0x10000 -> do - putWord8 0xCD - putWord16be $ fromIntegral n - _ | n >= 0 && n < 0x100000000 -> do - putWord8 0xCE - putWord32be $ fromIntegral n - _ | n >= 0 -> do - putWord8 0xCF - putWord64be $ fromIntegral n - _ | n >= -0x100 -> do - putWord8 0xD0 - putWord8 $ fromIntegral n - _ | n >= -0x10000 -> do - putWord8 0xD1 - putWord16be $ fromIntegral n - _ | n >= -0x100000000 -> do - putWord8 0xD2 - putWord32be $ fromIntegral n - _ -> do - putWord8 0xD3 - putWord64be $ fromIntegral n - -putNil :: Put -putNil = putWord8 0xC0 - -putBool :: Bool -> Put -putBool True = putWord8 0xC3 -putBool False = putWord8 0xC2 - -putDouble :: Double -> Put -putDouble d = do - putWord8 0xCB - putFloat64be d - -putRAW :: B.ByteString -> Put -putRAW bs = do - case len of - _ | len <= 31 -> do - putWord8 $ 0xA0 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDA - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDB - putWord32be $ fromIntegral len - putByteString bs - where - len = B.length bs - -putArray :: ObjectPut a => [a] -> Put -putArray arr = do - case len of - _ | len <= 15 -> - putWord8 $ 0x90 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDC - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDD - putWord32be $ fromIntegral len - mapM_ put arr - where - len = length arr - -putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put -putMap m = do - case len of - _ | len <= 15 -> - putWord8 $ 0x80 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDE - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDF - putWord16be $ fromIntegral len - mapM_ (\(k, v) -> put k >> put v) m - where - len = length m From 799935e44c6f27e81d780b324dd69bdbd71066d5 Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 14:03:47 +0900 Subject: [PATCH 126/259] haskel: incr version and update infos. --- haskell/msgpack.cabal | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 8346c1f8..18ae3d86 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,15 +1,15 @@ Name: msgpack -Version: 0.3.0 +Version: 0.3.1 Synopsis: A Haskell binding to MessagePack Description: - A Haskell binding to MessagePack + A Haskell binding to MessagePack License: BSD3 License-File: LICENSE Category: Data Author: Hideyuki Tanaka Maintainer: Hideyuki Tanaka -Homepage: http://github.com/tanakh/hsmsgpack +Homepage: http://github.com/msgpack/msgpack Stability: Experimental Tested-with: GHC == 6.12.3 Cabal-Version: >= 1.2 From 802589516870df83cf209191e234266b09b1abee Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 5 Sep 2010 16:20:37 +0900 Subject: [PATCH 127/259] Checking in changes prior to tagging of version 0.16_01. Changelog diff is: --- perl/MANIFEST.SKIP | 2 ++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index f6340354..71a24e5c 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -23,3 +23,5 @@ \.o$ \.bs$ ^Data-MessagePack-[0-9.]+/ +^\.testenv/test_pp.pl +^ppport.h$ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index d53ff226..4da67ff6 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.16'; +our $VERSION = '0.16_01'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From e781831032c9091ab7e90bbbd9560828a8b69a30 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 6 Sep 2010 14:19:31 +0900 Subject: [PATCH 128/259] upgraded docs --- perl/README | 16 +++++++++++++--- perl/lib/Data/MessagePack.pm | 15 ++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/perl/README b/perl/README index 31aae992..2ef686c2 100644 --- a/perl/README +++ b/perl/README @@ -1,12 +1,16 @@ NAME - Data::MessagePack - messagepack + Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); DESCRIPTION - Data::MessagePack is a binary packer for perl. + This module converts Perl data structures to MessagePack and vice versa. + + MessagePack is a binary-based efficient object serialization format. It + enables to exchange structured objects between many languages like JSON. + But unlike JSON, it is very fast and small. METHODS my $packed = Data::MessagePack->pack($data); @@ -22,13 +26,19 @@ Configuration Variables AUTHORS Tokuhiro Matsuno + Makamaka Hannyaharamitu + THANKS TO Jun Kuriyama + Dan Kogai + + FURUHASHI Sadayuki + LICENSE This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. SEE ALSO - + diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 4da67ff6..ecdc1e48 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -30,7 +30,7 @@ __END__ =head1 NAME -Data::MessagePack - messagepack +Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS @@ -39,7 +39,10 @@ Data::MessagePack - messagepack =head1 DESCRIPTION -Data::MessagePack is a binary packer for perl. +This module converts Perl data structures to MessagePack and vice versa. + +MessagePack is a binary-based efficient object serialization format. +It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. =head1 METHODS @@ -69,10 +72,16 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). Tokuhiro Matsuno +Makamaka Hannyaharamitu + =head1 THANKS TO Jun Kuriyama +Dan Kogai + +FURUHASHI Sadayuki + =head1 LICENSE This library is free software; you can redistribute it and/or modify @@ -81,5 +90,5 @@ it under the same terms as Perl itself. =head1 SEE ALSO -L +L From c7555f1c3c471278b320db5ca71e5afdbcb52867 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 6 Sep 2010 14:31:53 +0900 Subject: [PATCH 129/259] Perl: added link to git repository. --- perl/Makefile.PL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 783e658d..7958bc6b 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -64,7 +64,7 @@ if($Module::Install::AUTHOR) { postamble qq{test :: test_pp\n\n}; } -auto_set_repository(); +repository('http://github.com/msgpack/msgpack'); auto_include; WriteAll; From 9281dba89672862ddf27384909264f4bd6ec12e8 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 6 Sep 2010 14:34:04 +0900 Subject: [PATCH 130/259] Checking in changes prior to tagging of version 0.16_02. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 9b061cf..68b58ba 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.16_02 + + - document enhancement(tokuhirom) + - M::I::XSUtil 0.26 is broken. use 0.27. + 0.16_01 - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS). --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 9b061cfb..68b58ba9 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.16_02 + + - document enhancement(tokuhirom) + - M::I::XSUtil 0.26 is broken. use 0.27. + 0.16_01 - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS). diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index ecdc1e48..79cc5311 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.16_01'; +our $VERSION = '0.16_02'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 8b90968cb111be903421083d8f3bebbef23e79c7 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 6 Sep 2010 14:34:48 +0900 Subject: [PATCH 131/259] Checking in changes prior to tagging of version 0.16_03. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 68b58ba..a4a3e36 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.16_03 + + - no feature changes + 0.16_02 - document enhancement(tokuhirom) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 68b58ba9..a4a3e364 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.16_03 + + - no feature changes + 0.16_02 - document enhancement(tokuhirom) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 79cc5311..b143e4ae 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.16_02'; +our $VERSION = '0.16_03'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From c5afe7a5739fa48d207d85403771de4a526ff437 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 6 Sep 2010 14:35:41 +0900 Subject: [PATCH 132/259] Checking in changes prior to tagging of version 0.16_04. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index a4a3e36..7910882 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,4 +1,4 @@ -0.16_03 +0.16_04 - no feature changes --- perl/Changes | 2 +- perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/Changes b/perl/Changes index a4a3e364..79108820 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,4 +1,4 @@ -0.16_03 +0.16_04 - no feature changes diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b143e4ae..b08bac2d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.16_03'; +our $VERSION = '0.16_04'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From aca2ba13c2f3ce3bc43897beb0a4a8529bab7a03 Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 15:37:55 +0900 Subject: [PATCH 133/259] haskell: refactoring --- haskell/src/Data/MessagePack/Parser.hs | 246 +++++++++++-------------- haskell/src/Data/MessagePack/Put.hs | 232 +++++++++-------------- 2 files changed, 195 insertions(+), 283 deletions(-) diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs index d0cd0846..312e95f3 100644 --- a/haskell/src/Data/MessagePack/Parser.hs +++ b/haskell/src/Data/MessagePack/Parser.hs @@ -40,166 +40,113 @@ class ObjectGet a where -- | Deserialize a value get :: A.Parser a +instance ObjectGet Object where + get = + A.choice + [ liftM ObjectInteger get + , liftM (\() -> ObjectNil) get + , liftM ObjectBool get + , liftM ObjectDouble get + , liftM ObjectRAW get + , liftM ObjectArray get + , liftM ObjectMap get + ] + instance ObjectGet Int where - get = parseInt + get = do + c <- A.anyWord8 + case c of + _ | c .&. 0x80 == 0x00 -> + return $ fromIntegral c + _ | c .&. 0xE0 == 0xE0 -> + return $ fromIntegral (fromIntegral c :: Int8) + 0xCC -> + return . fromIntegral =<< A.anyWord8 + 0xCD -> + return . fromIntegral =<< parseUint16 + 0xCE -> + return . fromIntegral =<< parseUint32 + 0xCF -> + return . fromIntegral =<< parseUint64 + 0xD0 -> + return . fromIntegral =<< parseInt8 + 0xD1 -> + return . fromIntegral =<< parseInt16 + 0xD2 -> + return . fromIntegral =<< parseInt32 + 0xD3 -> + return . fromIntegral =<< parseInt64 + _ -> + fail $ printf "invlid integer tag: 0x%02X" c instance ObjectGet () where - get = parseNil + get = do + c <- A.anyWord8 + case c of + 0xC0 -> + return () + _ -> + fail $ printf "invlid nil tag: 0x%02X" c instance ObjectGet Bool where - get = parseBool + get = do + c <- A.anyWord8 + case c of + 0xC3 -> + return True + 0xC2 -> + return False + _ -> + fail $ printf "invlid bool tag: 0x%02X" c instance ObjectGet Double where - get = parseDouble + get = do + c <- A.anyWord8 + case c of + 0xCA -> + return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4 + 0xCB -> + return . runGet getFloat64be . toLBS =<< A.take 8 + _ -> + fail $ printf "invlid double tag: 0x%02X" c instance ObjectGet B.ByteString where - get = parseRAW + get = do + c <- A.anyWord8 + case c of + _ | c .&. 0xE0 == 0xA0 -> + A.take . fromIntegral $ c .&. 0x1F + 0xDA -> + A.take . fromIntegral =<< parseUint16 + 0xDB -> + A.take . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid raw tag: 0x%02X" c instance ObjectGet a => ObjectGet [a] where - get = parseArray + get = parseArray (flip replicateM get) instance ObjectGet a => ObjectGet (V.Vector a) where - get = parseArrayVector + get = parseArray (flip V.replicateM get) + +parseArray :: (Int -> A.Parser a) -> A.Parser a +parseArray aget = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x90 -> + aget . fromIntegral $ c .&. 0x0F + 0xDC -> + aget . fromIntegral =<< parseUint16 + 0xDD -> + aget . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid array tag: 0x%02X" c instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where - get = parseMap + get = parseMap (flip replicateM parsePair) instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where - get = parseMapVector - -instance ObjectGet Object where - get = parseObject - -parseInt :: A.Parser Int -parseInt = do - c <- A.anyWord8 - case c of - _ | c .&. 0x80 == 0x00 -> - return $ fromIntegral c - _ | c .&. 0xE0 == 0xE0 -> - return $ fromIntegral (fromIntegral c :: Int8) - 0xCC -> - return . fromIntegral =<< A.anyWord8 - 0xCD -> - return . fromIntegral =<< parseUint16 - 0xCE -> - return . fromIntegral =<< parseUint32 - 0xCF -> - return . fromIntegral =<< parseUint64 - 0xD0 -> - return . fromIntegral =<< parseInt8 - 0xD1 -> - return . fromIntegral =<< parseInt16 - 0xD2 -> - return . fromIntegral =<< parseInt32 - 0xD3 -> - return . fromIntegral =<< parseInt64 - _ -> - fail $ printf "invlid integer tag: 0x%02X" c - -parseNil :: A.Parser () -parseNil = do - _ <- A.word8 0xC0 - return () - -parseBool :: A.Parser Bool -parseBool = do - c <- A.anyWord8 - case c of - 0xC3 -> - return True - 0xC2 -> - return False - _ -> - fail $ printf "invlid bool tag: 0x%02X" c - -parseDouble :: A.Parser Double -parseDouble = do - c <- A.anyWord8 - case c of - 0xCA -> - return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4 - 0xCB -> - return . runGet getFloat64be . toLBS =<< A.take 8 - _ -> - fail $ printf "invlid double tag: 0x%02X" c - -parseRAW :: A.Parser B.ByteString -parseRAW = do - c <- A.anyWord8 - case c of - _ | c .&. 0xE0 == 0xA0 -> - A.take . fromIntegral $ c .&. 0x1F - 0xDA -> - A.take . fromIntegral =<< parseUint16 - 0xDB -> - A.take . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid raw tag: 0x%02X" c - -parseArray :: ObjectGet a => A.Parser [a] -parseArray = do - c <- A.anyWord8 - case c of - _ | c .&. 0xF0 == 0x90 -> - flip replicateM get . fromIntegral $ c .&. 0x0F - 0xDC -> - flip replicateM get . fromIntegral =<< parseUint16 - 0xDD -> - flip replicateM get . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid array tag: 0x%02X" c - -parseArrayVector :: ObjectGet a => A.Parser (V.Vector a) -parseArrayVector = do - c <- A.anyWord8 - case c of - _ | c .&. 0xF0 == 0x90 -> - flip V.replicateM get . fromIntegral $ c .&. 0x0F - 0xDC -> - flip V.replicateM get . fromIntegral =<< parseUint16 - 0xDD -> - flip V.replicateM get . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid array tag: 0x%02X" c - -parseMap :: (ObjectGet k, ObjectGet v) => A.Parser [(k, v)] -parseMap = do - c <- A.anyWord8 - case c of - _ | c .&. 0xF0 == 0x80 -> - flip replicateM parsePair . fromIntegral $ c .&. 0x0F - 0xDE -> - flip replicateM parsePair . fromIntegral =<< parseUint16 - 0xDF -> - flip replicateM parsePair . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid map tag: 0x%02X" c - -parseMapVector :: (ObjectGet k, ObjectGet v) => A.Parser (V.Vector (k, v)) -parseMapVector = do - c <- A.anyWord8 - case c of - _ | c .&. 0xF0 == 0x80 -> - flip V.replicateM parsePair . fromIntegral $ c .&. 0x0F - 0xDE -> - flip V.replicateM parsePair . fromIntegral =<< parseUint16 - 0xDF -> - flip V.replicateM parsePair . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid map tag: 0x%02X" c - -parseObject :: A.Parser Object -parseObject = - A.choice - [ liftM ObjectInteger parseInt - , liftM (const ObjectNil) parseNil - , liftM ObjectBool parseBool - , liftM ObjectDouble parseDouble - , liftM ObjectRAW parseRAW - , liftM ObjectArray parseArray - , liftM ObjectMap parseMap - ] + get = parseMap (flip V.replicateM parsePair) parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v) parsePair = do @@ -207,6 +154,19 @@ parsePair = do b <- get return (a, b) +parseMap :: (Int -> A.Parser a) -> A.Parser a +parseMap aget = do + c <- A.anyWord8 + case c of + _ | c .&. 0xF0 == 0x80 -> + aget . fromIntegral $ c .&. 0x0F + 0xDE -> + aget . fromIntegral =<< parseUint16 + 0xDF -> + aget . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid map tag: 0x%02X" c + parseUint16 :: A.Parser Word16 parseUint16 = do b0 <- A.anyWord8 diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs index 8d0af2b2..95582dd8 100644 --- a/haskell/src/Data/MessagePack/Put.hs +++ b/haskell/src/Data/MessagePack/Put.hs @@ -35,168 +35,120 @@ class ObjectPut a where put :: a -> Put instance ObjectPut Object where - put = putObject + put obj = + case obj of + ObjectInteger n -> + put n + ObjectNil -> + put () + ObjectBool b -> + put b + ObjectDouble d -> + put d + ObjectRAW raw -> + put raw + ObjectArray arr -> + put arr + ObjectMap m -> + put m instance ObjectPut Int where - put = putInteger + put n = + case n of + _ | n >= 0 && n <= 127 -> + putWord8 $ fromIntegral n + _ | n >= -32 && n <= -1 -> + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x100 -> do + putWord8 0xCC + putWord8 $ fromIntegral n + _ | n >= 0 && n < 0x10000 -> do + putWord8 0xCD + putWord16be $ fromIntegral n + _ | n >= 0 && n < 0x100000000 -> do + putWord8 0xCE + putWord32be $ fromIntegral n + _ | n >= 0 -> do + putWord8 0xCF + putWord64be $ fromIntegral n + _ | n >= -0x80 -> do + putWord8 0xD0 + putWord8 $ fromIntegral n + _ | n >= -0x8000 -> do + putWord8 0xD1 + putWord16be $ fromIntegral n + _ | n >= -0x80000000 -> do + putWord8 0xD2 + putWord32be $ fromIntegral n + _ -> do + putWord8 0xD3 + putWord64be $ fromIntegral n instance ObjectPut () where - put _ = putNil + put _ = + putWord8 0xC0 instance ObjectPut Bool where - put = putBool + put True = putWord8 0xC3 + put False = putWord8 0xC2 instance ObjectPut Double where - put = putDouble + put d = do + putWord8 0xCB + putFloat64be d instance ObjectPut B.ByteString where - put = putRAW + put bs = do + case len of + _ | len <= 31 -> do + putWord8 $ 0xA0 .|. fromIntegral len + _ | len < 0x10000 -> do + putWord8 0xDA + putWord16be $ fromIntegral len + _ -> do + putWord8 0xDB + putWord32be $ fromIntegral len + putByteString bs + where + len = B.length bs instance ObjectPut a => ObjectPut [a] where - put = putArray + put = putArray length (mapM_ put) instance ObjectPut a => ObjectPut (V.Vector a) where - put = putArrayVector + put = putArray V.length (V.mapM_ put) + +putArray :: (a -> Int) -> (a -> Put) -> a -> Put +putArray lf pf arr = do + case lf arr of + len | len <= 15 -> + putWord8 $ 0x90 .|. fromIntegral len + len | len < 0x10000 -> do + putWord8 0xDC + putWord16be $ fromIntegral len + len -> do + putWord8 0xDD + putWord32be $ fromIntegral len + pf arr instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where - put = putMap + put = putMap length (mapM_ putPair) instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where - put = putMapVector + put = putMap V.length (V.mapM_ putPair) -putObject :: Object -> Put -putObject obj = - case obj of - ObjectInteger n -> - putInteger n - ObjectNil -> - putNil - ObjectBool b -> - putBool b - ObjectDouble d -> - putDouble d - ObjectRAW raw -> - putRAW raw - ObjectArray arr -> - putArray arr - ObjectMap m -> - putMap m +putPair :: (ObjectPut a, ObjectPut b) => (a, b) -> Put +putPair (a, b) = put a >> put b -putInteger :: Int -> Put -putInteger n = - case n of - _ | n >= 0 && n <= 127 -> - putWord8 $ fromIntegral n - _ | n >= -32 && n <= -1 -> - putWord8 $ fromIntegral n - _ | n >= 0 && n < 0x100 -> do - putWord8 0xCC - putWord8 $ fromIntegral n - _ | n >= 0 && n < 0x10000 -> do - putWord8 0xCD - putWord16be $ fromIntegral n - _ | n >= 0 && n < 0x100000000 -> do - putWord8 0xCE - putWord32be $ fromIntegral n - _ | n >= 0 -> do - putWord8 0xCF - putWord64be $ fromIntegral n - _ | n >= -0x80 -> do - putWord8 0xD0 - putWord8 $ fromIntegral n - _ | n >= -0x8000 -> do - putWord8 0xD1 - putWord16be $ fromIntegral n - _ | n >= -0x80000000 -> do - putWord8 0xD2 - putWord32be $ fromIntegral n - _ -> do - putWord8 0xD3 - putWord64be $ fromIntegral n - -putNil :: Put -putNil = putWord8 0xC0 - -putBool :: Bool -> Put -putBool True = putWord8 0xC3 -putBool False = putWord8 0xC2 - -putDouble :: Double -> Put -putDouble d = do - putWord8 0xCB - putFloat64be d - -putRAW :: B.ByteString -> Put -putRAW bs = do - case len of - _ | len <= 31 -> do - putWord8 $ 0xA0 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDA - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDB - putWord32be $ fromIntegral len - putByteString bs - where - len = B.length bs - -putArray :: ObjectPut a => [a] -> Put -putArray arr = do - case len of - _ | len <= 15 -> - putWord8 $ 0x90 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDC - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDD - putWord32be $ fromIntegral len - mapM_ put arr - where - len = length arr - -putArrayVector :: ObjectPut a => V.Vector a -> Put -putArrayVector arr = do - case len of - _ | len <= 15 -> - putWord8 $ 0x90 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDC - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDD - putWord32be $ fromIntegral len - V.mapM_ put arr - where - len = V.length arr - -putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put -putMap m = do - case len of - _ | len <= 15 -> +putMap :: (a -> Int) -> (a -> Put) -> a -> Put +putMap lf pf m = do + case lf m of + len | len <= 15 -> putWord8 $ 0x80 .|. fromIntegral len - _ | len < 0x10000 -> do + len | len < 0x10000 -> do putWord8 0xDE putWord16be $ fromIntegral len - _ -> do + len -> do putWord8 0xDF putWord32be $ fromIntegral len - mapM_ (\(k, v) -> put k >> put v) m - where - len = length m - -putMapVector :: (ObjectPut k, ObjectPut v) => V.Vector (k, v) -> Put -putMapVector m = do - case len of - _ | len <= 15 -> - putWord8 $ 0x80 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDE - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDF - putWord32be $ fromIntegral len - V.mapM_ (\(k, v) -> put k >> put v) m - where - len = V.length m + pf m From 9e50ba6ec6f48071a5cc31b44864194446b9aa6f Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 16:33:36 +0900 Subject: [PATCH 134/259] haskell: instance tupples and String and lazy ByteString --- haskell/src/Data/MessagePack/Parser.hs | 73 ++++++++++++++++++++++---- haskell/src/Data/MessagePack/Put.hs | 68 +++++++++++++++++++----- haskell/test/Test.hs | 25 +++++++-- 3 files changed, 139 insertions(+), 27 deletions(-) diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs index 312e95f3..200ad962 100644 --- a/haskell/src/Data/MessagePack/Parser.hs +++ b/haskell/src/Data/MessagePack/Parser.hs @@ -1,6 +1,7 @@ {-# Language FlexibleInstances #-} {-# Language IncoherentInstances #-} {-# Language OverlappingInstances #-} +{-# Language TypeSynonymInstances #-} -------------------------------------------------------------------- -- | @@ -27,6 +28,7 @@ import Data.Binary.Get import Data.Binary.IEEE754 import Data.Bits import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Lazy as L import Data.Int import qualified Data.Vector as V @@ -110,18 +112,27 @@ instance ObjectGet Double where _ -> fail $ printf "invlid double tag: 0x%02X" c +instance ObjectGet String where + get = parseString (\n -> return . B8.unpack =<< A.take n) + instance ObjectGet B.ByteString where - get = do - c <- A.anyWord8 - case c of - _ | c .&. 0xE0 == 0xA0 -> - A.take . fromIntegral $ c .&. 0x1F - 0xDA -> - A.take . fromIntegral =<< parseUint16 - 0xDB -> - A.take . fromIntegral =<< parseUint32 - _ -> - fail $ printf "invlid raw tag: 0x%02X" c + get = parseString A.take + +instance ObjectGet L.ByteString where + get = parseString (\n -> do bs <- A.take n; return $ L.fromChunks [bs]) + +parseString :: (Int -> A.Parser a) -> A.Parser a +parseString aget = do + c <- A.anyWord8 + case c of + _ | c .&. 0xE0 == 0xA0 -> + aget . fromIntegral $ c .&. 0x1F + 0xDA -> + aget . fromIntegral =<< parseUint16 + 0xDB -> + aget . fromIntegral =<< parseUint32 + _ -> + fail $ printf "invlid raw tag: 0x%02X" c instance ObjectGet a => ObjectGet [a] where get = parseArray (flip replicateM get) @@ -129,6 +140,46 @@ instance ObjectGet a => ObjectGet [a] where instance ObjectGet a => ObjectGet (V.Vector a) where get = parseArray (flip V.replicateM get) +instance (ObjectGet a1, ObjectGet a2) => ObjectGet (a1, a2) where + get = parseArray f where + f 2 = get >>= \a1 -> get >>= \a2 -> return (a1, a2) + f n = fail $ printf "wrong tupple size: expected 2 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3) => ObjectGet (a1, a2, a3) where + get = parseArray f where + f 3 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> return (a1, a2, a3) + f n = fail $ printf "wrong tupple size: expected 3 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4) => ObjectGet (a1, a2, a3, a4) where + get = parseArray f where + f 4 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> return (a1, a2, a3, a4) + f n = fail $ printf "wrong tupple size: expected 4 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5) => ObjectGet (a1, a2, a3, a4, a5) where + get = parseArray f where + f 5 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> return (a1, a2, a3, a4, a5) + f n = fail $ printf "wrong tupple size: expected 5 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6) => ObjectGet (a1, a2, a3, a4, a5, a6) where + get = parseArray f where + f 6 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> return (a1, a2, a3, a4, a5, a6) + f n = fail $ printf "wrong tupple size: expected 6 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7) => ObjectGet (a1, a2, a3, a4, a5, a6, a7) where + get = parseArray f where + f 7 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> return (a1, a2, a3, a4, a5, a6, a7) + f n = fail $ printf "wrong tupple size: expected 7 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8) where + get = parseArray f where + f 8 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> return (a1, a2, a3, a4, a5, a6, a7, a8) + f n = fail $ printf "wrong tupple size: expected 8 but got " n + +instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8, ObjectGet a9) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8, a9) where + get = parseArray f where + f 9 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> get >>= \a9 -> return (a1, a2, a3, a4, a5, a6, a7, a8, a9) + f n = fail $ printf "wrong tupple size: expected 9 but got " n + parseArray :: (Int -> A.Parser a) -> A.Parser a parseArray aget = do c <- A.anyWord8 diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs index 95582dd8..24ec3059 100644 --- a/haskell/src/Data/MessagePack/Put.hs +++ b/haskell/src/Data/MessagePack/Put.hs @@ -1,6 +1,7 @@ {-# Language FlexibleInstances #-} {-# Language IncoherentInstances #-} {-# Language OverlappingInstances #-} +{-# Language TypeSynonymInstances #-} -------------------------------------------------------------------- -- | @@ -25,6 +26,8 @@ import Data.Binary.Put import Data.Binary.IEEE754 import Data.Bits import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 +import qualified Data.ByteString.Lazy as L import qualified Data.Vector as V import Data.MessagePack.Object @@ -97,20 +100,27 @@ instance ObjectPut Double where putWord8 0xCB putFloat64be d +instance ObjectPut String where + put = putString length (putByteString . B8.pack) + instance ObjectPut B.ByteString where - put bs = do - case len of - _ | len <= 31 -> do - putWord8 $ 0xA0 .|. fromIntegral len - _ | len < 0x10000 -> do - putWord8 0xDA - putWord16be $ fromIntegral len - _ -> do - putWord8 0xDB - putWord32be $ fromIntegral len - putByteString bs - where - len = B.length bs + put = putString B.length putByteString + +instance ObjectPut L.ByteString where + put = putString (fromIntegral . L.length) putLazyByteString + +putString :: (s -> Int) -> (s -> Put) -> s -> Put +putString lf pf str = do + case lf str of + len | len <= 31 -> do + putWord8 $ 0xA0 .|. fromIntegral len + len | len < 0x10000 -> do + putWord8 0xDA + putWord16be $ fromIntegral len + len -> do + putWord8 0xDB + putWord32be $ fromIntegral len + pf str instance ObjectPut a => ObjectPut [a] where put = putArray length (mapM_ put) @@ -118,6 +128,38 @@ instance ObjectPut a => ObjectPut [a] where instance ObjectPut a => ObjectPut (V.Vector a) where put = putArray V.length (V.mapM_ put) +instance (ObjectPut a1, ObjectPut a2) => ObjectPut (a1, a2) where + put = putArray (const 2) f where + f (a1, a2) = put a1 >> put a2 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3) => ObjectPut (a1, a2, a3) where + put = putArray (const 3) f where + f (a1, a2, a3) = put a1 >> put a2 >> put a3 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4) => ObjectPut (a1, a2, a3, a4) where + put = putArray (const 4) f where + f (a1, a2, a3, a4) = put a1 >> put a2 >> put a3 >> put a4 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5) => ObjectPut (a1, a2, a3, a4, a5) where + put = putArray (const 5) f where + f (a1, a2, a3, a4, a5) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6) => ObjectPut (a1, a2, a3, a4, a5, a6) where + put = putArray (const 6) f where + f (a1, a2, a3, a4, a5, a6) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7) => ObjectPut (a1, a2, a3, a4, a5, a6, a7) where + put = putArray (const 7) f where + f (a1, a2, a3, a4, a5, a6, a7) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8) where + put = putArray (const 8) f where + f (a1, a2, a3, a4, a5, a6, a7, a8) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 + +instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8, ObjectPut a9) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8, a9) where + put = putArray (const 9) f where + f (a1, a2, a3, a4, a5, a6, a7, a8, a9) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 >> put a9 + putArray :: (a -> Int) -> (a -> Put) -> a -> Put putArray lf pf arr = do case lf arr of diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs index 1bb551c1..a73ac9ab 100644 --- a/haskell/test/Test.hs +++ b/haskell/test/Test.hs @@ -4,6 +4,7 @@ import Test.QuickCheck import Control.Monad import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Lazy.Char8 as L import Data.MessagePack mid :: (ObjectGet a, ObjectPut a) => a -> a @@ -17,15 +18,27 @@ prop_mid_bool a = a == mid a where types = a :: Bool prop_mid_double a = a == mid a where types = a :: Double -prop_mid_string a = a == B.unpack (mid (B.pack a)) +prop_mid_string a = a == mid a + where types = a :: String +prop_mid_bytestring a = B.pack a == mid (B.pack a) + where types = a :: String +prop_mid_lazy_bytestring a = (L.pack a) == mid (L.pack a) where types = a :: String prop_mid_array_int a = a == mid a where types = a :: [Int] -prop_mid_array_string a = a == map B.unpack (mid (map B.pack a)) +prop_mid_array_string a = a == mid a where types = a :: [String] +prop_mid_pair2 a = a == mid a + where types = a :: (Int, Int) +prop_mid_pair3 a = a == mid a + where types = a :: (Int, Int, Int) +prop_mid_pair4 a = a == mid a + where types = a :: (Int, Int, Int, Int) +prop_mid_pair5 a = a == mid a + where types = a :: (Int, Int, Int, Int, Int) prop_mid_map_int_double a = a == mid a where types = a :: [(Int, Double)] -prop_mid_map_string_string a = a == map (\(x, y) -> (B.unpack x, B.unpack y)) (mid (map (\(x, y) -> (B.pack x, B.pack y)) a)) +prop_mid_map_string_string a = a == mid a where types = a :: [(String, String)] tests = @@ -35,8 +48,14 @@ tests = , testProperty "bool" prop_mid_bool , testProperty "double" prop_mid_double , testProperty "string" prop_mid_string + , testProperty "bytestring" prop_mid_bytestring + , testProperty "lazy-bytestring" prop_mid_lazy_bytestring , testProperty "[int]" prop_mid_array_int , testProperty "[string]" prop_mid_array_string + , testProperty "(int, int)" prop_mid_pair2 + , testProperty "(int, int, int)" prop_mid_pair3 + , testProperty "(int, int, int, int)" prop_mid_pair4 + , testProperty "(int, int, int, int, int)" prop_mid_pair5 , testProperty "[(int, double)]" prop_mid_map_int_double , testProperty "[(string, string)]" prop_mid_map_string_string ] From b75db110dceef9bf75c8410ca4b4fc031e1aad89 Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 17:00:22 +0900 Subject: [PATCH 135/259] haskell: add Iteratee interface --- haskell/msgpack.cabal | 1 + haskell/src/Data/MessagePack.hs | 45 ++++++++++++----------- haskell/src/Data/MessagePack/Iteratee.hs | 46 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 haskell/src/Data/MessagePack/Iteratee.hs diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 18ae3d86..3baff77f 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -33,3 +33,4 @@ Library Data.MessagePack.Object Data.MessagePack.Put Data.MessagePack.Parser + Data.MessagePack.Iteratee diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs index 010eaab0..92353c53 100644 --- a/haskell/src/Data/MessagePack.hs +++ b/haskell/src/Data/MessagePack.hs @@ -16,6 +16,7 @@ module Data.MessagePack( module Data.MessagePack.Object, module Data.MessagePack.Put, module Data.MessagePack.Parser, + module Data.MessagePack.Iteratee, -- * Simple functions of Pack and Unpack pack, @@ -30,6 +31,9 @@ module Data.MessagePack( unpackFromString, unpackFromHandle, unpackFromFile, + unpackFromStringI, + unpackFromHandleI, + unpackFromFileI, ) where @@ -47,6 +51,7 @@ import System.IO import Data.MessagePack.Object import Data.MessagePack.Put import Data.MessagePack.Parser +import Data.MessagePack.Iteratee bufferSize :: Int bufferSize = 4 * 1024 @@ -67,7 +72,7 @@ pack = packToString . put -- | Unpack MessagePack string to Haskell data. unpack :: (ObjectGet a, IsByteString s) => s -> a unpack bs = - runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) (parserToIteratee get) + runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) getI -- TODO: tryUnpack @@ -86,32 +91,32 @@ packToFile path = L.writeFile path . packToString -- | Unpack from ByteString unpackFromString :: (Monad m, IsByteString s) => s -> A.Parser a -> m a unpackFromString bs = - I.run . I.joinIM . I.enumPure1Chunk (toBS bs) . parserToIteratee + unpackFromStringI bs . parserToIteratee -- | Unpack from Handle unpackFromHandle :: CIO.MonadCatchIO m => Handle -> A.Parser a -> m a unpackFromHandle h = - I.run . I.joinIM . I.enumHandle bufferSize h . parserToIteratee + unpackFromHandleI h .parserToIteratee -- | Unpack from File unpackFromFile :: CIO.MonadCatchIO m => FilePath -> A.Parser a -> m a -unpackFromFile path p = +unpackFromFile path = + unpackFromFileI path . parserToIteratee + +-- | Iteratee interface to unpack from ByteString +unpackFromStringI :: (Monad m, IsByteString s) => s -> I.Iteratee B.ByteString m a -> m a +unpackFromStringI bs = + I.run . I.joinIM . I.enumPure1Chunk (toBS bs) + +-- | Iteratee interface to unpack from Handle +unpackFromHandleI :: CIO.MonadCatchIO m => Handle -> I.Iteratee B.ByteString m a -> m a +unpackFromHandleI h = + I.run . I.joinIM . I.enumHandle bufferSize h + +-- | Iteratee interface to unpack from File +unpackFromFileI :: CIO.MonadCatchIO m => FilePath -> I.Iteratee B.ByteString m a -> m a +unpackFromFileI path p = CIO.bracket (liftIO $ openBinaryFile path ReadMode) (liftIO . hClose) - (flip unpackFromHandle p) - -parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a -parserToIteratee p = I.icont (itr (A.parse p)) Nothing - where - itr pcont s = case s of - I.EOF _ -> - I.throwErr (I.setEOF s) - I.Chunk bs -> - case pcont bs of - A.Fail _ _ msg -> - I.throwErr (I.iterStrExc msg) - A.Partial cont -> - I.icont (itr cont) Nothing - A.Done remain ret -> - I.idone ret (I.Chunk remain) + (flip unpackFromHandleI p) diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs new file mode 100644 index 00000000..789b714a --- /dev/null +++ b/haskell/src/Data/MessagePack/Iteratee.hs @@ -0,0 +1,46 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Iteratee +-- Copyright : (c) Hideyuki Tanaka, 2009-2010 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- MessagePack Deserializer interface to @Data.Iteratee@ +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Iteratee( + -- * Iteratee version of deserializer + getI, + -- * Convert Parser to Iteratee + parserToIteratee, + ) where + +import qualified Data.Attoparsec as A +import qualified Data.ByteString as B +import qualified Data.Iteratee as I + +import Data.MessagePack.Parser + +-- | Deserialize a value +getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a +getI = parserToIteratee get + +-- | Convert Parser to Iteratee +parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a +parserToIteratee p = I.icont (itr (A.parse p)) Nothing + where + itr pcont s = case s of + I.EOF _ -> + I.throwErr (I.setEOF s) + I.Chunk bs -> + case pcont bs of + A.Fail _ _ msg -> + I.throwErr (I.iterStrExc msg) + A.Partial cont -> + I.icont (itr cont) Nothing + A.Done remain ret -> + I.idone ret (I.Chunk remain) From dfe19d308caa43e8d763750faafc2baade7d013c Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 18:14:47 +0900 Subject: [PATCH 136/259] haskell: add overlapping instances --- haskell/src/Data/MessagePack/Object.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs index 19a3aeba..df0e89dd 100644 --- a/haskell/src/Data/MessagePack/Object.hs +++ b/haskell/src/Data/MessagePack/Object.hs @@ -1,5 +1,6 @@ -{-# LANGUAGE TypeSynonymInstances #-} -{-# LANGUAGE FlexibleInstances #-} +{-# Language TypeSynonymInstances #-} +{-# Language FlexibleInstances #-} +{-# Language OverlappingInstances #-} -------------------------------------------------------------------- -- | From c6424c2ce71f3e79e8aacfe9c76846cf94e168de Mon Sep 17 00:00:00 2001 From: tanakh Date: Mon, 6 Sep 2010 23:27:50 +0900 Subject: [PATCH 137/259] haskell: nonblocking enumerator --- haskell/src/Data/MessagePack.hs | 3 +- haskell/src/Data/MessagePack/Iteratee.hs | 36 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs index 92353c53..b53066b1 100644 --- a/haskell/src/Data/MessagePack.hs +++ b/haskell/src/Data/MessagePack.hs @@ -45,7 +45,6 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as L import Data.Functor.Identity import qualified Data.Iteratee as I -import qualified Data.Iteratee.IO as I import System.IO import Data.MessagePack.Object @@ -111,7 +110,7 @@ unpackFromStringI bs = -- | Iteratee interface to unpack from Handle unpackFromHandleI :: CIO.MonadCatchIO m => Handle -> I.Iteratee B.ByteString m a -> m a unpackFromHandleI h = - I.run . I.joinIM . I.enumHandle bufferSize h + I.run . I.joinIM . enumHandleNonBlocking bufferSize h -- | Iteratee interface to unpack from File unpackFromFileI :: CIO.MonadCatchIO m => FilePath -> I.Iteratee B.ByteString m a -> m a diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs index 789b714a..4258cf68 100644 --- a/haskell/src/Data/MessagePack/Iteratee.hs +++ b/haskell/src/Data/MessagePack/Iteratee.hs @@ -15,13 +15,18 @@ module Data.MessagePack.Iteratee( -- * Iteratee version of deserializer getI, + -- * Non Blocking Enumerator + enumHandleNonBlocking, -- * Convert Parser to Iteratee parserToIteratee, ) where +import Control.Exception +import Control.Monad.IO.Class import qualified Data.Attoparsec as A import qualified Data.ByteString as B import qualified Data.Iteratee as I +import System.IO import Data.MessagePack.Parser @@ -29,6 +34,37 @@ import Data.MessagePack.Parser getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a getI = parserToIteratee get +-- | Enumerator +enumHandleNonBlocking :: MonadIO m => Int -> Handle -> I.Enumerator B.ByteString m a +enumHandleNonBlocking bufSize h = + I.enumFromCallback $ readSome bufSize h + +readSome :: MonadIO m => Int -> Handle -> m (Either SomeException (Bool, B.ByteString)) +readSome bufSize h = liftIO $ do + ebs <- try $ hGetSome bufSize h + case ebs of + Left exc -> + return $ Left (exc :: SomeException) + Right bs | B.null bs -> + return $ Right (False, B.empty) + Right bs -> + return $ Right (True, bs) + +hGetSome :: Int -> Handle -> IO B.ByteString +hGetSome bufSize h = do + bs <- B.hGetNonBlocking h bufSize + if B.null bs + then do + hd <- B.hGet h 1 + if B.null hd + then do + return B.empty + else do + rest <- B.hGetNonBlocking h (bufSize - 1) + return $ B.cons (B.head hd) rest + else do + return bs + -- | Convert Parser to Iteratee parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a parserToIteratee p = I.icont (itr (A.parse p)) Nothing From c56926428c2b66fd3f112b9095c46f46e0527cd7 Mon Sep 17 00:00:00 2001 From: tanakh Date: Tue, 7 Sep 2010 16:14:00 +0900 Subject: [PATCH 138/259] haskell: add packToHandle' --- haskell/src/Data/MessagePack.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs index b53066b1..7137589f 100644 --- a/haskell/src/Data/MessagePack.hs +++ b/haskell/src/Data/MessagePack.hs @@ -25,6 +25,7 @@ module Data.MessagePack( -- * Pack functions packToString, packToHandle, + packToHandle', packToFile, -- * Unpack functions @@ -83,6 +84,10 @@ packToString = runPut packToHandle :: Handle -> Put -> IO () packToHandle h = L.hPutStr h . packToString +-- | Pack to Handle and Flush Handle +packToHandle' :: Handle -> Put -> IO () +packToHandle' h p = packToHandle h p >> hFlush h + -- | Pack to File packToFile :: FilePath -> Put -> IO () packToFile path = L.writeFile path . packToString From 169f287970a68e52d766b485f9c870ef83916b59 Mon Sep 17 00:00:00 2001 From: tanakh Date: Tue, 7 Sep 2010 16:14:29 +0900 Subject: [PATCH 139/259] haskell: Now, Object is an instance of NFData. --- haskell/msgpack.cabal | 4 +++- haskell/src/Data/MessagePack/Object.hs | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 3baff77f..bd10c4aa 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -24,7 +24,9 @@ Library iteratee >= 0.4 && < 0.5, attoparsec >= 0.8.1 && < 0.8.2, binary >= 0.5.0 && < 0.5.1, - data-binary-ieee754 >= 0.4 && < 0.5 + data-binary-ieee754 >= 0.4 && < 0.5, + deepseq >= 1.1 && <1.2 + Ghc-options: -Wall -O2 Hs-source-dirs: src diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs index df0e89dd..6806722b 100644 --- a/haskell/src/Data/MessagePack/Object.hs +++ b/haskell/src/Data/MessagePack/Object.hs @@ -25,6 +25,7 @@ module Data.MessagePack.Object( Result, ) where +import Control.DeepSeq import Control.Monad import Control.Monad.Trans.Error () import qualified Data.ByteString as B @@ -41,6 +42,17 @@ data Object = | ObjectMap [(Object, Object)] deriving (Show) +instance NFData Object where + rnf obj = + case obj of + ObjectNil -> () + ObjectBool b -> rnf b + ObjectInteger n -> rnf n + ObjectDouble d -> rnf d + ObjectRAW bs -> bs `seq` () + ObjectArray a -> rnf a + ObjectMap m -> rnf m + -- | The class of types serializable to and from MessagePack object class OBJECT a where -- | Encode a value to MessagePack object From 5e19bc6f844500e729d498ee6275a6a2e6557ba2 Mon Sep 17 00:00:00 2001 From: tanakh Date: Tue, 7 Sep 2010 17:35:24 +0900 Subject: [PATCH 140/259] haskell: Object is Eq, Ord, Typeable. --- haskell/src/Data/MessagePack/Object.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs index 6806722b..87f24bd9 100644 --- a/haskell/src/Data/MessagePack/Object.hs +++ b/haskell/src/Data/MessagePack/Object.hs @@ -1,6 +1,7 @@ {-# Language TypeSynonymInstances #-} {-# Language FlexibleInstances #-} {-# Language OverlappingInstances #-} +{-# Language DeriveDataTypeable #-} -------------------------------------------------------------------- -- | @@ -30,6 +31,7 @@ import Control.Monad import Control.Monad.Trans.Error () import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as C8 +import Data.Typeable -- | Object Representation of MessagePack data. data Object = @@ -40,7 +42,7 @@ data Object = | ObjectRAW B.ByteString | ObjectArray [Object] | ObjectMap [(Object, Object)] - deriving (Show) + deriving (Show, Eq, Ord, Typeable) instance NFData Object where rnf obj = From a99870645244f9073075a43c30dc20511de89097 Mon Sep 17 00:00:00 2001 From: tanakh Date: Wed, 8 Sep 2010 13:36:45 +0900 Subject: [PATCH 141/259] haskell: update cabal file --- haskell/msgpack.cabal | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index bd10c4aa..ccdb2f7f 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,18 +1,18 @@ Name: msgpack -Version: 0.3.1 +Version: 0.3.1.1 Synopsis: A Haskell binding to MessagePack Description: A Haskell binding to MessagePack License: BSD3 License-File: LICENSE +Copyright: Copyright (c) 2009-2010, Hideyuki Tanaka Category: Data Author: Hideyuki Tanaka Maintainer: Hideyuki Tanaka Homepage: http://github.com/msgpack/msgpack Stability: Experimental -Tested-with: GHC == 6.12.3 -Cabal-Version: >= 1.2 +Cabal-Version: >= 1.6 Build-Type: Simple Library @@ -27,7 +27,7 @@ Library data-binary-ieee754 >= 0.4 && < 0.5, deepseq >= 1.1 && <1.2 - Ghc-options: -Wall -O2 + Ghc-options: -Wall Hs-source-dirs: src Exposed-modules: @@ -36,3 +36,7 @@ Library Data.MessagePack.Put Data.MessagePack.Parser Data.MessagePack.Iteratee + +Source-repository head + Type: git + Location: git://github.com/msgpack/msgpack.git From 9f684e7670877fe04d02afe8377e4a6191d74f31 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 09:35:39 +0900 Subject: [PATCH 142/259] Checking in changes prior to tagging of version 0.20. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 7910882..dc3dd5c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.20 + + - first production ready release with PP driver. + 0.16_04 - no feature changes --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 79108820..dc3dd5cf 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.20 + + - first production ready release with PP driver. + 0.16_04 - no feature changes diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b08bac2d..eca24ec6 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.16_04'; +our $VERSION = '0.20'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From f6f675d1e12b2c5994099df3d1af1513b1d83bf2 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 20:27:11 +0900 Subject: [PATCH 143/259] updated benchmark script --- perl/benchmark/deserialize.pl | 9 ++++++--- perl/benchmark/serialize.pl | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index fd21f086..0ddcec93 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -3,18 +3,21 @@ use warnings; use Data::MessagePack; use JSON::XS; use Benchmark ':all'; +use Storable; my $a = [0..2**24]; my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); +my $s = Storable::nfreeze($a); print "-- deserialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; -cmpthese( +timethese( -1 => { - json => sub { JSON::XS::decode_json($j) }, - mp => sub { Data::MessagePack->unpack($m) }, + json => sub { JSON::XS::decode_json($j) }, + mp => sub { Data::MessagePack->unpack($m) }, + storable => sub { Storable::thaw($s) }, } ); diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index 626ae039..b811bfe5 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -2,6 +2,7 @@ use strict; use warnings; use Data::MessagePack; use JSON::XS; +use Storable; use Benchmark ':all'; my $a = [0..2**24]; @@ -9,9 +10,11 @@ my $a = [0..2**24]; print "-- serialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; -cmpthese( +print "Storable: $Storable::VERSION\n"; +timethese( -1 => { json => sub { JSON::XS::encode_json($a) }, + storable => sub { Storable::nfreeze($a) }, mp => sub { Data::MessagePack->pack($a) }, } ); From 5bb8b6f16c426ebfeee0027c1026ab8f610bec05 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 20:38:37 +0900 Subject: [PATCH 144/259] perl: ugpraded benchmarking script. and added result to docs. --- perl/benchmark/deserialize.pl | 12 ++++++--- perl/benchmark/serialize.pl | 15 +++++++---- perl/lib/Data/MessagePack.pm | 50 ++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 0ddcec93..750704e9 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -5,16 +5,22 @@ use JSON::XS; use Benchmark ':all'; use Storable; -my $a = [0..2**24]; +my $a = { + "method" => "handleMessage", + "params" => [ "user1", "we were just talking" ], + "id" => undef, + "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ] +}; my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); -my $s = Storable::nfreeze($a); +my $s = Storable::freeze($a); print "-- deserialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +print "Storable: $Storable::VERSION\n"; timethese( - -1 => { + 1000000 => { json => sub { JSON::XS::decode_json($j) }, mp => sub { Data::MessagePack->unpack($m) }, storable => sub { Storable::thaw($s) }, diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index b811bfe5..c5ab15bc 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -5,17 +5,22 @@ use JSON::XS; use Storable; use Benchmark ':all'; -my $a = [0..2**24]; +my $a = { + "method" => "handleMessage", + "params" => [ "user1", "we were just talking" ], + "id" => undef, + "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ] +}; print "-- serialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; timethese( - -1 => { - json => sub { JSON::XS::encode_json($a) }, - storable => sub { Storable::nfreeze($a) }, - mp => sub { Data::MessagePack->pack($a) }, + 1000000 => { + json => sub { JSON::XS::encode_json($a) }, + storable => sub { Storable::freeze($a) }, + mp => sub { Data::MessagePack->pack($a) }, } ); diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index eca24ec6..fbf305a5 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -41,9 +41,35 @@ Data::MessagePack - MessagePack serialising/deserialising This module converts Perl data structures to MessagePack and vice versa. +=head1 ABOUT MESSAGEPACK FORMAT + MessagePack is a binary-based efficient object serialization format. It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. +=head2 ADVANTAGES + +=over 4 + +=item PORTABILITY + +Messagepack is language independent binary serialize format. + +=item SMALL SIZE + + say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 + say length(Storable::nfreeze({a=>1, b=>2})); # => 21 + say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 + +MessagePack format saves memory than JSON and Storable format. + +=item STREAMING DESERIALIZER + +MessagePack supports streaming deserializer. It is useful for networking such as RPC. + +=back + +If you want to get more informations about messagepack format, please visit to L. + =head1 METHODS =over 4 @@ -68,6 +94,28 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). =back +=head1 SPEED + +This is result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + + -- serialize + JSON::XS: 2.3 + Data::MessagePack: 0.20 + Storable: 2.21 + Benchmark: timing 1000000 iterations of json, mp, storable... + json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) + mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) + storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + + -- deserialize + JSON::XS: 2.3 + Data::MessagePack: 0.20 + Storable: 2.21 + Benchmark: timing 1000000 iterations of json, mp, storable... + json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) + mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) + storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + =head1 AUTHORS Tokuhiro Matsuno @@ -90,5 +138,5 @@ it under the same terms as Perl itself. =head1 SEE ALSO -L +L is official web site for MessagePack format. From b79c1345b92d8cdb6427e0d83d7191262331fd5a Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 20:42:40 +0900 Subject: [PATCH 145/259] use gfx's standard header. --- perl/README | 43 ++++++++++++++++++++++++- perl/perlxs.h | 76 ++++++++++++++++++++++++++++++++++++++++++++ perl/xs-src/pack.c | 11 +------ perl/xs-src/unpack.c | 46 ++++++++++++--------------- 4 files changed, 140 insertions(+), 36 deletions(-) create mode 100644 perl/perlxs.h diff --git a/perl/README b/perl/README index 2ef686c2..d5fc2693 100644 --- a/perl/README +++ b/perl/README @@ -8,10 +8,29 @@ SYNOPSIS DESCRIPTION This module converts Perl data structures to MessagePack and vice versa. +ABOUT MESSAGEPACK FORMAT MessagePack is a binary-based efficient object serialization format. It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. + ADVANTAGES + PORTABILITY + Messagepack is language independent binary serialize format. + + SMALL SIZE + say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 + say length(Storable::nfreeze({a=>1, b=>2})); # => 21 + say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 + + MessagePack format saves memory than JSON and Storable format. + + STREAMING DESERIALIZER + MessagePack supports streaming deserializer. It is useful for + networking such as RPC. + + If you want to get more informations about messagepack format, please + visit to . + METHODS my $packed = Data::MessagePack->pack($data); pack the $data to messagepack format string. @@ -23,6 +42,28 @@ Configuration Variables $Data::MessagePack::PreferInteger Pack the string as int when the value looks like int(EXPERIMENTAL). +SPEED + This is result of benchmark/serialize.pl and benchmark/deserialize.pl on + my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + + -- serialize + JSON::XS: 2.3 + Data::MessagePack: 0.20 + Storable: 2.21 + Benchmark: timing 1000000 iterations of json, mp, storable... + json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) + mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) + storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + + -- deserialize + JSON::XS: 2.3 + Data::MessagePack: 0.20 + Storable: 2.21 + Benchmark: timing 1000000 iterations of json, mp, storable... + json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) + mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) + storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + AUTHORS Tokuhiro Matsuno @@ -40,5 +81,5 @@ LICENSE under the same terms as Perl itself. SEE ALSO - + is official web site for MessagePack format. diff --git a/perl/perlxs.h b/perl/perlxs.h new file mode 100644 index 00000000..441682de --- /dev/null +++ b/perl/perlxs.h @@ -0,0 +1,76 @@ +/* + perlxs.h - Standard XS header file + Copyright (c) Fuji, Goro (gfx) +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#define PERL_NO_GET_CONTEXT /* we want efficiency */ +#include + +#include +#define NO_XSLOCKS /* for exceptions */ +#include + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "ppport.h" + +/* portability stuff not supported by ppport.h yet */ + +#ifndef STATIC_INLINE /* from 5.13.4 */ +# if defined(__GNUC__) || defined(__cplusplus__) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) +# define STATIC_INLINE static inline +# else +# define STATIC_INLINE static +# endif +#endif /* STATIC_INLINE */ + +#ifndef __attribute__format__ +#define __attribute__format__(a,b,c) /* nothing */ +#endif + +#ifndef LIKELY /* they are just a compiler's hint */ +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#ifndef newSVpvs_share +#define newSVpvs_share(s) Perl_newSVpvn_share(aTHX_ STR_WITH_LEN(s), 0U) +#endif + +#ifndef get_cvs +#define get_cvs(name, flags) get_cv(name, flags) +#endif + +#ifndef GvNAME_get +#define GvNAME_get GvNAME +#endif +#ifndef GvNAMELEN_get +#define GvNAMELEN_get GvNAMELEN +#endif + +#ifndef CvGV_set +#define CvGV_set(cv, gv) (CvGV(cv) = (gv)) +#endif + +/* general utility */ + +#if PERL_BCDVERSION >= 0x5008005 +#define LooksLikeNumber(x) looks_like_number(x) +#else +#define LooksLikeNumber(x) (SvPOKp(x) ? looks_like_number(x) : (I32)SvNIOKp(x)) +#endif + +#define newAV_mortal() (AV*)sv_2mortal((SV*)newAV()) +#define newHV_mortal() (HV*)sv_2mortal((SV*)newHV()) + +#define DECL_BOOT(name) EXTERN_C XS(CAT2(boot_, name)) +#define CALL_BOOT(name) STMT_START { \ + PUSHMARK(SP); \ + CALL_FPTR(CAT2(boot_, name))(aTHX_ cv); \ + } STMT_END diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 93b2e2f4..62eb0024 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -2,16 +2,7 @@ * code is written by tokuhirom. * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. */ -#ifdef __cplusplus -extern "C" { -#endif -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" -#include "ppport.h" -#ifdef __cplusplus -}; -#endif +#include "perlxs.h" #include "msgpack/pack_define.h" diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index eb6e0ddb..20a07372 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -2,13 +2,9 @@ extern "C" { #endif -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" -#include "util.h" #define NEED_newRV_noinc #define NEED_sv_2pv_flags -#include "ppport.h" +#include "perlxs.h" #ifdef __cplusplus }; @@ -38,7 +34,7 @@ typedef struct { /* ---------------------------------------------------------------------- */ /* utility functions */ -static INLINE SV * +STATIC_INLINE SV * get_bool (const char *name) { SV * sv = sv_mortalcopy(get_sv( name, 1 )); @@ -60,19 +56,19 @@ static SV* template_data(msgpack_unpack_t* u); static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static INLINE SV* template_callback_root(unpack_user* u) +STATIC_INLINE SV* template_callback_root(unpack_user* u) { return &PL_sv_undef; } -static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) { *o = sv_2mortal(newSVuv(d)); return 0; } -static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) { *o = sv_2mortal(newSVuv(d)); return 0; } -static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) { *o = sv_2mortal(newSVuv(d)); return 0; } -static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) { #if IVSIZE==4 *o = sv_2mortal(newSVnv(d)); @@ -82,47 +78,47 @@ static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) return 0; } -static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) +STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) { *o = sv_2mortal(newSViv((long)d)); return 0; } -static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) +STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) { *o = sv_2mortal(newSViv((long)d)); return 0; } -static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) +STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) { *o = sv_2mortal(newSViv((long)d)); return 0; } -static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) +STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) { *o = sv_2mortal(newSViv(d)); return 0; } -static INLINE int template_callback_float(unpack_user* u, float d, SV** o) +STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o) { *o = sv_2mortal(newSVnv(d)); return 0; } -static INLINE int template_callback_double(unpack_user* u, double d, SV** o) +STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o) { *o = sv_2mortal(newSVnv(d)); return 0; } /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */ -static INLINE int template_callback_nil(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o) { *o = sv_newmortal(); return 0; } -static INLINE int template_callback_true(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_true(unpack_user* u, SV** o) { *o = get_bool("Data::MessagePack::true") ; return 0; } -static INLINE int template_callback_false(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_false(unpack_user* u, SV** o) { *o = get_bool("Data::MessagePack::false") ; return 0; } -static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) +STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) { AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; } -static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) +STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) { av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ -static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) +STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; } -static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } -static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) { *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } /* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; } <= this does not works. */ From ef0a86e7ccc78bf632a3dea4b49fe8507d711151 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 20:45:17 +0900 Subject: [PATCH 146/259] perl: more inline --- perl/xs-src/unpack.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 20a07372..16a52d78 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -131,7 +131,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha #include "msgpack/unpack_template.h" -SV* _msgpack_unpack(SV* data, int limit) { +STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) { msgpack_unpack_t mp; unpack_user u = {0, &PL_sv_undef}; int ret; @@ -194,7 +194,7 @@ XS(xs_unpack) { /* ------------------------------ stream -- */ /* http://twitter.com/frsyuki/status/13249304748 */ -static void _reset(SV* self) { +STATIC_INLINE void _reset(SV* self) { unpack_user u = {0, &PL_sv_undef, 0}; UNPACKER(self, mp); @@ -220,7 +220,7 @@ XS(xs_unpacker_new) { XSRETURN(1); } -static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { +STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { UNPACKER(self, mp); size_t from = off; From 0cd31a4b96d1b8b4084083d3b7ed99b403338e2b Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 21:00:27 +0900 Subject: [PATCH 147/259] perl: inlining utility functions --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 16a52d78..f82fe072 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -22,7 +22,7 @@ typedef struct { struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template ## name + STATIC_INLINE ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name From 0c4f0de13dd9cfaa2f50b48177a0545e258c81b7 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 21:18:45 +0900 Subject: [PATCH 148/259] perl: inlining the small functions --- perl/xs-src/pack.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 62eb0024..e7a7c35b 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -43,7 +43,7 @@ static void need(enc_t *enc, STRLEN len); #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)" -static void need(enc_t *enc, STRLEN len) +STATIC_INLINE void need(enc_t *enc, STRLEN len) { if (enc->cur + len >= enc->end) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); @@ -56,7 +56,7 @@ static void need(enc_t *enc, STRLEN len) static int s_pref_int = 0; -static int pref_int_set(pTHX_ SV* sv, MAGIC* mg) { +STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg) { if (SvTRUE(sv)) { s_pref_int = 1; } else { @@ -85,7 +85,7 @@ void boot_Data__MessagePack_pack(void) { } -static int try_int(enc_t* enc, const char *p, size_t len) { +STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) { int negative = 0; const char* pe = p + len; uint64_t num = 0; @@ -141,7 +141,7 @@ static int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); -static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { +STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); @@ -176,7 +176,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { } } -static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { +STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); From beb22844408b218a9ae7f494f3caa2aefe779a0e Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 21:25:46 +0900 Subject: [PATCH 149/259] perl: added docs for circular reference and blessed object. --- perl/lib/Data/MessagePack.pm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index fbf305a5..0229bcab 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -74,9 +74,13 @@ If you want to get more informations about messagepack format, please visit to L =over 4 -=item my $packed = Data::MessagePack->pack($data); +=item my $packed = Data::MessagePack->pack($data[, $max_depth]); -pack the $data to messagepack format string. +Pack the $data to messagepack format string. + +This method throws exception when nesting perl structure more than $max_depth(default: 512) for detecting circular reference. + +Data::MessagePack->pack() throws exception when encountered blessed object. Because MessagePack is language independent format. =item my $unpacked = Data::MessagePack->unpack($msgpackstr); From 1e6262f24f131b28b7c6d25d0c2464d49450ed08 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 10 Sep 2010 21:27:38 +0900 Subject: [PATCH 150/259] Checking in changes prior to tagging of version 0.21. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index dc3dd5c..5d5a5e2 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.21 + + - doc enhancment + - micro performance tuning. + 0.20 - first production ready release with PP driver. --- perl/Changes | 5 +++++ perl/README | 10 ++++++++-- perl/lib/Data/MessagePack.pm | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/perl/Changes b/perl/Changes index dc3dd5cf..5d5a5e21 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.21 + + - doc enhancment + - micro performance tuning. + 0.20 - first production ready release with PP driver. diff --git a/perl/README b/perl/README index d5fc2693..dec61252 100644 --- a/perl/README +++ b/perl/README @@ -32,8 +32,14 @@ ABOUT MESSAGEPACK FORMAT visit to . METHODS - my $packed = Data::MessagePack->pack($data); - pack the $data to messagepack format string. + my $packed = Data::MessagePack->pack($data[, $max_depth]); + Pack the $data to messagepack format string. + + This method throws exception when nesting perl structure more than + $max_depth(default: 512) for detecting circular reference. + + Data::MessagePack->pack() throws exception when encountered blessed + object. Because MessagePack is language independent format. my $unpacked = Data::MessagePack->unpack($msgpackstr); unpack the $msgpackstr to messagepack format string. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 0229bcab..197d7445 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.20'; +our $VERSION = '0.21'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From a41f7ce3bd106937900576c00108fefd87559174 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 12 Sep 2010 00:09:44 +0900 Subject: [PATCH 151/259] oops. 0.21 breakes ithreads support! --- perl/.gitignore | 1 + perl/Makefile.PL | 2 +- perl/xs-src/pack.c | 6 +++++- perl/xs-src/unpack.c | 43 ++++++++++++++++++++++++------------------- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/perl/.gitignore b/perl/.gitignore index b806d2b3..b64dcdfe 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -12,3 +12,4 @@ unpack.o MANIFEST ppport.h .testenv/ +xshelper.h diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 7958bc6b..7053cf96 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -18,7 +18,7 @@ if ( $] >= 5.008005 and want_xs() ) { my $has_c99 = c99_available(); # msgpack C library requires C99. if ( $has_c99 ) { - use_ppport 3.19; + use_xshelper(); cc_src_paths('xs-src'); if ($ENV{DEBUG}) { cc_append_to_ccflags '-g'; diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index e7a7c35b..0aa32502 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -2,7 +2,7 @@ * code is written by tokuhirom. * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. */ -#include "perlxs.h" +#include "xshelper.h" #include "msgpack/pack_define.h" @@ -45,6 +45,7 @@ static void need(enc_t *enc, STRLEN len); STATIC_INLINE void need(enc_t *enc, STRLEN len) { + dTHX; if (enc->cur + len >= enc->end) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); @@ -79,6 +80,7 @@ MGVTBL pref_int_vtbl = { }; void boot_Data__MessagePack_pack(void) { + dTHX; SV* var = get_sv("Data::MessagePack::PreferInteger", 0); sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0); SvSETMAGIC(var); @@ -142,6 +144,7 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { + dTHX; if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); @@ -178,6 +181,7 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; + dTHX; if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); svt = SvTYPE(sv); diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index f82fe072..c329e99c 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -4,7 +4,7 @@ extern "C" { #define NEED_newRV_noinc #define NEED_sv_2pv_flags -#include "perlxs.h" +#include "xshelper.h" #ifdef __cplusplus }; @@ -36,6 +36,7 @@ typedef struct { STATIC_INLINE SV * get_bool (const char *name) { + dTHX; SV * sv = sv_mortalcopy(get_sv( name, 1 )); SvREADONLY_on(sv); @@ -57,19 +58,20 @@ static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); STATIC_INLINE SV* template_callback_root(unpack_user* u) -{ return &PL_sv_undef; } +{ dTHX; return &PL_sv_undef; } STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) -{ *o = sv_2mortal(newSVuv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) -{ *o = sv_2mortal(newSVuv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) -{ *o = sv_2mortal(newSVuv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) { + dTHX; #if IVSIZE==4 *o = sv_2mortal(newSVnv(d)); #else @@ -79,47 +81,47 @@ STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) } STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) -{ *o = sv_2mortal(newSViv((long)d)); return 0; } +{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) -{ *o = sv_2mortal(newSViv((long)d)); return 0; } +{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) -{ *o = sv_2mortal(newSViv((long)d)); return 0; } +{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) -{ *o = sv_2mortal(newSViv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSViv(d)); return 0; } STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o) -{ *o = sv_2mortal(newSVnv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; } STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o) -{ *o = sv_2mortal(newSVnv(d)); return 0; } +{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; } /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */ STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o) -{ *o = sv_newmortal(); return 0; } +{ dTHX; *o = sv_newmortal(); return 0; } STATIC_INLINE int template_callback_true(unpack_user* u, SV** o) -{ *o = get_bool("Data::MessagePack::true") ; return 0; } +{ dTHX; *o = get_bool("Data::MessagePack::true") ; return 0; } STATIC_INLINE int template_callback_false(unpack_user* u, SV** o) -{ *o = get_bool("Data::MessagePack::false") ; return 0; } +{ dTHX; *o = get_bool("Data::MessagePack::false") ; return 0; } STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) -{ AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; } +{ dTHX; AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; } STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ +{ dTHX; av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) -{ HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; } +{ dTHX; HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; } STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) -{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } +{ dTHX; hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) -{ *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } +{ dTHX; *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } /* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; } <= this does not works. */ #define UNPACKER(from, name) \ @@ -133,6 +135,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) { msgpack_unpack_t mp; + dTHX; unpack_user u = {0, &PL_sv_undef}; int ret; size_t from = 0; @@ -195,6 +198,7 @@ XS(xs_unpack) { /* http://twitter.com/frsyuki/status/13249304748 */ STATIC_INLINE void _reset(SV* self) { + dTHX; unpack_user u = {0, &PL_sv_undef, 0}; UNPACKER(self, mp); @@ -221,6 +225,7 @@ XS(xs_unpacker_new) { } STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { + dTHX; UNPACKER(self, mp); size_t from = off; From 65befb84a0d33d5dd4533cb5aea34cbbeffabfa0 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 12 Sep 2010 00:11:31 +0900 Subject: [PATCH 152/259] Checking in changes prior to tagging of version 0.22. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 5d5a5e2..dd47b98 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.22 + + - fixed issue on ithreads(broken from 0.21) + 0.21 - doc enhancment --- perl/Changes | 4 ++++ perl/MANIFEST.SKIP | 1 + perl/lib/Data/MessagePack.pm | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 5d5a5e21..dd47b988 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.22 + + - fixed issue on ithreads(broken from 0.21) + 0.21 - doc enhancment diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 71a24e5c..372742ca 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -25,3 +25,4 @@ ^Data-MessagePack-[0-9.]+/ ^\.testenv/test_pp.pl ^ppport.h$ +^xshelper.h$ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 197d7445..516e98b2 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.21'; +our $VERSION = '0.22'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 1242ffa4c6e640a9d649028f3f7659c881a61982 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 12 Sep 2010 05:38:15 +0900 Subject: [PATCH 153/259] Checking in changes prior to tagging of version 0.23. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index dd47b98..4120376 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,10 +1,15 @@ +0.23 + + (NO FEATURE CHANGES) + - fixed english docs(hanekomu++) + 0.22 - fixed issue on ithreads(broken from 0.21) 0.21 - - doc enhancment + - doc enhancments - micro performance tuning. 0.20 --- perl/Changes | 7 ++++++- perl/README | 22 +++++++++++++--------- perl/lib/Data/MessagePack.pm | 16 +++++++++------- perl/lib/Data/MessagePack/Unpacker.pod | 8 ++++---- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/perl/Changes b/perl/Changes index dd47b988..41203763 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,10 +1,15 @@ +0.23 + + (NO FEATURE CHANGES) + - fixed english docs(hanekomu++) + 0.22 - fixed issue on ithreads(broken from 0.21) 0.21 - - doc enhancment + - doc enhancments - micro performance tuning. 0.20 diff --git a/perl/README b/perl/README index dec61252..e46323da 100644 --- a/perl/README +++ b/perl/README @@ -22,35 +22,37 @@ ABOUT MESSAGEPACK FORMAT say length(Storable::nfreeze({a=>1, b=>2})); # => 21 say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 - MessagePack format saves memory than JSON and Storable format. + The MessagePack format saves memory than JSON and Storable format. STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for networking such as RPC. - If you want to get more informations about messagepack format, please + If you want to get more information about the MessagePack format, please visit to . METHODS my $packed = Data::MessagePack->pack($data[, $max_depth]); Pack the $data to messagepack format string. - This method throws exception when nesting perl structure more than - $max_depth(default: 512) for detecting circular reference. + This method throws an exception when the perl structure is nested + more than $max_depth levels(default: 512) in order to detect + circular references. - Data::MessagePack->pack() throws exception when encountered blessed - object. Because MessagePack is language independent format. + Data::MessagePack->pack() throws an exception when encountering + blessed object, because MessagePack is language-independent format. my $unpacked = Data::MessagePack->unpack($msgpackstr); - unpack the $msgpackstr to messagepack format string. + unpack the $msgpackstr to a MessagePack format string. Configuration Variables $Data::MessagePack::PreferInteger Pack the string as int when the value looks like int(EXPERIMENTAL). SPEED - This is result of benchmark/serialize.pl and benchmark/deserialize.pl on - my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + This is the result of benchmark/serialize.pl and + benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu + SMP). -- serialize JSON::XS: 2.3 @@ -82,6 +84,8 @@ THANKS TO FURUHASHI Sadayuki + hanekomu + LICENSE This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 516e98b2..3511628c 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.22'; +our $VERSION = '0.23'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; @@ -60,7 +60,7 @@ Messagepack is language independent binary serialize format. say length(Storable::nfreeze({a=>1, b=>2})); # => 21 say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 -MessagePack format saves memory than JSON and Storable format. +The MessagePack format saves memory than JSON and Storable format. =item STREAMING DESERIALIZER @@ -68,7 +68,7 @@ MessagePack supports streaming deserializer. It is useful for networking such as =back -If you want to get more informations about messagepack format, please visit to L. +If you want to get more information about the MessagePack format, please visit to L. =head1 METHODS @@ -78,13 +78,13 @@ If you want to get more informations about messagepack format, please visit to L Pack the $data to messagepack format string. -This method throws exception when nesting perl structure more than $max_depth(default: 512) for detecting circular reference. +This method throws an exception when the perl structure is nested more than $max_depth levels(default: 512) in order to detect circular references. -Data::MessagePack->pack() throws exception when encountered blessed object. Because MessagePack is language independent format. +Data::MessagePack->pack() throws an exception when encountering blessed object, because MessagePack is language-independent format. =item my $unpacked = Data::MessagePack->unpack($msgpackstr); -unpack the $msgpackstr to messagepack format string. +unpack the $msgpackstr to a MessagePack format string. =back @@ -100,7 +100,7 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). =head1 SPEED -This is result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). +This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). -- serialize JSON::XS: 2.3 @@ -134,6 +134,8 @@ Dan Kogai FURUHASHI Sadayuki +hanekomu + =head1 LICENSE This library is free software; you can redistribute it and/or modify diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod index c24eaf1d..2bc4549c 100644 --- a/perl/lib/Data/MessagePack/Unpacker.pod +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -14,7 +14,7 @@ Data::MessagePack::Unpacker - messagepack streaming deserializer =head1 DESCRIPTION -This is an streaming deserializer for messagepack. +This is a streaming deserializer for messagepack. =head1 METHODS @@ -22,7 +22,7 @@ This is an streaming deserializer for messagepack. =item my $up = Data::MessagePack::Unpacker->new() -create new instance of stream deserializer. +creates a new instance of stream deserializer. =item my $ret = $up->execute($data, $offset); @@ -39,11 +39,11 @@ is this deserializer finished? =item my $data = $up->data(); -returns deserialized object. +returns the deserialized object. =item $up->reset(); -reset the stream deserializer, without memory zone. +resets the stream deserializer, without memory zone. =back From a9566b31be76ebc8aaf9149c1a09d3720c3f2ec9 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Sun, 12 Sep 2010 22:21:33 +0900 Subject: [PATCH 154/259] add annotation utilities for Java --- .../org/msgpack/tmp/DynamicCodeGenerator.java | 206 -------- .../src/main/java/org/msgpack/tmp/Image1.java | 47 -- .../src/main/java/org/msgpack/tmp/Image2.java | 85 --- .../src/main/java/org/msgpack/tmp/Image3.java | 38 -- .../msgpack/tmp/ImagePackUnpackPerfTest.java | 134 ----- .../util/annotation/MessagePackOptional.java | 12 + .../util/annotation/MessagePackRequired.java | 12 + .../annotation/MessagePackUnpackable.java | 11 + .../util/annotation/PackUnpackUtil.java | 498 ++++++++++++++++++ .../annotation/PackUnpackUtilException.java | 12 + 10 files changed, 545 insertions(+), 510 deletions(-) delete mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java delete mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java deleted file mode 100644 index 78238b4f..00000000 --- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java +++ /dev/null @@ -1,206 +0,0 @@ -package org.msgpack.tmp; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; -import java.util.HashMap; - -import javassist.ClassPool; -import javassist.CtClass; -import javassist.CtConstructor; -import javassist.CtMethod; -import javassist.CtNewConstructor; -import javassist.CtNewMethod; - -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class DynamicCodeGenerator { - - private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced"; - - private static final String SPACE = " "; - - private static final String MODIFIER_PUBLIC = "public"; - - private static final String METHOD_NAME_MESSAGEPACK = "messagePack"; - - private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack"; - - private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); - - private static final String VOID_TYPE_NAME = "void"; - - private static final String PACKER_OBJECT_NAME = "pk"; - - private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class - .getName(); - - private static final String UNPACKER_OBJECT_NAME = "pk"; - - private HashMap> classMap; - - private HashMap objectCacheMap; - - private ClassPool pool; - - public DynamicCodeGenerator() { - classMap = new HashMap>(); - objectCacheMap = new HashMap(); - pool = ClassPool.getDefault(); - } - - public Object newEnhancedInstance(Class targetClass) throws Exception { - String targetClassName = targetClass.getName(); - // Class enhancedClass = classMap.get(targetClassName); - Object enhancedObject = objectCacheMap.get(targetClassName); - // if (enhancedClass == null) { - if (enhancedObject == null) { - CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); - // System.out.println("enhanced class name: " - // + enhancedCtClass.getName()); - addSuperclass(enhancedCtClass, targetClassName); - addConstructor(enhancedCtClass); - createMessagePackMethod(enhancedCtClass, targetClass); - createMessageUnpackMethod(enhancedCtClass, targetClass); - Class enhancedClass = loadEnhancedClass(enhancedCtClass); - // classMap.put(targetClassName, enhancedClass); - enhancedObject = enhancedClass.newInstance(); - objectCacheMap.put(targetClassName, enhancedObject); - } - // return newEnhancedInstance0(enhancedClass); - return enhancedObject; - } - - private CtClass createEnhancedCtClass(final String targetClassName) - throws Exception { - return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX); - } - - private void addSuperclass(CtClass enhancedCtClass, - final String targetClassName) throws Exception { - CtClass targetCtClass = pool.get(targetClassName); - enhancedCtClass.setSuperclass(targetCtClass); - } - - private void addConstructor(CtClass enhancedCtClass) throws Exception { - CtConstructor newCtConstructor = CtNewConstructor - .defaultConstructor(enhancedCtClass); - enhancedCtClass.addConstructor(newCtConstructor); - } - - private void createMessagePackMethod(CtClass enhancedCtClass, - Class targetClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append( - SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append( - PACKER_CLASS_TYPE_NAME).append(SPACE) - .append(PACKER_OBJECT_NAME).append(")").append(SPACE).append( - "throws").append(SPACE).append("java.io.IOException") - .append(SPACE).append("{"); - Field[] fields = targetClass.getFields(); - sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append( - "(").append(fields.length).append(")").append(";"); - for (Field field : fields) { - insertCodeOfMessagePack(field, sb); - } - sb.append("}"); - // System.out.println("messagePack method: " + sb.toString()); - CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); - enhancedCtClass.addMethod(newCtMethod); - } - - private void insertCodeOfMessagePack(Field field, StringBuilder sb) { - Class type = field.getType(); - if (type.equals(int.class)) { - sb.append(PACKER_OBJECT_NAME).append(".").append("pack") - .append("(").append(field.getName()).append(")") - .append(";"); - } else if (type.equals(String.class)) { - sb.append(PACKER_OBJECT_NAME).append(".").append("pack") - .append("(").append(field.getName()).append(")") - .append(";"); - } else { - throw new UnsupportedOperationException(); - } - } - - private void createMessageUnpackMethod(CtClass enhancedCtClass, - Class targetClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb - .append(MODIFIER_PUBLIC) - .append(SPACE) - .append(VOID_TYPE_NAME) - .append(SPACE) - .append(METHOD_NAME_MESSAGEUNPACK) - .append("(") - .append(UNPACKER_CLASS_TYPE_NAME) - .append(SPACE) - .append(UNPACKER_OBJECT_NAME) - .append(")") - .append(SPACE) - .append("throws") - .append(SPACE) - .append("org.msgpack.MessageTypeException, java.io.IOException") - .append(SPACE).append("{"); - Field[] fields = targetClass.getFields(); - sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()") - .append(";"); - // TODO - for (Field field : fields) { - insertCodeOfMessageUnpack(field, sb); - } - sb.append("}"); - // System.out.println("messageUnpack method: " + sb.toString()); - CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); - enhancedCtClass.addMethod(newCtMethod); - } - - private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) { - Class type = field.getType(); - if (type.equals(int.class)) { - sb.append(field.getName()).append(SPACE).append("=").append(SPACE) - .append(PACKER_OBJECT_NAME).append(".").append("unpackInt") - .append("(").append(")").append(";"); - } else if (type.equals(String.class)) { - sb.append(field.getName()).append(SPACE).append("=").append(SPACE) - .append(PACKER_OBJECT_NAME).append(".").append( - "unpackString").append("(").append(")").append(";"); - } else { - throw new UnsupportedOperationException(); - } - } - - private Class loadEnhancedClass(CtClass enhancedCtClass) - throws Exception { - return enhancedCtClass.toClass(null, null); - } - - private Object newEnhancedInstance0(Class enhancedClass) { - try { - return enhancedClass.newInstance(); - } catch (Exception e) { - throw new UnsupportedOperationException(); - } - } - - public static void main(String[] args) throws Exception { - DynamicCodeGenerator gen = new DynamicCodeGenerator(); - Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - -// System.out.println(dst.equals(src)); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java deleted file mode 100644 index f1819ed6..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image1.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.msgpack.tmp; - -import org.msgpack.*; -import java.io.*; - -public class Image1 implements MessagePackable, MessageUnpackable { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - pk.packArray(5); - pk.pack(uri); - pk.pack(title); - pk.pack(width); - pk.pack(height); - pk.pack(size); - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - int length = pac.unpackArray(); - if (length != 5) { - throw new MessageTypeException(); - } - uri = pac.unpackString(); - title = pac.unpackString(); - width = pac.unpackInt(); - height = pac.unpackInt(); - size = pac.unpackInt(); - } - - public boolean equals(Image1 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image1.class) { - return false; - } - return equals((Image1) obj); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java deleted file mode 100644 index 9c357bf3..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image2.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.msgpack.tmp; - -import java.io.IOException; -import java.lang.reflect.Field; - -import org.msgpack.MessageTypeException; -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class Image2 { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - messagePackWithReflection(pk); - } - - public void messagePackWithReflection(Packer pk) throws IOException { - Class cl = this.getClass(); - Field[] fields = cl.getFields(); - pk.packArray(fields.length); - for (Field field : fields) { - try { - Object obj = field.get(this); - pk.pack(obj); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - } - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - messageUnpackWithReflection(pac); - } - - public void messageUnpackWithReflection(Unpacker pac) throws IOException, - MessageTypeException { - Class cl = this.getClass(); - Field[] fields = cl.getFields(); - int length = pac.unpackArray(); - if (length != fields.length) { - throw new MessageTypeException(); - } - for (Field field : fields) { - try { - field.set(this, unpack(pac, field.getType())); - } catch (IOException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - } - } - - public static Object unpack(Unpacker pk, Class cl) throws IOException { - if (cl == int.class) { - return pk.unpackInt(); - } else if (cl == String.class) { - return pk.unpackString(); - } else { - throw new UnsupportedOperationException(); - } - } - - public boolean equals(Image2 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image2.class) { - return false; - } - return equals((Image2) obj); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java deleted file mode 100644 index a28ff53f..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image3.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.msgpack.tmp; - -import java.io.IOException; -import org.msgpack.MessagePackable; -import org.msgpack.MessageTypeException; -import org.msgpack.MessageUnpackable; -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class Image3 implements MessagePackable, MessageUnpackable { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - // empty - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - // empty - } - - public boolean equals(Image3 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image3.class) { - return false; - } - return equals((Image3) obj); - } -} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java deleted file mode 100644 index 1f27c1b1..00000000 --- a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.msgpack.tmp; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; - -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class ImagePackUnpackPerfTest { - - private static int BIG_LOOP = 5; - - private static int SMALL_LOOP = 50000; - - private static DynamicCodeGenerator gen; - - private static void doGc() { - try { - Thread.sleep(50L); - } catch (InterruptedException ie) { - System.err - .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); - } - System.gc(); - try { // longer sleep afterwards (not needed by GC, but may help with - // scheduling) - Thread.sleep(200L); - } catch (InterruptedException ie) { - System.err - .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); - } - } - - public void doIt(int versionID) { - try { - doIt0(versionID); - } catch (Exception e) { - e.printStackTrace(); - } - try { - for (int j = 0; j < BIG_LOOP; ++j) { - long t = System.currentTimeMillis(); - doIt0(versionID); - t = System.currentTimeMillis() - t; - System.out.println("time: " + t); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void doIt0(int versionID) throws Exception { - for (int i = 0; i < SMALL_LOOP; ++i) { - switch (versionID) { - case 0: - doCurrentVersion(); - break; - case 1: - doWithReflection(); - break; - case 2: - doWithDynamicCodeGeneration(); - break; - default: - throw new RuntimeException(); - } - } - } - - public void doCurrentVersion() throws Exception { - Image1 src = new Image1(); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image1 dst = new Image1(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public void doWithReflection() throws Exception { - Image2 src = new Image2(); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image2 dst = new Image2(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public void doWithDynamicCodeGeneration() throws Exception { - Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public static void main(String[] args) { - ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest(); - - doGc(); - System.out.println("test current version"); - test.doIt(0); - - doGc(); - System.out.println("test with reflection"); - test.doIt(1); - - doGc(); - System.out.println("test with dynamic codegen"); - gen = new DynamicCodeGenerator(); - test.doIt(2); - } -} diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java new file mode 100644 index 00000000..a565292d --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackOptional { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java new file mode 100644 index 00000000..03e50cf3 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackRequired { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java new file mode 100644 index 00000000..473b5412 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java @@ -0,0 +1,11 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackUnpackable { +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java new file mode 100644 index 00000000..37b9e0d5 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -0,0 +1,498 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtField; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; +import javassist.NotFoundException; + +import org.msgpack.MessageConvertable; +import org.msgpack.MessagePackable; +import org.msgpack.MessageTypeException; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class PackUnpackUtil { + static class Constants { + static final String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced"; + + static final String KEYWORD_MODIFIER_PUBLIC = "public"; + + static final String KEYWORD_THROWS = "throws"; + + static final String TYPE_NAME_VOID = void.class.getName(); + + static final String TYPE_NAME_BOOLEAN = boolean.class.getName(); + + static final String TYPE_NAME_BYTE = byte.class.getName(); + + static final String TYPE_NAME_DOUBLE = double.class.getName(); + + static final String TYPE_NAME_FLOAT = float.class.getName(); + + static final String TYPE_NAME_INT = int.class.getName(); + + static final String TYPE_NAME_LONG = long.class.getName(); + + static final String TYPE_NAME_SHORT = short.class.getName(); + + static final String TYPE_NAME_OBJECT = Object.class.getName(); + + static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName(); + + static final String TYPE_NAME_STRING = String.class.getName(); + + static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName(); + + static final String TYPE_NAME_BYTE2 = Byte.class.getName(); + + static final String TYPE_NAME_DOUBLE2 = Double.class.getName(); + + static final String TYPE_NAME_FLOAT2 = Float.class.getName(); + + static final String TYPE_NAME_INT2 = Integer.class.getName(); + + static final String TYPE_NAME_LONG2 = Long.class.getName(); + + static final String TYPE_NAME_SHORT2 = Short.class.getName(); + + static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + + static final String TYPE_NAME_LIST = List.class.getName(); + + static final String TYPE_NAME_MAP = Map.class.getName(); + + static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); + + static final String TYPE_NAME_PACKER = Packer.class.getName(); + + static final String TYPE_NAME_UNPACKER = Unpacker.class.getName(); + + static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class + .getName(); + + static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class + .getName(); + + static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class + .getName(); + + static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class + .getName(); + + static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class + .getName(); + + static final String CHAR_NAME_SPACE = " "; + + static final String CHAR_NAME_COMMA = ","; + + static final String CHAR_NAME_EQUAL = "="; + + static final String CHAR_NAME_RIGHT_PARENTHESIS = ")"; + + static final String CHAR_NAME_LEFT_PARENTHESIS = "("; + + static final String CHAR_NAME_RIGHT_CURLY_BRACHET = "}"; + + static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{"; + + static final String CHAR_NAME_DOT = "."; + + static final String CHAR_NAME_SEMICOLON = ";"; + + static final String VARIABLE_NAME_PK = "pk"; + + static final String VARIABLE_NAME_OBJ = "obj"; + + static final String METHOD_NAME_MSGPACK = "messagePack"; + + static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; + + static final String METHOD_NAME_MSGCONVERT = "messageConvert"; + + static final String METHOD_NAME_PACK = "pack"; + + static final String METHOD_NAME_PACKARRAY = "packArray"; + + static final String METHOD_NAME_UNPACK = "unpack"; + + static final String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean"; + + static final String METHOD_NAME_UNPACKBYTE = "unpackByte"; + + static final String METHOD_NAME_UNPACKDOUBLE = "unpackDouble"; + + static final String METHOD_NAME_UNPACKFLOAT = "unpackFloat"; + + static final String METHOD_NAME_UNPACKINT = "unpackInt"; + + static final String METHOD_NAME_UNPACKLONG = "unpackLong"; + + static final String METHOD_NAME_UNPACKSHORT = "unpackShort"; + + static final String METHOD_NAME_UNPACKSTRING = "unpackString"; + + static final String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger"; + + static final String METHOD_NAME_UNPACKOBJECT = "unpackObject"; + + static final String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray"; + + static final String METHOD_NAME_UNPACKARRAY = "unpackArray"; + + static final String METHOD_NAME_UNPACKMAP = "unpackMap"; + } + + public static class Enhancer { + + private ConcurrentHashMap> classCache; + + private ClassPool pool; + + protected Enhancer() { + classCache = new ConcurrentHashMap>(); + pool = ClassPool.getDefault(); + } + + protected Class getCache(String origName) { + return classCache.get(origName); + } + + protected void setCache(String origName, Class enhClass) { + classCache.putIfAbsent(origName, enhClass); + } + + protected Class generate(Class origClass) + throws NotFoundException, CannotCompileException { + String origName = origClass.getName(); + String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; + CtClass origCtClass = pool.get(origName); + checkPackUnpackAnnotation(origCtClass); + CtClass enhCtClass = pool.makeClass(enhName); + setSuperclass(enhCtClass, origCtClass); + setInterfaces(enhCtClass); + addConstructor(enhCtClass); + addMessagePackMethod(enhCtClass, origCtClass); + addMessageUnpackMethod(enhCtClass, origCtClass); + addMessageConvertMethod(enhCtClass, origCtClass); + return createClass(enhCtClass); + } + + private void checkPackUnpackAnnotation(CtClass origCtClass) { + try { + Object[] objs = origCtClass.getAnnotations(); + for (Object obj : objs) { + if (obj instanceof MessagePackUnpackable) { + return; + } + } + throw new PackUnpackUtilException( + "Not annotated with this class: " + + origCtClass.getName()); + } catch (ClassNotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + } + + private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) + throws CannotCompileException { + enhCtClass.setSuperclass(origCtClass); + } + + private void setInterfaces(CtClass enhCtClass) throws NotFoundException { + CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE); + enhCtClass.addInterface(pacCtClass); + CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE); + enhCtClass.addInterface(unpacCtClass); + CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE); + enhCtClass.addInterface(convCtClass); + } + + private void addConstructor(CtClass enhCtClass) + throws CannotCompileException { + CtConstructor newCtCons = CtNewConstructor + .defaultConstructor(enhCtClass); + enhCtClass.addConstructor(newCtCons); + } + + private void addMessagePackMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException, + NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGPACK).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_PACKER).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_IOEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + CtField[] fields = origCtClass.getDeclaredFields(); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append( + Constants.METHOD_NAME_PACKARRAY).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length) + .append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + for (CtField field : fields) { + insertCodeOfMessagePack(field, sb); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messagePack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessagePack(CtField field, StringBuilder sb) + throws NotFoundException { + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK) + .append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + field.getName()).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + } + + private void addMessageUnpackMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException, + NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGUNPACK).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_UNPACKER).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( + Constants.CHAR_NAME_COMMA).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_IOEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + CtField[] fields = origCtClass.getFields(); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append( + Constants.METHOD_NAME_UNPACKARRAY).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + for (CtField field : fields) { + insertCodeOfMessageUnpack(field, sb); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messageUnpack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) + throws NotFoundException { + CtClass type = field.getType(); + sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) + .append(Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT); + if (type.equals(CtClass.booleanType)) { // boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(CtClass.byteType)) { // byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(CtClass.doubleType)) { // double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(CtClass.floatType)) { // float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(CtClass.intType)) { // int + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(CtClass.longType)) { // long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(CtClass.shortType)) { // short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else if (type + .equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger + sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER); + } else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String + sb.append(Constants.METHOD_NAME_UNPACKSTRING); + } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[] + sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY); + } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map + sb.append(Constants.METHOD_NAME_UNPACKMAP); + } else if (type.subtypeOf(pool + .get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable + sb.append(Constants.METHOD_NAME_UNPACKOBJECT); + } else { + throw new NotFoundException("unknown type: " + + type.getName()); + } + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + } + + private void addMessageConvertMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGCONVERT).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_OBJECT).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_OBJ).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + // TODO + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messageConvert method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private Class createClass(CtClass enhCtClass) + throws CannotCompileException { + return enhCtClass.toClass(null, null); + } + } + + private static Enhancer enhancer; + + public static Class getEnhancedClass(Class origClass) { + if (enhancer == null) { + enhancer = new Enhancer(); + } + + String origName = origClass.getName(); + Class enhClass = enhancer.getCache(origName); + if (enhClass == null) { + // generate a class object related to the original class + try { + enhClass = enhancer.generate(origClass); + } catch (NotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (CannotCompileException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + // set the generated class to the cache + enhancer.setCache(origName, enhClass); + } + return enhClass; + } + + public static Object newEnhancedInstance(Class origClass) { + try { + Class enhClass = getEnhancedClass(origClass); + // create a new object of the generated class + return enhClass.newInstance(); + } catch (InstantiationException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + } + + public static void main(final String[] args) throws Exception { + @MessagePackUnpackable + class Image { + public String uri = ""; + + public String title = ""; + + public int width = 0; + + public int height = 0; + + public int size = 0; + + public boolean equals(Image obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image.class) { + return false; + } + return equals((Image) obj); + } + } + + PackUnpackUtil.getEnhancedClass(Image.class); + Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + Image dst = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + System.out.println(src.equals(dst)); + } +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java new file mode 100644 index 00000000..df3af058 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +public class PackUnpackUtilException extends RuntimeException { + + public PackUnpackUtilException(String reason) { + super(reason); + } + + public PackUnpackUtilException(String reason, Throwable t) { + super(reason, t); + } +} From 599b200ca5dda7957b47fb5156199a75a50a57ae Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Sun, 12 Sep 2010 23:04:23 +0900 Subject: [PATCH 155/259] change the version of javassist 3.12.1.GA --- java/pom.xml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 6591a727..70da6a6f 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,11 +29,11 @@ 4.8.1 test - - jboss + javassist javassist - 3.7.ga + 3.12.1.GA + compile @@ -107,11 +107,12 @@ MessagePack Maven2 Repository http://msgpack.org/maven2 - - repo2.maven.org - repo2.maven.org Maven2 Repository - http://repo2.maven.org/maven2 + repository.jboss.org + https://repository.jboss.org/nexus/content/groups/public/ + + false + From 95b820305ae55823ee5b64476acfa5c2a7f44beb Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 13:34:11 +0900 Subject: [PATCH 156/259] add a new test program for annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 116 +++++++- .../annotation/TestMessagePackUnpackable.java | 280 ++++++++++++++++++ 2 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 37b9e0d5..5efef792 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -3,6 +3,7 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.reflect.Modifier; import java.math.BigInteger; import java.util.List; import java.util.Map; @@ -124,6 +125,8 @@ public class PackUnpackUtil { static final String VARIABLE_NAME_OBJ = "obj"; + static final String METHOD_NAME_VALUEOF = "valueOf"; + static final String METHOD_NAME_MSGPACK = "messagePack"; static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; @@ -187,7 +190,8 @@ public class PackUnpackUtil { String origName = origClass.getName(); String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; CtClass origCtClass = pool.get(origName); - checkPackUnpackAnnotation(origCtClass); + checkClassValidation(origCtClass); + checkDefaultConstructorValidation(origCtClass); CtClass enhCtClass = pool.makeClass(enhName); setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); @@ -198,6 +202,28 @@ public class PackUnpackUtil { return createClass(enhCtClass); } + private void checkClassValidation(CtClass origCtClass) { + // not public, abstract, final + int mod = origCtClass.getModifiers(); + if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod) + || Modifier.isFinal(mod)) { + throwClassValidationException(origCtClass); + } + // interface, enum + if (origCtClass.isInterface() || origCtClass.isEnum()) { + throwClassValidationException(origCtClass); + } + // annotation + checkPackUnpackAnnotation(origCtClass); + } + + private static void throwClassValidationException(CtClass origCtClass) { + throw new PackUnpackUtilException( + "it must be a public class and have @" + + MessagePackUnpackable.class.getName() + ": " + + origCtClass.getName()); + } + private void checkPackUnpackAnnotation(CtClass origCtClass) { try { Object[] objs = origCtClass.getAnnotations(); @@ -206,14 +232,32 @@ public class PackUnpackUtil { return; } } - throw new PackUnpackUtilException( - "Not annotated with this class: " - + origCtClass.getName()); + throwClassValidationException(origCtClass); } catch (ClassNotFoundException e) { throw new PackUnpackUtilException(e.getMessage(), e); } } + private void checkDefaultConstructorValidation(CtClass origCtClass) { + CtConstructor cons = null; + try { + cons = origCtClass.getDeclaredConstructor(new CtClass[0]); + } catch (NotFoundException e) { + throwConstructoValidationException(origCtClass); + } + int mod = cons.getModifiers(); + if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) { + throwConstructoValidationException(origCtClass); + } + } + + private static void throwConstructoValidationException( + CtClass origCtClass) { + throw new PackUnpackUtilException( + "it must have a public zero-argument constructor: " + + origCtClass.getName()); + } + private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) throws CannotCompileException { enhCtClass.setSuperclass(origCtClass); @@ -328,9 +372,43 @@ public class PackUnpackUtil { CtClass type = field.getType(); sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT); + Constants.CHAR_NAME_SPACE); + insertValueOfMethodAndLeftParenthesis(sb, type); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT); + insertUnpackMethod(sb, type); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS); + insertValueOfMethodAndRightParenthesis(sb, type); + sb.append(Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + + } + + private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + ) { + sb.append(type.getName()).append(Constants.CHAR_NAME_DOT) + .append(Constants.METHOD_NAME_VALUEOF).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS); + } else { + return; + } + } + } + + private void insertUnpackMethod(StringBuilder sb, CtClass type) + throws NotFoundException { if (type.equals(CtClass.booleanType)) { // boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); } else if (type.equals(CtClass.byteType)) { // byte @@ -379,10 +457,26 @@ public class PackUnpackUtil { + type.getName()); } } - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); + } + + private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + ) { + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + } else { + return; + } + } } private void addMessageConvertMethod(CtClass enhCtClass, diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java new file mode 100644 index 00000000..1ec7d64e --- /dev/null +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -0,0 +1,280 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class TestMessagePackUnpackable extends TestCase { + + @Test + public void testGeneralPrimitiveTypeFieldsClass() throws Exception { + GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + src.f0 = (byte) 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f6 = false; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + } + + @MessagePackUnpackable + public static class GeneralPrimitiveTypeFieldsClass { + public byte f0; + public short f1; + public int f2; + public long f3; + public float f4; + public double f5; + public boolean f6; + + public GeneralPrimitiveTypeFieldsClass() { + } + } + + @Test + public void testGeneralReferenceTypeFieldsClass() throws Exception { + GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + src.f0 = 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = (long) 3; + src.f4 = (float) 4; + src.f5 = (double) 5; + src.f6 = false; + src.f7 = new BigInteger("7"); + src.f8 = "8"; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + assertEquals(src.f7, dst.f7); + assertEquals(src.f8, dst.f8); + } + + @MessagePackUnpackable + public static class GeneralReferenceTypeFieldsClass { + public Byte f0; + public Short f1; + public Integer f2; + public Long f3; + public Float f4; + public Double f5; + public Boolean f6; + public BigInteger f7; + public String f8; + + public GeneralReferenceTypeFieldsClass() { + } + } + + @Test + public void testPublicDefaultConstructorClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public static class NoDefaultConstructorClass { + public NoDefaultConstructorClass(int i) { + } + } + + @MessagePackUnpackable + public static class PrivateDefaultConstructorClass { + private PrivateDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class ProtectedDefaultConstructorClass { + protected ProtectedDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class PackageDefaultConstructorClass { + PackageDefaultConstructorClass() { + } + } + + @Test + public void testPublicModifierClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + private static class PrivateModifierClass { + } + + @MessagePackUnpackable + protected static class ProtectedModifierClass { + protected ProtectedModifierClass() { + } + } + + @MessagePackUnpackable + static class PackageModifierClass { + } + + @Test + public void testFinalAndAbstractModifierClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public final static class FinalModifierClass { + } + + @MessagePackUnpackable + public abstract static class AbstractModifierClass { + } + + @Test + public void testInterfaceAndEnum() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(SampleInterface.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(SampleEnum.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public interface SampleInterface { + } + + @MessagePackUnpackable + public enum SampleEnum { + } + + @Test + public void testFinalFieldClass() throws Exception { + + } + + @Test + public void testPrivateFieldClass() throws Exception { + + } + + @Test + public void testProtectedFieldClass() throws Exception { + + } + + @Test + public void testNonModifierFieldClass() throws Exception { + + } + + @Test + public void testNestedAnnotatedFieldClass() throws Exception { + + } + + @Test + public void testSuperClass() throws Exception { + + } +} From c7f8b94ccdde64512aced14930ebdc0e14825e66 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 22:26:04 +0900 Subject: [PATCH 157/259] fixed several bugs of a verify error within annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 170 ++++++++++++++---- .../annotation/TestMessagePackUnpackable.java | 160 ++++++++++++++--- 2 files changed, 270 insertions(+), 60 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 5efef792..a58b4bd5 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -5,6 +5,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Modifier; import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -196,9 +197,10 @@ public class PackUnpackUtil { setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); addConstructor(enhCtClass); - addMessagePackMethod(enhCtClass, origCtClass); - addMessageUnpackMethod(enhCtClass, origCtClass); - addMessageConvertMethod(enhCtClass, origCtClass); + CtField[] fields = getDeclaredFields(origCtClass); + addMessagePackMethod(enhCtClass, origCtClass, fields); + addMessageUnpackMethod(enhCtClass, origCtClass, fields); + addMessageConvertMethod(enhCtClass, origCtClass, fields); return createClass(enhCtClass); } @@ -279,9 +281,54 @@ public class PackUnpackUtil { enhCtClass.addConstructor(newCtCons); } + private CtField[] getDeclaredFields(CtClass origCtClass) { + ArrayList allFields = new ArrayList(); + try { + CtClass nextCtClass = origCtClass; + while (!nextCtClass + .equals(pool.get(Constants.TYPE_NAME_OBJECT))) { + CtField[] fields = nextCtClass.getDeclaredFields(); + for (CtField field : fields) { + try { + checkFieldValidation(field, allFields); + allFields.add(field); + } catch (PackUnpackUtilException e) { // ignore + } + } + nextCtClass = nextCtClass.getSuperclass(); + } + + } catch (NotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + return allFields.toArray(new CtField[0]); + } + + private void checkFieldValidation(CtField field, + ArrayList allFields) { + // check modifiers (public or protected) + int mod = field.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isStatic(mod) || Modifier.isFinal(mod) + || Modifier.isTransient(mod)) { + throwFieldValidationException(field); + } + // check same name + for (CtField f : allFields) { + if (f.getName().equals(field.getName())) { + throwFieldValidationException(field); + } + } + } + + private static void throwFieldValidationException(CtField field) { + throw new PackUnpackUtilException("it must be a public field: " + + field.getName()); + } + private void addMessagePackMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException, - NotFoundException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -299,7 +346,6 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE).append( Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( Constants.CHAR_NAME_SPACE); - CtField[] fields = origCtClass.getDeclaredFields(); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT).append( Constants.METHOD_NAME_PACKARRAY).append( @@ -328,8 +374,8 @@ public class PackUnpackUtil { } private void addMessageUnpackMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException, - NotFoundException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -350,7 +396,6 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE).append( Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( Constants.CHAR_NAME_SPACE); - CtField[] fields = origCtClass.getFields(); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT).append( Constants.METHOD_NAME_UNPACKARRAY).append( @@ -370,21 +415,53 @@ public class PackUnpackUtil { private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) throws NotFoundException { CtClass type = field.getType(); - sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) - .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); + insertRightVariable(sb, field, type); insertValueOfMethodAndLeftParenthesis(sb, type); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT); insertUnpackMethod(sb, type); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertUnpackMethodArgumenet(sb, field, type); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); insertValueOfMethodAndRightParenthesis(sb, type); sb.append(Constants.CHAR_NAME_SEMICOLON).append( Constants.CHAR_NAME_SPACE); } + private void insertRightVariable(StringBuilder sb, CtField field, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) + .append(Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE); + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + || type + .equals(pool + .get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger + || type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String + || type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[] + || type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List + || type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map + ) { + sb.append(field.getName()) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE); + } else { // MessageUnpackable + return; + } + } + + } + private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, CtClass type) throws NotFoundException { if (type.isPrimitive()) { // primitive type @@ -424,6 +501,7 @@ public class PackUnpackUtil { } else if (type.equals(CtClass.shortType)) { // short sb.append(Constants.METHOD_NAME_UNPACKSHORT); } else { // reference type + Class c = null; if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte @@ -450,8 +528,9 @@ public class PackUnpackUtil { } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map sb.append(Constants.METHOD_NAME_UNPACKMAP); } else if (type.subtypeOf(pool - .get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable - sb.append(Constants.METHOD_NAME_UNPACKOBJECT); + .get(Constants.TYPE_NAME_MSGUNPACKABLE)) + || ((c = getCache(type.getName())) != null)) { // MessageUnpackable + sb.append(Constants.METHOD_NAME_UNPACK); } else { throw new NotFoundException("unknown type: " + type.getName()); @@ -459,6 +538,22 @@ public class PackUnpackUtil { } } + private void insertUnpackMethodArgumenet(StringBuilder sb, + CtField field, CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + Class c = null; + if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE)) + || ((c = getCache(type.getName())) != null)) { + sb.append("(org.msgpack.MessageUnpackable)"); + sb.append(field.getName()); + } else { + return; + } + } + } + private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, CtClass type) throws NotFoundException { if (type.isPrimitive()) { // primitive type @@ -480,7 +575,8 @@ public class PackUnpackUtil { } private void addMessageConvertMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -547,33 +643,33 @@ public class PackUnpackUtil { } } - public static void main(final String[] args) throws Exception { - @MessagePackUnpackable - class Image { - public String uri = ""; + @MessagePackUnpackable + public static class Image { + public String uri = ""; - public String title = ""; + public String title = ""; - public int width = 0; + public int width = 0; - public int height = 0; + public int height = 0; - public int size = 0; + public int size = 0; - public boolean equals(Image obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image.class) { - return false; - } - return equals((Image) obj); - } + public boolean equals(Image obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; } + public boolean equals(Object obj) { + if (obj.getClass() != Image.class) { + return false; + } + return equals((Image) obj); + } + } + + public static void main(final String[] args) throws Exception { PackUnpackUtil.getEnhancedClass(Image.class); Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); src.title = "msgpack"; diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 1ec7d64e..4f6e74ef 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -3,6 +3,8 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.math.BigInteger; +import java.util.List; +import java.util.Map; import junit.framework.TestCase; @@ -14,7 +16,7 @@ import org.msgpack.Unpacker; public class TestMessagePackUnpackable extends TestCase { @Test - public void testGeneralPrimitiveTypeFieldsClass() throws Exception { + public void testGeneralPrimitiveTypeFields() throws Exception { GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); src.f0 = (byte) 0; @@ -101,8 +103,21 @@ public class TestMessagePackUnpackable extends TestCase { } } + public void testListAndMap() throws Exception { + // TODO + } + + @MessagePackUnpackable + public static class ListAndMapClass { + public List f0; + public Map f1; + + public ListAndMapClass() { + } + } + @Test - public void testPublicDefaultConstructorClass() throws Exception { + public void testDefaultConstructorModifiers() throws Exception { try { PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class); fail(); @@ -111,21 +126,24 @@ public class TestMessagePackUnpackable extends TestCase { } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(PrivateDefaultConstructorClass.class); fail(); } catch (PackUnpackUtilException e) { assertTrue(true); } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(ProtectedDefaultConstructorClass.class); assertTrue(true); } catch (PackUnpackUtilException e) { fail(); } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(PackageDefaultConstructorClass.class); fail(); } catch (PackUnpackUtilException e) { assertTrue(true); @@ -158,7 +176,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testPublicModifierClass() throws Exception { + public void testClassModifiers() throws Exception { try { PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class); fail(); @@ -197,7 +215,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testFinalAndAbstractModifierClass() throws Exception { + public void testFinalClassAndAbstractClass() throws Exception { try { PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class); fail(); @@ -223,7 +241,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testInterfaceAndEnum() throws Exception { + public void testInterfaceAndEnumType() throws Exception { try { PackUnpackUtil.newEnhancedInstance(SampleInterface.class); fail(); @@ -249,32 +267,128 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testFinalFieldClass() throws Exception { - + public void testFieldModifiers() throws Exception { + FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); } - @Test - public void testPrivateFieldClass() throws Exception { - - } - - @Test - public void testProtectedFieldClass() throws Exception { - - } - - @Test - public void testNonModifierFieldClass() throws Exception { + @MessagePackUnpackable + public static class FieldModifiersClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + public FieldModifiersClass() { + } } @Test public void testNestedAnnotatedFieldClass() throws Exception { + NestedClass src2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass src = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + src.f0 = 0; + src2.f2 = 2; + src.f1 = src2; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + NestedClass dst2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass dst = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + dst.f1 = dst2; + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src2.f2 == dst.f1.f2); + } + @MessagePackUnpackable + public static class BaseClass { + public int f0; + public NestedClass f1; + + public BaseClass() { + } + } + + @MessagePackUnpackable + public static class NestedClass { + public int f2; + + public NestedClass() { + } } @Test - public void testSuperClass() throws Exception { + public void testExtendedClass() throws Exception { + SampleSubClass src = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f8 = 8; + src.f9 = 9; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleSubClass dst = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + assertTrue(src.f5 == dst.f5); + assertTrue(src.f6 == dst.f6); + assertTrue(src.f8 == dst.f8); + assertTrue(src.f9 != dst.f9); + } + @MessagePackUnpackable + public static class SampleSubClass extends SampleSuperClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + + public SampleSubClass() { + } + } + + public static class SampleSuperClass { + public int f5; + public final int f6 = 2; + private int f7; + protected int f8; + int f9; + + public SampleSuperClass() { + } } } From 56ece4db0f03f086c2d4608472de4b20e6a83a19 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 23:23:09 +0900 Subject: [PATCH 158/259] fixed a bug the program for packing and unpacking byte[] in annotation-utility --- .../java/org/msgpack/util/annotation/PackUnpackUtil.java | 3 ++- .../msgpack/util/annotation/TestMessagePackUnpackable.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index a58b4bd5..f92c51a0 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -71,7 +71,8 @@ public class PackUnpackUtil { static final String TYPE_NAME_SHORT2 = Short.class.getName(); - static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + //static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + static final String TYPE_NAME_BYTEARRAY = "byte[]"; static final String TYPE_NAME_LIST = List.class.getName(); diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 4f6e74ef..1cde65f2 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -69,6 +69,7 @@ public class TestMessagePackUnpackable extends TestCase { src.f6 = false; src.f7 = new BigInteger("7"); src.f8 = "8"; + src.f9 = new byte[] { 0x01, 0x02 }; ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(src); GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil @@ -85,6 +86,8 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f6, dst.f6); assertEquals(src.f7, dst.f7); assertEquals(src.f8, dst.f8); + assertEquals(src.f9[0], dst.f9[0]); + assertEquals(src.f9[1], dst.f9[1]); } @MessagePackUnpackable @@ -98,13 +101,13 @@ public class TestMessagePackUnpackable extends TestCase { public Boolean f6; public BigInteger f7; public String f8; + public byte[] f9; public GeneralReferenceTypeFieldsClass() { } } public void testListAndMap() throws Exception { - // TODO } @MessagePackUnpackable From 987248ccbb4d18d532f2d07c28f47d37934b9c59 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:31:01 +0900 Subject: [PATCH 159/259] Use xshelper.h in all the C files --- perl/MANIFEST.SKIP | 1 - perl/perlxs.h | 76 --------------------------------------- perl/util.h | 11 ------ perl/xs-src/MessagePack.c | 14 +------- perl/xs-src/unpack.c | 8 ----- 5 files changed, 1 insertion(+), 109 deletions(-) delete mode 100644 perl/perlxs.h delete mode 100644 perl/util.h diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 372742ca..71a24e5c 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -25,4 +25,3 @@ ^Data-MessagePack-[0-9.]+/ ^\.testenv/test_pp.pl ^ppport.h$ -^xshelper.h$ diff --git a/perl/perlxs.h b/perl/perlxs.h deleted file mode 100644 index 441682de..00000000 --- a/perl/perlxs.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - perlxs.h - Standard XS header file - Copyright (c) Fuji, Goro (gfx) -*/ - -#ifdef __cplusplus -extern "C" { -#endif - -#define PERL_NO_GET_CONTEXT /* we want efficiency */ -#include - -#include -#define NO_XSLOCKS /* for exceptions */ -#include - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#include "ppport.h" - -/* portability stuff not supported by ppport.h yet */ - -#ifndef STATIC_INLINE /* from 5.13.4 */ -# if defined(__GNUC__) || defined(__cplusplus__) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) -# define STATIC_INLINE static inline -# else -# define STATIC_INLINE static -# endif -#endif /* STATIC_INLINE */ - -#ifndef __attribute__format__ -#define __attribute__format__(a,b,c) /* nothing */ -#endif - -#ifndef LIKELY /* they are just a compiler's hint */ -#define LIKELY(x) (x) -#define UNLIKELY(x) (x) -#endif - -#ifndef newSVpvs_share -#define newSVpvs_share(s) Perl_newSVpvn_share(aTHX_ STR_WITH_LEN(s), 0U) -#endif - -#ifndef get_cvs -#define get_cvs(name, flags) get_cv(name, flags) -#endif - -#ifndef GvNAME_get -#define GvNAME_get GvNAME -#endif -#ifndef GvNAMELEN_get -#define GvNAMELEN_get GvNAMELEN -#endif - -#ifndef CvGV_set -#define CvGV_set(cv, gv) (CvGV(cv) = (gv)) -#endif - -/* general utility */ - -#if PERL_BCDVERSION >= 0x5008005 -#define LooksLikeNumber(x) looks_like_number(x) -#else -#define LooksLikeNumber(x) (SvPOKp(x) ? looks_like_number(x) : (I32)SvNIOKp(x)) -#endif - -#define newAV_mortal() (AV*)sv_2mortal((SV*)newAV()) -#define newHV_mortal() (HV*)sv_2mortal((SV*)newHV()) - -#define DECL_BOOT(name) EXTERN_C XS(CAT2(boot_, name)) -#define CALL_BOOT(name) STMT_START { \ - PUSHMARK(SP); \ - CALL_FPTR(CAT2(boot_, name))(aTHX_ cv); \ - } STMT_END diff --git a/perl/util.h b/perl/util.h deleted file mode 100644 index 2b4ed072..00000000 --- a/perl/util.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef __PERL_MSGPACK_UTIL_H__ -#define __PERL_MSGPACK_UTIL_H__ - -#if __GNUC__ >= 3 -# define INLINE inline -#else -# define INLINE -#endif - -#endif // __PERL_MSGPACK_UTIL_H__ - diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c index fd1b344d..259fd156 100644 --- a/perl/xs-src/MessagePack.c +++ b/perl/xs-src/MessagePack.c @@ -1,14 +1,4 @@ -#ifdef __cplusplus -extern "C" { -#endif -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" -#define NEED_newCONSTSUB -#include "ppport.h" -#ifdef __cplusplus -}; -#endif +#include "xshelper.h" XS(xs_pack); XS(xs_unpack); @@ -24,13 +14,11 @@ void boot_Data__MessagePack_pack(void); XS(boot_Data__MessagePack) { dXSARGS; - HV * stash; boot_Data__MessagePack_pack(); newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); - stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index c329e99c..2edfdf79 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -1,15 +1,7 @@ -#ifdef __cplusplus -extern "C" { -#endif - #define NEED_newRV_noinc #define NEED_sv_2pv_flags #include "xshelper.h" -#ifdef __cplusplus -}; -#endif - typedef struct { int finished; SV* source; From bebcc24ab8908a07449c8ff6992a5dea2371e136 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:32:01 +0900 Subject: [PATCH 160/259] Depends on XSUtil 0.32 --- perl/Makefile.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 7053cf96..e7b8c474 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,4 +1,5 @@ use inc::Module::Install; +use Module::Install::XSUtil 0.32; use Config; name 'Data-MessagePack'; From 197205853fb552234250e1616af85b60552c9869 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:35:10 +0900 Subject: [PATCH 161/259] Use newSV(). NEWSV() is deprecated. --- perl/xs-src/pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 0aa32502..4378b056 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -252,7 +252,7 @@ XS(xs_pack) { if (items >= 3) depth = SvIV(ST(2)); enc_t enc; - enc.sv = sv_2mortal(NEWSV(0, INIT_SIZE)); + enc.sv = sv_2mortal(newSV(INIT_SIZE)); enc.cur = SvPVX(enc.sv); enc.end = SvEND(enc.sv); SvPOK_only(enc.sv); From 0768cf17b61465ebc2fe5e0132e0494399eb94d1 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:36:43 +0900 Subject: [PATCH 162/259] Taking NULL is a bug --- perl/xs-src/pack.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 4378b056..7bad6145 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -143,14 +143,13 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); -STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) { +STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) { dTHX; + assert(sv); if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); - if (sv==NULL) { - msgpack_pack_nil(enc); - } else if (SvPOKp(sv)) { + if (SvPOKp(sv)) { STRLEN len; char * csv = SvPV(sv, len); From 0f02ef20a9ff77e830d5f199a4deac1c9a4d7d87 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:46:11 +0900 Subject: [PATCH 163/259] Improve benchmarks --- perl/benchmark/deserialize.pl | 6 +++--- perl/benchmark/serialize.pl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 750704e9..9658c0c8 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -9,7 +9,7 @@ my $a = { "method" => "handleMessage", "params" => [ "user1", "we were just talking" ], "id" => undef, - "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ] + "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ] }; my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); @@ -19,8 +19,8 @@ print "-- deserialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; -timethese( - 1000000 => { +cmpthese timethese( + -1 => { json => sub { JSON::XS::decode_json($j) }, mp => sub { Data::MessagePack->unpack($m) }, storable => sub { Storable::thaw($s) }, diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index c5ab15bc..ee9e7a45 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -9,15 +9,15 @@ my $a = { "method" => "handleMessage", "params" => [ "user1", "we were just talking" ], "id" => undef, - "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ] + "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ] }; print "-- serialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; -timethese( - 1000000 => { +cmpthese timethese( + -1 => { json => sub { JSON::XS::encode_json($a) }, storable => sub { Storable::freeze($a) }, mp => sub { Data::MessagePack->pack($a) }, From d86104ed5dc3b2a99542a8e6c4468b778200f5fc Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:48:23 +0900 Subject: [PATCH 164/259] Tweaks --- perl/xs-src/pack.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 7bad6145..67483dcf 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -48,7 +48,7 @@ STATIC_INLINE void need(enc_t *enc, STRLEN len) dTHX; if (enc->cur + len >= enc->end) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); - SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + sv_grow (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); enc->cur = SvPVX (enc->sv) + cur; enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; } @@ -159,13 +159,17 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de msgpack_pack_raw(enc, len); msgpack_pack_raw_body(enc, csv, len); } - } else if (SvNOKp(sv)) { - /* XXX long double is not supported yet. */ - msgpack_pack_double(enc, (double)SvNVX(sv)); - } else if (SvIOK_UV(sv)) { - msgpack_pack_uint32(enc, SvUV(sv)); - } else if (SvIOKp(sv)) { - PACK_IV(enc, SvIV(sv)); + } else if (SvNIOKp(sv)) { + if(SvUOK(sv)) { + msgpack_pack_uint32(enc, SvUV(sv)); + } + else if(SvIOKp(sv)) { + PACK_IV(enc, SvIV(sv)); + } + else { + /* XXX long double is not supported yet. */ + msgpack_pack_double(enc, (double)SvNVX(sv)); + } } else if (SvROK(sv)) { _msgpack_pack_rv(enc, SvRV(sv), depth-1); } else if (!SvOK(sv)) { From 60b36ffaa38ba32c1a97b4143138bb2e511fb1ae Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:50:11 +0900 Subject: [PATCH 165/259] Micro optimizations --- perl/xs-src/pack.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 67483dcf..30bc0325 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -32,10 +32,13 @@ static void need(enc_t *enc, STRLEN len); #if IVSIZE == 8 # define PACK_IV msgpack_pack_int64 +# define PACK_UV msgpack_pack_uint64 #elif IVSIZE == 4 # define PACK_IV msgpack_pack_int32 +# define PACK_UV msgpack_pack_uint32 #elif IVSIZE == 2 # define PACK_IV msgpack_pack_int16 +# define PACK_UV msgpack_pack_uint16 #else # error "msgpack only supports IVSIZE = 8,4,2 environment." #endif @@ -150,21 +153,21 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de SvGETMAGIC(sv); if (SvPOKp(sv)) { - STRLEN len; - char * csv = SvPV(sv, len); + STRLEN const len = SvCUR(sv); + const char* const pv = SvPVX_const(sv); - if (s_pref_int && try_int(enc, csv, len)) { + if (s_pref_int && try_int(enc, pv, len)) { return; } else { msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, csv, len); + msgpack_pack_raw_body(enc, pv, len); } } else if (SvNIOKp(sv)) { if(SvUOK(sv)) { - msgpack_pack_uint32(enc, SvUV(sv)); + PACK_UV(enc, SvUVX(sv)); } else if(SvIOKp(sv)) { - PACK_IV(enc, SvIV(sv)); + PACK_IV(enc, SvIVX(sv)); } else { /* XXX long double is not supported yet. */ From 4adcdb5ba81544517d65953497f0b0ac17b95a0a Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:51:33 +0900 Subject: [PATCH 166/259] Remove a duplicated depth check --- perl/xs-src/pack.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 30bc0325..c01ba44b 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -149,7 +149,7 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) { dTHX; assert(sv); - if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); + if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); if (SvPOKp(sv)) { @@ -188,7 +188,6 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; dTHX; - if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED); SvGETMAGIC(sv); svt = SvTYPE(sv); From c5e15123fd14fe448222689d460b0b02a7659e14 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:52:07 +0900 Subject: [PATCH 167/259] Add an assertion --- perl/xs-src/pack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index c01ba44b..5feba4ab 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -188,6 +188,7 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { svtype svt; dTHX; + assert(sv); SvGETMAGIC(sv); svt = SvTYPE(sv); From 50c74103aaecaff6f72ada55a1c9dc683c3017ef Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 12:56:13 +0900 Subject: [PATCH 168/259] Avoid compiler's warnings --- perl/xs-src/pack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 5feba4ab..c0c610e7 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -60,7 +60,7 @@ STATIC_INLINE void need(enc_t *enc, STRLEN len) static int s_pref_int = 0; -STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg) { +STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) { if (SvTRUE(sv)) { s_pref_int = 1; } else { @@ -211,7 +211,7 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { msgpack_pack_map(enc, count); - while (he = hv_iternext(hval)) { + while ((he = hv_iternext(hval))) { _msgpack_pack_sv(enc, hv_iterkeysv(he), depth); _msgpack_pack_sv(enc, HeVAL(he), depth); } From 9953218de18a87173869e8ffe97aa274046ddae1 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:03:47 +0900 Subject: [PATCH 169/259] Tidy --- perl/xs-src/unpack.c | 117 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 21 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 2edfdf79..05d953a8 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -50,16 +50,31 @@ static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); STATIC_INLINE SV* template_callback_root(unpack_user* u) -{ dTHX; return &PL_sv_undef; } +{ + dTHX; + return &PL_sv_undef; +} STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSVuv(d)); + return 0; +} STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSVuv(d)); + return 0; +} STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSVuv(d)); + return 0; +} STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) { @@ -72,49 +87,109 @@ STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) return 0; } -STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } +STATIC_INLINE int template_callback_int8(unpack_user* u PERL_UNUSED_DECL, int8_t d, SV** o) +{ + dTHX; + *o = sv_2mortal(newSViv(d)); + return 0; +} -STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } +STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16_t d, SV** o) +{ + dTHX; + *o = sv_2mortal(newSViv(d)); + return 0; +} STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSViv(d)); + return 0; +} STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) -{ dTHX; *o = sv_2mortal(newSViv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSViv(d)); + return 0; +} STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o) -{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSVnv(d)); + return 0; +} STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o) -{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; } +{ + dTHX; + *o = sv_2mortal(newSVnv(d)); + return 0; +} /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */ STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o) -{ dTHX; *o = sv_newmortal(); return 0; } +{ + dTHX; + *o = sv_newmortal(); + return 0; +} STATIC_INLINE int template_callback_true(unpack_user* u, SV** o) -{ dTHX; *o = get_bool("Data::MessagePack::true") ; return 0; } +{ + dTHX; + *o = get_bool("Data::MessagePack::true"); + return 0; +} STATIC_INLINE int template_callback_false(unpack_user* u, SV** o) -{ dTHX; *o = get_bool("Data::MessagePack::false") ; return 0; } +{ + dTHX; *o = get_bool("Data::MessagePack::false"); + return 0; +} STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) -{ dTHX; AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; } +{ + dTHX; + AV* a = (AV*)sv_2mortal((SV*)newAV()); + *o = sv_2mortal((SV*)newRV_inc((SV*)a)); + av_extend(a, n); + return 0; +} STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ dTHX; av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ +{ + dTHX; + av_push((AV*)SvRV(*c), o); + SvREFCNT_inc(o); + return 0; +} STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) -{ dTHX; HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; } +{ + dTHX; + HV* h = (HV*)sv_2mortal((SV*)newHV()); + *o = sv_2mortal(newRV_inc((SV*)h)); + return 0; +} STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) -{ dTHX; hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } +{ + dTHX; + hv_store_ent((HV*)SvRV(*c), k, v, 0); + SvREFCNT_inc(v); + return 0; +} STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) -{ dTHX; *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } -/* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; } <= this does not works. */ +{ + dTHX; + /* *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */ + *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); + return 0; +} #define UNPACKER(from, name) \ msgpack_unpack_t *name; \ From 10bf3ee9dec23337f30074785d7c21b81ffe2c40 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:07:44 +0900 Subject: [PATCH 170/259] Avoid compiler's warnings --- perl/xs-src/unpack.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 05d953a8..108f9fda 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -46,37 +46,37 @@ static void template_init(msgpack_unpack_t* u); static SV* template_data(msgpack_unpack_t* u); -static int template_execute(msgpack_unpack_t* u, +static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL, const char* data, size_t len, size_t* off); -STATIC_INLINE SV* template_callback_root(unpack_user* u) +STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL) { dTHX; return &PL_sv_undef; } -STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o) { dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } -STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +STATIC_INLINE int template_callback_uint16(unpack_user* u PERL_UNUSED_DECL, uint16_t d, SV** o) { dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } -STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +STATIC_INLINE int template_callback_uint32(unpack_user* u PERL_UNUSED_DECL, uint32_t d, SV** o) { dTHX; *o = sv_2mortal(newSVuv(d)); return 0; } -STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t d, SV** o) { dTHX; #if IVSIZE==4 @@ -101,28 +101,28 @@ STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16 return 0; } -STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) +STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32_t d, SV** o) { dTHX; *o = sv_2mortal(newSViv(d)); return 0; } -STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) +STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o) { dTHX; *o = sv_2mortal(newSViv(d)); return 0; } -STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o) +STATIC_INLINE int template_callback_float(unpack_user* u PERL_UNUSED_DECL, float d, SV** o) { dTHX; *o = sv_2mortal(newSVnv(d)); return 0; } -STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o) +STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o) { dTHX; *o = sv_2mortal(newSVnv(d)); @@ -130,27 +130,27 @@ STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o) } /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */ -STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o) { dTHX; *o = sv_newmortal(); return 0; } -STATIC_INLINE int template_callback_true(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o) { dTHX; *o = get_bool("Data::MessagePack::true"); return 0; } -STATIC_INLINE int template_callback_false(unpack_user* u, SV** o) +STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o) { dTHX; *o = get_bool("Data::MessagePack::false"); return 0; } -STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) +STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o) { dTHX; AV* a = (AV*)sv_2mortal((SV*)newAV()); @@ -159,7 +159,7 @@ STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o return 0; } -STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) +STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o) { dTHX; av_push((AV*)SvRV(*c), o); @@ -167,7 +167,7 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) return 0; } -STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) +STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o) { dTHX; HV* h = (HV*)sv_2mortal((SV*)newHV()); @@ -175,15 +175,15 @@ STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) return 0; } -STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v) { dTHX; - hv_store_ent((HV*)SvRV(*c), k, v, 0); + (void)hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } -STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o) { dTHX; /* *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */ @@ -203,7 +203,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) { msgpack_unpack_t mp; dTHX; - unpack_user u = {0, &PL_sv_undef}; + unpack_user u = {0, &PL_sv_undef, false}; int ret; size_t from = 0; STRLEN dlen; @@ -248,7 +248,6 @@ XS(xs_unpack_limit) { XS(xs_unpack) { dXSARGS; - msgpack_unpack_t mp; if (items != 2) { Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); @@ -300,7 +299,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { long dlen = limit; int ret; - if(from >= dlen) { + if(from >= (size_t)dlen) { Perl_croak(aTHX_ "offset is bigger than data buffer size."); } From 6981234736b10631fc79ffe59933ee3a009c0cc2 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:09:14 +0900 Subject: [PATCH 171/259] Fix a possible mis-unpack on int64 --- perl/xs-src/unpack.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 108f9fda..4d4e965c 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -111,7 +111,11 @@ STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32 STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o) { dTHX; +#if IVSIZE==4 + *o = sv_2mortal(newSVnv(d)); +#else *o = sv_2mortal(newSViv(d)); +#endif return 0; } From d36543b20419fa87ec71c42ee5f80ca1e03f71a1 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:12:17 +0900 Subject: [PATCH 172/259] Micro optimizations --- perl/xs-src/unpack.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 4d4e965c..0a79ef5f 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -157,9 +157,9 @@ STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o) { dTHX; - AV* a = (AV*)sv_2mortal((SV*)newAV()); - *o = sv_2mortal((SV*)newRV_inc((SV*)a)); - av_extend(a, n); + AV* const a = newAV(); + *o = sv_2mortal(newRV_noinc((SV*)a)); + av_extend(a, n + 1); return 0; } @@ -167,15 +167,15 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, { dTHX; av_push((AV*)SvRV(*c), o); - SvREFCNT_inc(o); + SvREFCNT_inc_simple_void_NN(o); return 0; } STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o) { dTHX; - HV* h = (HV*)sv_2mortal((SV*)newHV()); - *o = sv_2mortal(newRV_inc((SV*)h)); + HV* const h = newHV(); + *o = sv_2mortal(newRV_noinc((SV*)h)); return 0; } @@ -183,7 +183,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV { dTHX; (void)hv_store_ent((HV*)SvRV(*c), k, v, 0); - SvREFCNT_inc(v); + SvREFCNT_inc_simple_void_NN(v); return 0; } From c694f1a4a9b5df801419d23265d6519c59f9e1ae Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:16:13 +0900 Subject: [PATCH 173/259] Tweaks --- perl/xs-src/unpack.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 0a79ef5f..e3bb901e 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -204,7 +204,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c #include "msgpack/unpack_template.h" -STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) { +STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) { msgpack_unpack_t mp; dTHX; unpack_user u = {0, &PL_sv_undef, false}; @@ -252,14 +252,20 @@ XS(xs_unpack_limit) { XS(xs_unpack) { dXSARGS; + SV* const data = ST(1); + size_t limit; - if (items != 2) { - Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); + if (items == 2) { + limit = sv_len(data); + } + else if(items == 3) { + limit = SvUVx(ST(2)); + } + else { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])"); } - { - ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1))); - } + ST(0) = _msgpack_unpack(data, limit); XSRETURN(1); } From 6852a8ca9d8585fd42ec761900d334ae964732c9 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:16:55 +0900 Subject: [PATCH 174/259] Remove an unused function: xs_unpack_limit --- perl/xs-src/unpack.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index e3bb901e..5e757173 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -235,21 +235,6 @@ STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) { } } -XS(xs_unpack_limit) { - dXSARGS; - - if (items != 3) { - Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)"); - } - - { - int limit = SvIV(ST(2)); - ST(0) = _msgpack_unpack(ST(1), limit); - } - XSRETURN(1); -} - - XS(xs_unpack) { dXSARGS; SV* const data = ST(1); From 859969241a8695e306789ed30659d05571db8fe2 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:20:20 +0900 Subject: [PATCH 175/259] Tweaks --- perl/xs-src/unpack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 5e757173..78942233 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -190,8 +190,8 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o) { dTHX; - /* *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */ - *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); + /* newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */ + *o = ((l==0) ? newSVpvs_flags("", SVs_TEMP) : newSVpvn_flags(p, l, SVs_TEMP)); return 0; } From af73b9d11b0a8e0a0d1b6a02ee0b8213cca3867c Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:22:39 +0900 Subject: [PATCH 176/259] Shortcut av_push() --- perl/xs-src/unpack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 78942233..080c4bde 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -166,7 +166,8 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o) { dTHX; - av_push((AV*)SvRV(*c), o); + AV* const a = (AV*)SvRV(*c); + (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o) SvREFCNT_inc_simple_void_NN(o); return 0; } From 0e0a2aa9810c22e3744694965e3d45cab6c135e6 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:34:18 +0900 Subject: [PATCH 177/259] Add various integers to benchmarks --- perl/benchmark/deserialize.pl | 4 +++- perl/benchmark/serialize.pl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 9658c0c8..634a79ed 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -5,11 +5,13 @@ use JSON::XS; use Benchmark ':all'; use Storable; +#$Data::MessagePack::PreferInteger = 1; + my $a = { "method" => "handleMessage", "params" => [ "user1", "we were just talking" ], "id" => undef, - "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ] + "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], }; my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index ee9e7a45..e0509ffa 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -9,7 +9,7 @@ my $a = { "method" => "handleMessage", "params" => [ "user1", "we were just talking" ], "id" => undef, - "array" => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ] + "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], }; print "-- serialize\n"; From 1de03fbe180e9abed96aec275cf67b6f7d46a232 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:41:10 +0900 Subject: [PATCH 178/259] Tweaks for unpacker --- perl/xs-src/unpack.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 080c4bde..320eb45b 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -196,12 +196,15 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c return 0; } -#define UNPACKER(from, name) \ - msgpack_unpack_t *name; \ - name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \ - if(name == NULL) { \ - Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ - } +#define UNPACKER(from, name) \ + msgpack_unpack_t *name; \ + if(!(SvROK(from) && SvIOK(SvRV(from)))) { \ + Perl_croak(aTHX_ "Invalid unpacker instance for " #name); \ + } \ + name = INT2PTR(msgpack_unpack_t*, SvIVX(SvRV((from)))); \ + if(name == NULL) { \ + Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ + } #include "msgpack/unpack_template.h" From 6a60cb4dc0d5c2e36a4aa8fd8dbe5b3c826a935d Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 13:54:18 +0900 Subject: [PATCH 179/259] Add const --- perl/xs-src/pack.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index c0c610e7..6926839b 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -13,17 +13,18 @@ static inline void msgpack_pack ## name typedef struct { - char *cur; /* SvPVX (sv) + current output position */ - char *end; /* SvEND (sv) */ - SV *sv; /* result scalar */ + char *cur; /* SvPVX (sv) + current output position */ + const char *end; /* SvEND (sv) */ + SV *sv; /* result scalar */ } enc_t; -static void need(enc_t *enc, STRLEN len); + +STATIC_INLINE void need(enc_t* const enc, STRLEN const len); #define msgpack_pack_user enc_t* #define msgpack_pack_append_buffer(enc, buf, len) \ - need(enc, len); \ - memcpy(enc->cur, buf, len); \ + need(enc, len); \ + memcpy(enc->cur, buf, len); \ enc->cur += len; #include "msgpack/pack_template.h" @@ -46,14 +47,14 @@ static void need(enc_t *enc, STRLEN len); #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)" -STATIC_INLINE void need(enc_t *enc, STRLEN len) +STATIC_INLINE void need(enc_t* const enc, STRLEN const len) { - dTHX; if (enc->cur + len >= enc->end) { - STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + dTHX; + STRLEN const cur = enc->cur - SvPVX_const(enc->sv); sv_grow (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); - enc->cur = SvPVX (enc->sv) + cur; - enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; + enc->cur = SvPVX_mutable(enc->sv) + cur; + enc->end = SvPVX_const(enc->sv) + SvLEN (enc->sv) - 1; } } From 83acd6529fe4902624117cab62c2377aa3fe2b27 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:06:10 +0900 Subject: [PATCH 180/259] Remove an unused user data: source (sv) --- perl/xs-src/unpack.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 320eb45b..90cfa7b8 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -3,9 +3,8 @@ #include "xshelper.h" typedef struct { - int finished; - SV* source; - int incremented; + bool finished; + bool incremented; } unpack_user; #include "msgpack/unpack_define.h" @@ -211,7 +210,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) { msgpack_unpack_t mp; dTHX; - unpack_user u = {0, &PL_sv_undef, false}; + unpack_user u = {false, false}; int ret; size_t from = 0; STRLEN dlen; @@ -221,10 +220,7 @@ STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) { template_init(&mp); mp.user = u; - mp.user.source = data; ret = template_execute(&mp, dptr, (size_t)dlen, &from); - mp.user.source = &PL_sv_undef; - obj = template_data(&mp); if(ret < 0) { @@ -264,7 +260,7 @@ XS(xs_unpack) { STATIC_INLINE void _reset(SV* self) { dTHX; - unpack_user u = {0, &PL_sv_undef, 0}; + unpack_user u = {false, false}; UNPACKER(self, mp); template_init(mp); @@ -302,17 +298,15 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { Perl_croak(aTHX_ "offset is bigger than data buffer size."); } - mp->user.source = data; ret = template_execute(mp, dptr, (size_t)dlen, &from); - mp->user.source = &PL_sv_undef; if(ret < 0) { Perl_croak(aTHX_ "parse error."); } else if(ret > 0) { - mp->user.finished = 1; + mp->user.finished = true; return sv_2mortal(newSVuv(from)); } else { - mp->user.finished = 0; + mp->user.finished = false; return sv_2mortal(newSVuv(from)); } } @@ -335,7 +329,7 @@ XS(xs_unpacker_execute) { SV * d2 = template_data(mp); if (!mp->user.incremented && d2) { SvREFCNT_inc(d2); - mp->user.incremented = 1; + mp->user.incremented = true; } } } @@ -366,7 +360,7 @@ XS(xs_unpacker_is_finished) { } UNPACKER(ST(0), mp); - ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; + ST(0) = boolSV(mp->user.finished); XSRETURN(1); } From f32234291e26301d4159b109b7fed78fb6e3ee56 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:07:33 +0900 Subject: [PATCH 181/259] Remove an useless local variable --- perl/xs-src/unpack.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 90cfa7b8..85136d97 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -285,20 +285,19 @@ XS(xs_unpacker_new) { XSRETURN(1); } -STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { +STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { dTHX; UNPACKER(self, mp); size_t from = off; const char* dptr = SvPV_nolen_const(data); - long dlen = limit; int ret; - if(from >= (size_t)dlen) { + if(from >= limit) { Perl_croak(aTHX_ "offset is bigger than data buffer size."); } - ret = template_execute(mp, dptr, (size_t)dlen, &from); + ret = template_execute(mp, dptr, limit, &from); if(ret < 0) { Perl_croak(aTHX_ "parse error."); @@ -323,7 +322,7 @@ XS(xs_unpacker_execute) { SV* data = ST(1); IV off = SvIV(ST(2)); /* offset of $data. normaly, 0. */ - ST(0) = _execute_impl(self, data, off, sv_len(data)); + ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data)); { SV * d2 = template_data(mp); @@ -348,7 +347,7 @@ XS(xs_unpacker_execute_limit) { IV off = SvIV(ST(2)); IV limit = SvIV(ST(3)); - ST(0) = _execute_impl(self, data, off, limit); + ST(0) = _execute_impl(self, data, off, (size_t)limit); XSRETURN(1); } From f0e044ecd8550a14a9fbda9448bd74a3dace4d1a Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:09:03 +0900 Subject: [PATCH 182/259] Cleanup --- perl/xs-src/unpack.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 85136d97..9500212c 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -301,11 +301,8 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { if(ret < 0) { Perl_croak(aTHX_ "parse error."); - } else if(ret > 0) { - mp->user.finished = true; - return sv_2mortal(newSVuv(from)); } else { - mp->user.finished = false; + mp->user.finished = (ret > 0) ? true : false; return sv_2mortal(newSVuv(from)); } } From 5bdac96375f0cc5b75313cc3a9a7bbeb21594a0b Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:18:38 +0900 Subject: [PATCH 183/259] The object root can be NULL --- perl/xs-src/unpack.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 9500212c..dce6782c 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -50,8 +50,7 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL, STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL) { - dTHX; - return &PL_sv_undef; + return NULL; } STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o) @@ -310,7 +309,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { XS(xs_unpacker_execute) { dXSARGS; if (items != 3) { - Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)"); } UNPACKER(ST(0), mp); @@ -398,8 +397,8 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); - SV * data = template_data(mp); - if (SvOK(data)) { + SV* const data = template_data(mp); + if (data) { SvREFCNT_dec(data); } Safefree(mp); From 0ae206b1bb5233b7f4538dcf5fc40ff8080588bf Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:19:22 +0900 Subject: [PATCH 184/259] Revert "The object root can be NULL" This reverts commit 5bdac96375f0cc5b75313cc3a9a7bbeb21594a0b. --- perl/xs-src/unpack.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index dce6782c..9500212c 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -50,7 +50,8 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL, STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL) { - return NULL; + dTHX; + return &PL_sv_undef; } STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o) @@ -309,7 +310,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { XS(xs_unpacker_execute) { dXSARGS; if (items != 3) { - Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)"); + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); } UNPACKER(ST(0), mp); @@ -397,8 +398,8 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); - SV* const data = template_data(mp); - if (data) { + SV * data = template_data(mp); + if (SvOK(data)) { SvREFCNT_dec(data); } Safefree(mp); From 07e68aa6945e4be0820f9c53a3cc1bbbb041a556 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:20:32 +0900 Subject: [PATCH 185/259] Fix an usage message --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 9500212c..c4ac22fd 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -310,7 +310,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { XS(xs_unpacker_execute) { dXSARGS; if (items != 3) { - Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)"); } UNPACKER(ST(0), mp); From 7644555d6bd0f17472f02c18dd65bd24aa14edd4 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:21:44 +0900 Subject: [PATCH 186/259] Use sv_mortalcopy() --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index c4ac22fd..2b6eafe2 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -368,7 +368,7 @@ XS(xs_unpacker_data) { } UNPACKER(ST(0), mp); - ST(0) = sv_2mortal(newSVsv(template_data(mp))); + ST(0) = sv_mortalcopy(template_data(mp)); XSRETURN(1); } From cd862409cc6028bac9fd5825cfdc0386804091de Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:25:50 +0900 Subject: [PATCH 187/259] Clean up --- perl/xs-src/unpack.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 2b6eafe2..2d659b4b 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -382,9 +382,7 @@ XS(xs_unpacker_reset) { UNPACKER(ST(0), mp); { SV * data = template_data(mp); - if (data) { - SvREFCNT_dec(data); - } + SvREFCNT_dec(data); } _reset(ST(0)); From 11cde61eab8e15f9936c3234897cef90fef05cd6 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 14:38:26 +0900 Subject: [PATCH 188/259] No debug output --- perl/xs-src/pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 6926839b..5eaf17e5 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -238,7 +238,7 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) { else if (len == 1 && *pv == '0') msgpack_pack_false(enc); else { - sv_dump(sv); + //sv_dump(sv); croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1", SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } From a11165830bc6075c39978adcb72af07d5d7c6234 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 15:06:03 +0900 Subject: [PATCH 189/259] More useful error messages --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 2d659b4b..9ccc44ad 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -294,7 +294,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { int ret; if(from >= limit) { - Perl_croak(aTHX_ "offset is bigger than data buffer size."); + Perl_croak(aTHX_ "offset (%lu) is bigger than data buffer size (%lu)", (unsigned long)off, (unsigned long)limit); } ret = template_execute(mp, dptr, limit, &from); From f8ee79ab72034223b0ae698b00e7e369d217cb47 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 15:25:48 +0900 Subject: [PATCH 190/259] Add failing tests --- perl/t/06_stream_unpack2.t | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index eaf2cb4b..bc158cdd 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -3,7 +3,7 @@ use warnings; use Data::MessagePack; use Test::More tests => 6; -my $input = [(undef)x16]; +my $input = [42, "foo", { x => [ (undef)x16 ] }, 3.14 ]; my $packed = Data::MessagePack->pack($input); is_deeply(Data::MessagePack->unpack($packed), $input); @@ -17,9 +17,33 @@ is_deeply(Data::MessagePack->unpack($packed), $input); { my $up = Data::MessagePack::Unpacker->new(); is $up->execute(substr($packed, 0, 3), 0), 3; + ok !$up->is_finished; $up->execute($packed, 3); ok $up->is_finished; is_deeply $up->data, $input; } +{ + my $up = Data::MessagePack::Unpacker->new(); + my $offset = 0; + my $size = 5; + + note "length: ", length($packed); + while(not $up->is_finished) { + note "offset: ", $offset; + my $bytes = substr($packed, $offset, $size); + note join " ", map { unpack 'H2', $_ } split //, $bytes; + my $x = $up->execute($bytes, 0); + if($x <= 0) { + diag "Something's wrong: $x"; + last; + } + else { + $offset += $x; + } + } + ok $up->is_finished; + is_deeply $up->data, $input; +} + From fe7e7a8d077de47f3c92dad3ec6eb39e8c1ae077 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 15:26:02 +0900 Subject: [PATCH 191/259] Add leaktrace tests --- perl/xt/leaks/leaktrace.t | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 perl/xt/leaks/leaktrace.t diff --git a/perl/xt/leaks/leaktrace.t b/perl/xt/leaks/leaktrace.t new file mode 100755 index 00000000..ff64f6ff --- /dev/null +++ b/perl/xt/leaks/leaktrace.t @@ -0,0 +1,40 @@ +#!perl -w +use strict; +use Test::Requires 'Test::LeakTrace'; +use Test::More; + +use Data::MessagePack; + +my $data = { + a => 'foo', + b => 42, + c => undef, + d => [qw(bar baz)], + e => 3.14, +}; + +no_leaks_ok { + my $s = Data::MessagePack->pack($data); +}; + +no_leaks_ok { + eval { Data::MessagePack->pack([\*STDIN]) }; + #note $@; + $@ or die "it must die"; +}; + +my $s = Data::MessagePack->pack($data); + +no_leaks_ok { + my $data = Data::MessagePack->unpack($s); +}; + +no_leaks_ok { + my $ss = $s; + chop $ss; + eval { Data::MessagePack->unpack($ss) }; + #note $@; + $@ or die "it must die"; +}; + +done_testing; From 4cb6d6995f4b07971078f684c035fa343b0cc567 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 15 Sep 2010 15:27:26 +0900 Subject: [PATCH 192/259] Make the code clearer --- perl/xs-src/unpack.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 9ccc44ad..9219ed26 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -37,7 +37,6 @@ get_bool (const char *name) { } /* ---------------------------------------------------------------------- */ - struct template_context; typedef struct template_context msgpack_unpack_t; @@ -166,6 +165,7 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, { dTHX; AV* const a = (AV*)SvRV(*c); + assert(SvTYPE(a) == SVt_PVAV); (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o) SvREFCNT_inc_simple_void_NN(o); return 0; @@ -182,7 +182,9 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v) { dTHX; - (void)hv_store_ent((HV*)SvRV(*c), k, v, 0); + HV* const h = (HV*)SvRV(*c); + assert(SvTYPE(h) == SVt_PVHV); + (void)hv_store_ent(h, k, v, 0); SvREFCNT_inc_simple_void_NN(v); return 0; } @@ -321,12 +323,10 @@ XS(xs_unpacker_execute) { ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data)); - { - SV * d2 = template_data(mp); - if (!mp->user.incremented && d2) { - SvREFCNT_inc(d2); - mp->user.incremented = true; - } + if (!mp->user.incremented) { + SV* tmp_obj = template_data(mp); + SvREFCNT_inc_simple_void_NN(tmp_obj); + mp->user.incremented = true; } } @@ -368,7 +368,7 @@ XS(xs_unpacker_data) { } UNPACKER(ST(0), mp); - ST(0) = sv_mortalcopy(template_data(mp)); + ST(0) = template_data(mp); XSRETURN(1); } @@ -380,10 +380,9 @@ XS(xs_unpacker_reset) { } UNPACKER(ST(0), mp); - { - SV * data = template_data(mp); - SvREFCNT_dec(data); - } + + SV* const data = template_data(mp); + SvREFCNT_dec(data); _reset(ST(0)); XSRETURN(0); @@ -396,10 +395,9 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); - SV * data = template_data(mp); - if (SvOK(data)) { - SvREFCNT_dec(data); - } + + SV* const data = template_data(mp); + SvREFCNT_dec(data); Safefree(mp); XSRETURN(0); From 9eeb702ca536f0e97e53202b177d9db5ac9e322f Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Wed, 15 Sep 2010 22:28:46 +0900 Subject: [PATCH 193/259] change an annotation-utility in Java. it allows users to pack and unpack a List object. --- .../annotation/MessagePackUnpackable.java | 2 +- .../util/annotation/PackUnpackUtil.java | 705 ++++++++++-------- .../annotation/TestMessagePackUnpackable.java | 105 ++- 3 files changed, 499 insertions(+), 313 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java index 473b5412..77f6e6de 100644 --- a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java @@ -6,6 +6,6 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Target(ElementType.TYPE) -@Retention(RetentionPolicy.CLASS) +@Retention(RetentionPolicy.RUNTIME) public @interface MessagePackUnpackable { } diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index f92c51a0..52384448 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -3,9 +3,14 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; import java.math.BigInteger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -14,7 +19,6 @@ import javassist.CannotCompileException; import javassist.ClassPool; import javassist.CtClass; import javassist.CtConstructor; -import javassist.CtField; import javassist.CtMethod; import javassist.CtNewConstructor; import javassist.CtNewMethod; @@ -33,51 +37,16 @@ public class PackUnpackUtil { static final String KEYWORD_MODIFIER_PUBLIC = "public"; + static final String KEYWORD_FOR = "for"; + static final String KEYWORD_THROWS = "throws"; + static final String KEYWORD_NEW = "new"; + static final String TYPE_NAME_VOID = void.class.getName(); - static final String TYPE_NAME_BOOLEAN = boolean.class.getName(); - - static final String TYPE_NAME_BYTE = byte.class.getName(); - - static final String TYPE_NAME_DOUBLE = double.class.getName(); - - static final String TYPE_NAME_FLOAT = float.class.getName(); - - static final String TYPE_NAME_INT = int.class.getName(); - - static final String TYPE_NAME_LONG = long.class.getName(); - - static final String TYPE_NAME_SHORT = short.class.getName(); - static final String TYPE_NAME_OBJECT = Object.class.getName(); - static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName(); - - static final String TYPE_NAME_STRING = String.class.getName(); - - static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName(); - - static final String TYPE_NAME_BYTE2 = Byte.class.getName(); - - static final String TYPE_NAME_DOUBLE2 = Double.class.getName(); - - static final String TYPE_NAME_FLOAT2 = Float.class.getName(); - - static final String TYPE_NAME_INT2 = Integer.class.getName(); - - static final String TYPE_NAME_LONG2 = Long.class.getName(); - - static final String TYPE_NAME_SHORT2 = Short.class.getName(); - - //static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); - static final String TYPE_NAME_BYTEARRAY = "byte[]"; - - static final String TYPE_NAME_LIST = List.class.getName(); - - static final String TYPE_NAME_MAP = Map.class.getName(); - static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); static final String TYPE_NAME_PACKER = Packer.class.getName(); @@ -111,6 +80,10 @@ public class PackUnpackUtil { static final String CHAR_NAME_EQUAL = "="; + static final String CHAR_NAME_PLUS = "+"; + + static final String CHAR_NAME_LESSTHAN = "<"; + static final String CHAR_NAME_RIGHT_PARENTHESIS = ")"; static final String CHAR_NAME_LEFT_PARENTHESIS = "("; @@ -127,8 +100,16 @@ public class PackUnpackUtil { static final String VARIABLE_NAME_OBJ = "obj"; + static final String VARIABLE_NAME_SIZE = "len"; + + static final String VARIABLE_NAME_I = "i"; + static final String METHOD_NAME_VALUEOF = "valueOf"; + static final String METHOD_NAME_ADD = "add"; + + static final String METHOD_NAME_PUT = "put"; + static final String METHOD_NAME_MSGPACK = "messagePack"; static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; @@ -192,73 +173,68 @@ public class PackUnpackUtil { String origName = origClass.getName(); String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; CtClass origCtClass = pool.get(origName); - checkClassValidation(origCtClass); - checkDefaultConstructorValidation(origCtClass); + checkClassValidation(origClass); + checkDefaultConstructorValidation(origClass); CtClass enhCtClass = pool.makeClass(enhName); setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); addConstructor(enhCtClass); - CtField[] fields = getDeclaredFields(origCtClass); + Field[] fields = getDeclaredFields(origClass); addMessagePackMethod(enhCtClass, origCtClass, fields); addMessageUnpackMethod(enhCtClass, origCtClass, fields); addMessageConvertMethod(enhCtClass, origCtClass, fields); return createClass(enhCtClass); } - private void checkClassValidation(CtClass origCtClass) { + private void checkClassValidation(Class origClass) { // not public, abstract, final - int mod = origCtClass.getModifiers(); - if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod) - || Modifier.isFinal(mod)) { - throwClassValidationException(origCtClass); + int mod = origClass.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isAbstract(mod) || Modifier.isFinal(mod)) { + throwClassValidationException(origClass); } // interface, enum - if (origCtClass.isInterface() || origCtClass.isEnum()) { - throwClassValidationException(origCtClass); + if (origClass.isInterface() || origClass.isEnum()) { + throwClassValidationException(origClass); } // annotation - checkPackUnpackAnnotation(origCtClass); + checkPackUnpackAnnotation(origClass); } - private static void throwClassValidationException(CtClass origCtClass) { + private static void throwClassValidationException(Class origClass) { throw new PackUnpackUtilException( "it must be a public class and have @" + MessagePackUnpackable.class.getName() + ": " - + origCtClass.getName()); + + origClass.getName()); } - private void checkPackUnpackAnnotation(CtClass origCtClass) { - try { - Object[] objs = origCtClass.getAnnotations(); - for (Object obj : objs) { - if (obj instanceof MessagePackUnpackable) { - return; - } - } - throwClassValidationException(origCtClass); - } catch (ClassNotFoundException e) { - throw new PackUnpackUtilException(e.getMessage(), e); + private void checkPackUnpackAnnotation(Class origClass) { + Annotation anno = origClass + .getAnnotation(MessagePackUnpackable.class); + if (anno == null) { + throwClassValidationException(origClass); } } - private void checkDefaultConstructorValidation(CtClass origCtClass) { - CtConstructor cons = null; + private void checkDefaultConstructorValidation(Class origClass) { + Constructor cons = null; try { - cons = origCtClass.getDeclaredConstructor(new CtClass[0]); - } catch (NotFoundException e) { - throwConstructoValidationException(origCtClass); + cons = origClass.getDeclaredConstructor(new Class[0]); + } catch (Exception e1) { + throwConstructoValidationException(origClass); } + int mod = cons.getModifiers(); if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) { - throwConstructoValidationException(origCtClass); + throwConstructoValidationException(origClass); } } private static void throwConstructoValidationException( - CtClass origCtClass) { + Class origClass) { throw new PackUnpackUtilException( "it must have a public zero-argument constructor: " - + origCtClass.getName()); + + origClass.getName()); } private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) @@ -282,301 +258,430 @@ public class PackUnpackUtil { enhCtClass.addConstructor(newCtCons); } - private CtField[] getDeclaredFields(CtClass origCtClass) { - ArrayList allFields = new ArrayList(); - try { - CtClass nextCtClass = origCtClass; - while (!nextCtClass - .equals(pool.get(Constants.TYPE_NAME_OBJECT))) { - CtField[] fields = nextCtClass.getDeclaredFields(); - for (CtField field : fields) { - try { - checkFieldValidation(field, allFields); - allFields.add(field); - } catch (PackUnpackUtilException e) { // ignore - } + private Field[] getDeclaredFields(Class origClass) { + ArrayList allFields = new ArrayList(); + Class nextClass = origClass; + while (nextClass != Object.class) { + Field[] fields = nextClass.getDeclaredFields(); + for (Field field : fields) { + try { + checkFieldValidation(field, allFields); + allFields.add(field); + } catch (PackUnpackUtilException e) { // ignore } - nextCtClass = nextCtClass.getSuperclass(); } - - } catch (NotFoundException e) { - throw new PackUnpackUtilException(e.getMessage(), e); + nextClass = nextClass.getSuperclass(); } - return allFields.toArray(new CtField[0]); + return allFields.toArray(new Field[0]); } - private void checkFieldValidation(CtField field, - ArrayList allFields) { + private void checkFieldValidation(Field field, List fields) { // check modifiers (public or protected) int mod = field.getModifiers(); if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) || Modifier.isStatic(mod) || Modifier.isFinal(mod) - || Modifier.isTransient(mod)) { + || Modifier.isTransient(mod) || field.isSynthetic()) { throwFieldValidationException(field); } // check same name - for (CtField f : allFields) { + for (Field f : fields) { if (f.getName().equals(field.getName())) { throwFieldValidationException(field); } } } - private static void throwFieldValidationException(CtField field) { + private static void throwFieldValidationException(Field field) { throw new PackUnpackUtilException("it must be a public field: " + field.getName()); } private void addMessagePackMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); - sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( - Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.METHOD_NAME_MSGPACK).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.TYPE_NAME_PACKER).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.KEYWORD_THROWS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_IOEXCEPTION).append( - Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( - Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append( - Constants.METHOD_NAME_PACKARRAY).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length) - .append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - for (CtField field : fields) { - insertCodeOfMessagePack(field, sb); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_PACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(fields.length); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessagePack(sb, field); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messagePack method: " + sb.toString()); + //System.out.println("messagePack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessagePack(CtField field, StringBuilder sb) - throws NotFoundException { - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK) - .append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - field.getName()).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); + private void insertCodeOfMessagePack(StringBuilder sb, Field field) { + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } private void addMessageUnpackMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); - sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( - Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.METHOD_NAME_MSGUNPACK).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.TYPE_NAME_UNPACKER).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.KEYWORD_THROWS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( - Constants.CHAR_NAME_COMMA).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_IOEXCEPTION).append( - Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( - Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append( - Constants.METHOD_NAME_UNPACKARRAY).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - for (CtField field : fields) { - insertCodeOfMessageUnpack(field, sb); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGUNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_UNPACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessageUnpack(sb, field, field.getType()); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messageUnpack method: " + sb.toString()); + //System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) - throws NotFoundException { - CtClass type = field.getType(); - insertRightVariable(sb, field, type); - insertValueOfMethodAndLeftParenthesis(sb, type); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT); - insertUnpackMethod(sb, type); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - insertUnpackMethodArgumenet(sb, field, type); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - insertValueOfMethodAndRightParenthesis(sb, type); - sb.append(Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - - } - - private void insertRightVariable(StringBuilder sb, CtField field, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) - .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - || type - .equals(pool - .get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger - || type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String - || type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[] - || type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List - || type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map - ) { - sb.append(field.getName()) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); - } else { // MessageUnpackable - return; - } - } - - } - - private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - ) { - sb.append(type.getName()).append(Constants.CHAR_NAME_DOT) - .append(Constants.METHOD_NAME_VALUEOF).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS); - } else { - return; - } + private void insertCodeOfMessageUnpack(StringBuilder sb, Field field, + Class type) throws NotFoundException { + if (type.isPrimitive()) { + // primitive type + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (type.equals(Boolean.class) || // Boolean + type.equals(Byte.class) || // Byte + type.equals(Double.class) || // Double + type.equals(Float.class) || // Float + type.equals(Integer.class) || // Integer + type.equals(Long.class) || // Long + type.equals(Short.class)) { // Short + // reference type (wrapper type) + insertCodeOfMessageUnpackForWrapperTypes(sb, field, type); + } else if (type.equals(BigInteger.class) || // BigInteger + type.equals(String.class) || // String + type.equals(byte[].class)) { // byte[] + // reference type (other type) + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (List.class.isAssignableFrom(type)) { + // List + insertCodeOfMessageUnpackForListType(sb, field, type); + } else if (Map.class.isAssignableFrom(type)) { + // Map + insertCodeOfMessageUnpackForMapType(sb, field, type); + } else if (MessageUnpackable.class.isAssignableFrom(type) + || (getCache(type.getName()) != null)) { + // MessageUnpackable + insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type); + } else { + throw new NotFoundException("unknown type: " + type.getName()); } } - private void insertUnpackMethod(StringBuilder sb, CtClass type) + private void insertCodeOfMessageUnpackForPrimitiveTypes( + StringBuilder sb, Field field, Class type) throws NotFoundException { - if (type.equals(CtClass.booleanType)) { // boolean + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert unpack method + if (type.equals(boolean.class)) { // boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); - } else if (type.equals(CtClass.byteType)) { // byte + } else if (type.equals(byte.class)) { // byte sb.append(Constants.METHOD_NAME_UNPACKBYTE); - } else if (type.equals(CtClass.doubleType)) { // double + } else if (type.equals(double.class)) { // double sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); - } else if (type.equals(CtClass.floatType)) { // float + } else if (type.equals(float.class)) { // float sb.append(Constants.METHOD_NAME_UNPACKFLOAT); - } else if (type.equals(CtClass.intType)) { // int + } else if (type.equals(int.class)) { // int sb.append(Constants.METHOD_NAME_UNPACKINT); - } else if (type.equals(CtClass.longType)) { // long + } else if (type.equals(long.class)) { // long sb.append(Constants.METHOD_NAME_UNPACKLONG); - } else if (type.equals(CtClass.shortType)) { // short + } else if (type.equals(short.class)) { // short sb.append(Constants.METHOD_NAME_UNPACKSHORT); } else { // reference type - Class c = null; - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean - sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); - } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte - sb.append(Constants.METHOD_NAME_UNPACKBYTE); - } else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double - sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); - } else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float - sb.append(Constants.METHOD_NAME_UNPACKFLOAT); - } else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer - sb.append(Constants.METHOD_NAME_UNPACKINT); - } else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long - sb.append(Constants.METHOD_NAME_UNPACKLONG); - } else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short - sb.append(Constants.METHOD_NAME_UNPACKSHORT); - } else if (type - .equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger + if (type.equals(BigInteger.class)) { // BigInteger sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER); - } else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String + } else if (type.equals(String.class)) { // String sb.append(Constants.METHOD_NAME_UNPACKSTRING); - } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[] + } else if (type.equals(byte[].class)) { // byte[] sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY); - } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List - sb.append(Constants.METHOD_NAME_UNPACKARRAY); - } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map - sb.append(Constants.METHOD_NAME_UNPACKMAP); - } else if (type.subtypeOf(pool - .get(Constants.TYPE_NAME_MSGUNPACKABLE)) - || ((c = getCache(type.getName())) != null)) { // MessageUnpackable - sb.append(Constants.METHOD_NAME_UNPACK); } else { throw new NotFoundException("unknown type: " + type.getName()); } } - } - - private void insertUnpackMethodArgumenet(StringBuilder sb, - CtField field, CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - Class c = null; - if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE)) - || ((c = getCache(type.getName())) != null)) { - sb.append("(org.msgpack.MessageUnpackable)"); - sb.append(field.getName()); - } else { - return; - } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } } - private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - ) { - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - } else { - return; - } + private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb, + Field field, Class type) throws NotFoundException { + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); } + sb.append(type.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_VALUEOF); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert the name of a unpack method + if (type.equals(Boolean.class)) { // Boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(Byte.class)) { // Byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(Double.class)) { // Double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(Float.class)) { // Float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(Integer.class)) { // Integer + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(Long.class)) { // Long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(Short.class)) { // Short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { + throw new NotFoundException("unknown type: " + type.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + + private void insertCodeOfMessageUnpackForListType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType = (Class) generic.getActualTypeArguments()[0]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(ArrayList.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ADD); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMapType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType0 = (Class) generic.getActualTypeArguments()[0]; + Class genericType1 = (Class) generic.getActualTypeArguments()[1]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKMAP); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(HashMap.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block map. + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PUT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType0); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessageUnpack(sb, null, genericType1); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMsgUnpackableType( + StringBuilder sb, Field field, Class type) { + // insert a right variable // ignore + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageUnpackable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } private void addMessageConvertMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( @@ -597,7 +702,7 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE); // TODO sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messageConvert method: " + sb.toString()); + //System.out.println("messageConvert method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } @@ -684,6 +789,6 @@ public class PackUnpackUtil { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); - System.out.println(src.equals(dst)); + //System.out.println(src.equals(dst)); } } diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 1cde65f2..dbea7cb2 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -3,6 +3,9 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -16,9 +19,9 @@ import org.msgpack.Unpacker; public class TestMessagePackUnpackable extends TestCase { @Test - public void testGeneralPrimitiveTypeFields() throws Exception { - GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil - .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + public void testPrimitiveTypeFields() throws Exception { + PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); src.f0 = (byte) 0; src.f1 = 1; src.f2 = 2; @@ -28,8 +31,8 @@ public class TestMessagePackUnpackable extends TestCase { src.f6 = false; ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(src); - GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil - .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); @@ -43,7 +46,7 @@ public class TestMessagePackUnpackable extends TestCase { } @MessagePackUnpackable - public static class GeneralPrimitiveTypeFieldsClass { + public static class PrimitiveTypeFieldsClass { public byte f0; public short f1; public int f2; @@ -52,7 +55,7 @@ public class TestMessagePackUnpackable extends TestCase { public double f5; public boolean f6; - public GeneralPrimitiveTypeFieldsClass() { + public PrimitiveTypeFieldsClass() { } } @@ -107,15 +110,93 @@ public class TestMessagePackUnpackable extends TestCase { } } - public void testListAndMap() throws Exception { + public void testListTypes() throws Exception { + SampleListTypes src = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + src.f0 = new ArrayList(); + src.f1 = new ArrayList(); + src.f1.add(1); + src.f1.add(2); + src.f1.add(3); + src.f2 = new ArrayList(); + src.f2.add("e1"); + src.f2.add("e2"); + src.f2.add("e3"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleListTypes dst = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + for (int i = 0; i < src.f1.size(); ++i) { + assertEquals(src.f1.get(i), dst.f1.get(i)); + } + assertEquals(src.f2.size(), dst.f2.size()); + for (int i = 0; i < src.f2.size(); ++i) { + assertEquals(src.f2.get(i), dst.f2.get(i)); + } } @MessagePackUnpackable - public static class ListAndMapClass { - public List f0; - public Map f1; + public static class SampleListTypes { + public List f0; + public List f1; + public List f2; - public ListAndMapClass() { + public SampleListTypes() { + } + } + + public void testMapTypes() throws Exception { + SampleMapTypes src = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + src.f0 = new HashMap(); + src.f1 = new HashMap(); + src.f1.put(1, 1); + src.f1.put(2, 2); + src.f1.put(3, 3); + src.f2 = new HashMap(); + src.f2.put("k1", 1); + src.f2.put("k2", 2); + src.f2.put("k3", 3); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + Iterator srcf1 = src.f1.keySet().iterator(); + Iterator dstf1 = dst.f1.keySet().iterator(); + while (srcf1.hasNext()) { + Integer s1 = srcf1.next(); + Integer d1 = dstf1.next(); + assertEquals(s1, d1); + assertEquals(src.f1.get(s1), dst.f1.get(d1)); + } + assertEquals(src.f2.size(), dst.f2.size()); + Iterator srcf2 = src.f2.keySet().iterator(); + Iterator dstf2 = dst.f2.keySet().iterator(); + while (srcf2.hasNext()) { + String s2 = srcf2.next(); + String d2 = dstf2.next(); + assertEquals(s2, d2); + assertEquals(src.f2.get(s2), dst.f2.get(d2)); + } + } + + @MessagePackUnpackable + public static class SampleMapTypes { + public Map f0; + public Map f1; + public Map f2; + + public SampleMapTypes() { } } From afbddbfcda1ee0acde3c5343a77c4c575f73ad95 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:24:01 +0900 Subject: [PATCH 194/259] Fix the stream unpacker --- perl/t/06_stream_unpack2.t | 33 ++--- perl/xs-src/MessagePack.c | 1 + perl/xs-src/pack.c | 2 +- perl/xs-src/unpack.c | 254 +++++++++++++++++-------------------- perl/xt/leaks/leaktrace.t | 22 +++- 5 files changed, 148 insertions(+), 164 deletions(-) diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index bc158cdd..68a28731 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -1,9 +1,9 @@ use strict; use warnings; use Data::MessagePack; -use Test::More tests => 6; +use Test::More tests => 9; -my $input = [42, "foo", { x => [ (undef)x16 ] }, 3.14 ]; +my $input = [ 42, "foo", { x => [ (1) x 16 ] }, undef, 1 ]; my $packed = Data::MessagePack->pack($input); is_deeply(Data::MessagePack->unpack($packed), $input); @@ -26,24 +26,19 @@ is_deeply(Data::MessagePack->unpack($packed), $input); { my $up = Data::MessagePack::Unpacker->new(); - my $offset = 0; - my $size = 5; + my $size = 8; - note "length: ", length($packed); - while(not $up->is_finished) { - note "offset: ", $offset; - my $bytes = substr($packed, $offset, $size); - note join " ", map { unpack 'H2', $_ } split //, $bytes; - my $x = $up->execute($bytes, 0); - if($x <= 0) { - diag "Something's wrong: $x"; - last; - } - else { - $offset += $x; - } + note "packed size: ", length($packed); + open my $stream, '<:bytes :scalar', \$packed; + my $buff; + while( read($stream, $buff, $size) ) { + note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; + + $up->execute($buff); } - ok $up->is_finished; - is_deeply $up->data, $input; + ok $up->is_finished, 'is_finished'; + my $data = $up->data; + note explain($data); + is_deeply $data, $input; } diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c index 259fd156..aba8ef91 100644 --- a/perl/xs-src/MessagePack.c +++ b/perl/xs-src/MessagePack.c @@ -14,6 +14,7 @@ void boot_Data__MessagePack_pack(void); XS(boot_Data__MessagePack) { dXSARGS; + PERL_UNUSED_VAR(items); boot_Data__MessagePack_pack(); diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 5eaf17e5..9a58ed05 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -145,7 +145,7 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) { } -static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); +STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) { dTHX; diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 9219ed26..a429ecd9 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -26,14 +26,9 @@ typedef struct { /* utility functions */ STATIC_INLINE SV * -get_bool (const char *name) { +get_bool (const char* const name) { dTHX; - SV * sv = sv_mortalcopy(get_sv( name, 1 )); - - SvREADONLY_on(sv); - SvREADONLY_on( SvRV(sv) ); - - return sv; + return newSVsv(get_sv( name, GV_ADD )); } /* ---------------------------------------------------------------------- */ @@ -49,85 +44,76 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL, STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL) { - dTHX; - return &PL_sv_undef; + return NULL; } -STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o) +#if IVSIZE == 4 + +STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o) { dTHX; - *o = sv_2mortal(newSVuv(d)); + *o = newSVuv(d); return 0; } -STATIC_INLINE int template_callback_uint16(unpack_user* u PERL_UNUSED_DECL, uint16_t d, SV** o) +STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - *o = sv_2mortal(newSVuv(d)); + *o = newSVnv((NV)d); return 0; } -STATIC_INLINE int template_callback_uint32(unpack_user* u PERL_UNUSED_DECL, uint32_t d, SV** o) +STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o) { dTHX; - *o = sv_2mortal(newSVuv(d)); + *o = newSViv(d); return 0; } -STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t d, SV** o) +STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) { dTHX; -#if IVSIZE==4 - *o = sv_2mortal(newSVnv(d)); -#else - *o = sv_2mortal(newSVuv(d)); -#endif + *o = newSVnv((NV)d); return 0; } -STATIC_INLINE int template_callback_int8(unpack_user* u PERL_UNUSED_DECL, int8_t d, SV** o) +#else /* IVSIZE == 8 */ + + +STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o) { dTHX; - *o = sv_2mortal(newSViv(d)); + *o = newSVuv(d); return 0; } -STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16_t d, SV** o) +#define template_callback_uint64 template_callback_UV + +STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o) { dTHX; - *o = sv_2mortal(newSViv(d)); + *o = newSViv(d); return 0; } -STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32_t d, SV** o) -{ - dTHX; - *o = sv_2mortal(newSViv(d)); - return 0; -} +#define template_callback_uint64 template_callback_IV -STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o) -{ - dTHX; -#if IVSIZE==4 - *o = sv_2mortal(newSVnv(d)); -#else - *o = sv_2mortal(newSViv(d)); -#endif - return 0; -} +#endif /* IVSIZE */ -STATIC_INLINE int template_callback_float(unpack_user* u PERL_UNUSED_DECL, float d, SV** o) -{ - dTHX; - *o = sv_2mortal(newSVnv(d)); - return 0; -} +#define template_callback_uint8 template_callback_UV +#define template_callback_uint16 template_callback_UV +#define template_callback_uint32 template_callback_UV + +#define template_callback_int8 template_callback_IV +#define template_callback_int16 template_callback_IV +#define template_callback_int32 template_callback_IV + +#define template_callback_float template_callback_double STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o) { dTHX; - *o = sv_2mortal(newSVnv(d)); + *o = newSVnv(d); return 0; } @@ -135,7 +121,7 @@ STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, doub STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o) { dTHX; - *o = sv_newmortal(); + *o = newSV(0); return 0; } @@ -148,7 +134,8 @@ STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o) { - dTHX; *o = get_bool("Data::MessagePack::false"); + dTHX; + *o = get_bool("Data::MessagePack::false"); return 0; } @@ -156,7 +143,7 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig { dTHX; AV* const a = newAV(); - *o = sv_2mortal(newRV_noinc((SV*)a)); + *o = newRV_noinc((SV*)a); av_extend(a, n + 1); return 0; } @@ -167,7 +154,6 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, AV* const a = (AV*)SvRV(*c); assert(SvTYPE(a) == SVt_PVAV); (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o) - SvREFCNT_inc_simple_void_NN(o); return 0; } @@ -175,7 +161,7 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne { dTHX; HV* const h = newHV(); - *o = sv_2mortal(newRV_noinc((SV*)h)); + *o = newRV_noinc((SV*)h); return 0; } @@ -185,7 +171,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV HV* const h = (HV*)SvRV(*c); assert(SvTYPE(h) == SVt_PVHV); (void)hv_store_ent(h, k, v, 0); - SvREFCNT_inc_simple_void_NN(v); + SvREFCNT_dec(k); return 0; } @@ -193,10 +179,12 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c { dTHX; /* newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */ - *o = ((l==0) ? newSVpvs_flags("", SVs_TEMP) : newSVpvn_flags(p, l, SVs_TEMP)); + *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l)); return 0; } +#include "msgpack/unpack_template.h" + #define UNPACKER(from, name) \ msgpack_unpack_t *name; \ if(!(SvROK(from) && SvIOK(SvRV(from)))) { \ @@ -207,36 +195,6 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ } -#include "msgpack/unpack_template.h" - -STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) { - msgpack_unpack_t mp; - dTHX; - unpack_user u = {false, false}; - int ret; - size_t from = 0; - STRLEN dlen; - const char * dptr = SvPV_const(data, dlen); - SV* obj; - - template_init(&mp); - mp.user = u; - - ret = template_execute(&mp, dptr, (size_t)dlen, &from); - obj = template_data(&mp); - - if(ret < 0) { - Perl_croak(aTHX_ "parse error."); - } else if(ret == 0) { - Perl_croak(aTHX_ "insufficient bytes."); - } else { - if(from < dlen) { - Perl_croak(aTHX_ "extra bytes."); - } - return obj; - } -} - XS(xs_unpack) { dXSARGS; SV* const data = ST(1); @@ -252,17 +210,40 @@ XS(xs_unpack) { Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])"); } - ST(0) = _msgpack_unpack(data, limit); + STRLEN dlen; + const char* const dptr = SvPV_const(data, dlen); + msgpack_unpack_t mp; + template_init(&mp); + + unpack_user const u = {false, false}; + mp.user = u; + + size_t from = 0; + int const ret = template_execute(&mp, dptr, (size_t)dlen, &from); + SV* const obj = template_data(&mp); + sv_2mortal(obj); + + if(ret < 0) { + Perl_croak(aTHX_ "Data::MessagePack->unpack: parse error"); + } else if(ret == 0) { + Perl_croak(aTHX_ "Data::MessagePack->unpack: insufficient bytes"); + } else { + if(from < dlen) { + Perl_croak(aTHX_ "Data::MessagePack->unpack: extra bytes"); + } + } + + ST(0) = obj; XSRETURN(1); } /* ------------------------------ stream -- */ /* http://twitter.com/frsyuki/status/13249304748 */ -STATIC_INLINE void _reset(SV* self) { +STATIC_INLINE void _reset(SV* const self) { dTHX; - unpack_user u = {false, false}; + unpack_user const u = {false, false}; UNPACKER(self, mp); template_init(mp); @@ -275,10 +256,10 @@ XS(xs_unpacker_new) { Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); } - SV* self = sv_newmortal(); - msgpack_unpack_t *mp; + SV* const self = sv_newmortal(); + msgpack_unpack_t *mp; - Newx(mp, 1, msgpack_unpack_t); + Newxz(mp, 1, msgpack_unpack_t); sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); _reset(self); @@ -287,65 +268,64 @@ XS(xs_unpacker_new) { XSRETURN(1); } -STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { +STATIC_INLINE SV* +_execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) { dTHX; + + if(offset >= limit) { + Perl_croak(aTHX_ "offset (%"UVuf") is bigger than data buffer size (%"UVuf")", + offset, limit); + } + UNPACKER(self, mp); - size_t from = off; - const char* dptr = SvPV_nolen_const(data); - int ret; + size_t from = offset; + const char* const dptr = SvPV_nolen_const(data); - if(from >= limit) { - Perl_croak(aTHX_ "offset (%lu) is bigger than data buffer size (%lu)", (unsigned long)off, (unsigned long)limit); - } + int const ret = template_execute(mp, dptr, limit, &from); - ret = template_execute(mp, dptr, limit, &from); - - if(ret < 0) { - Perl_croak(aTHX_ "parse error."); - } else { - mp->user.finished = (ret > 0) ? true : false; - return sv_2mortal(newSVuv(from)); - } + if(ret < 0) { + Perl_croak(aTHX_ "Data::MessagePack::Unpacker: parse error while executing"); + } else { + mp->user.finished = (ret > 0) ? true : false; + return sv_2mortal(newSVuv(from)); + } } XS(xs_unpacker_execute) { dXSARGS; - if (items != 3) { - Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)"); + SV* const self = ST(0); + SV* const data = ST(1); + UV offset; + + if (items == 2) { + offset = 0; + } + else if (items == 3) { + offset = SvUVx(ST(2)); + } + else { + Perl_croak(aTHX_ "Usage: $unpacker->execute(data, offset = 0)"); } - UNPACKER(ST(0), mp); - { - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); /* offset of $data. normaly, 0. */ - - ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data)); - - if (!mp->user.incremented) { - SV* tmp_obj = template_data(mp); - SvREFCNT_inc_simple_void_NN(tmp_obj); - mp->user.incremented = true; - } - } + UNPACKER(self, mp); + ST(0) = _execute_impl(self, data, offset, sv_len(data)); XSRETURN(1); } XS(xs_unpacker_execute_limit) { dXSARGS; if (items != 4) { - Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, offset, limit)"); } - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); - IV limit = SvIV(ST(3)); - - ST(0) = _execute_impl(self, data, off, (size_t)limit); + SV* const self = ST(0); + SV* const data = ST(1); + UV const offset = SvUVx(ST(2)); + UV const limit = SvUVx(ST(3)); + ST(0) = _execute_impl(self, data, offset, limit); XSRETURN(1); } @@ -355,9 +335,8 @@ XS(xs_unpacker_is_finished) { Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); } - UNPACKER(ST(0), mp); + UNPACKER(ST(0), mp); ST(0) = boolSV(mp->user.finished); - XSRETURN(1); } @@ -367,9 +346,8 @@ XS(xs_unpacker_data) { Perl_croak(aTHX_ "Usage: $unpacker->data()"); } - UNPACKER(ST(0), mp); - ST(0) = template_data(mp); - + UNPACKER(ST(0), mp); + ST(0) = template_data(mp); XSRETURN(1); } @@ -379,10 +357,10 @@ XS(xs_unpacker_reset) { Perl_croak(aTHX_ "Usage: $unpacker->reset()"); } - UNPACKER(ST(0), mp); + UNPACKER(ST(0), mp); SV* const data = template_data(mp); - SvREFCNT_dec(data); + sv_2mortal(data); _reset(ST(0)); XSRETURN(0); @@ -394,10 +372,10 @@ XS(xs_unpacker_destroy) { Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()"); } - UNPACKER(ST(0), mp); + UNPACKER(ST(0), mp); SV* const data = template_data(mp); - SvREFCNT_dec(data); + sv_2mortal(data); Safefree(mp); XSRETURN(0); diff --git a/perl/xt/leaks/leaktrace.t b/perl/xt/leaks/leaktrace.t index ff64f6ff..1836ad6b 100755 --- a/perl/xt/leaks/leaktrace.t +++ b/perl/xt/leaks/leaktrace.t @@ -5,7 +5,8 @@ use Test::More; use Data::MessagePack; -my $data = { +my $simple_data = "xyz"; +my $complex_data = { a => 'foo', b => 42, c => undef, @@ -13,8 +14,10 @@ my $data = { e => 3.14, }; +note 'pack'; + no_leaks_ok { - my $s = Data::MessagePack->pack($data); + my $s = Data::MessagePack->pack($complex_data); }; no_leaks_ok { @@ -23,16 +26,23 @@ no_leaks_ok { $@ or die "it must die"; }; -my $s = Data::MessagePack->pack($data); +note 'unpack'; + +my $s = Data::MessagePack->pack($simple_data); +my $c = Data::MessagePack->pack($complex_data); no_leaks_ok { my $data = Data::MessagePack->unpack($s); }; no_leaks_ok { - my $ss = $s; - chop $ss; - eval { Data::MessagePack->unpack($ss) }; + my $data = Data::MessagePack->unpack($c); +}; + +no_leaks_ok { + my $broken = $s; + chop $broken; + eval { Data::MessagePack->unpack($broken) }; #note $@; $@ or die "it must die"; }; From 7c1e0ea95d1aed954ee97bc3a72c7779e085f9ec Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:27:25 +0900 Subject: [PATCH 195/259] Add binmode() for stream unpacking --- perl/t/06_stream_unpack2.t | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index 68a28731..78ca8f7c 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -30,6 +30,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input); note "packed size: ", length($packed); open my $stream, '<:bytes :scalar', \$packed; + binmode $stream; my $buff; while( read($stream, $buff, $size) ) { note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; From bd887b660d9bcbdb6dd3458223331eea8dcdc654 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:31:34 +0900 Subject: [PATCH 196/259] Preallocate hv keys --- perl/xs-src/unpack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index a429ecd9..1d7de711 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -157,10 +157,11 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, return 0; } -STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o) +STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o) { dTHX; HV* const h = newHV(); + hv_ksplit(h, n); *o = newRV_noinc((SV*)h); return 0; } From e239bfda8add9d7fd42dcd5f2e2bfc01eae9e824 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:36:07 +0900 Subject: [PATCH 197/259] Make leaktrace.t as a regular test --- .../{xt/leaks/leaktrace.t => t/50_leaktrace.t} | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) rename perl/{xt/leaks/leaktrace.t => t/50_leaktrace.t} (72%) diff --git a/perl/xt/leaks/leaktrace.t b/perl/t/50_leaktrace.t similarity index 72% rename from perl/xt/leaks/leaktrace.t rename to perl/t/50_leaktrace.t index 1836ad6b..29485270 100755 --- a/perl/xt/leaks/leaktrace.t +++ b/perl/t/50_leaktrace.t @@ -1,6 +1,6 @@ #!perl -w use strict; -use Test::Requires 'Test::LeakTrace'; +use Test::Requires { 'Test::LeakTrace' => 0.13 }; use Test::More; use Data::MessagePack; @@ -22,8 +22,8 @@ no_leaks_ok { no_leaks_ok { eval { Data::MessagePack->pack([\*STDIN]) }; - #note $@; - $@ or die "it must die"; + note $@; + $@ or warn "# it must die"; }; note 'unpack'; @@ -43,8 +43,16 @@ no_leaks_ok { my $broken = $s; chop $broken; eval { Data::MessagePack->unpack($broken) }; - #note $@; - $@ or die "it must die"; + note $@; + $@ or warn "# it must die"; +}; + +note 'stream'; + +no_leaks_ok { + my $up = Data::MessagePack::Unpacker->new(); + $up->execute($c); + my $data = $up->data(); }; done_testing; From 3cffd46008c8a25dcc77818dbc4fbfd675923ea2 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:41:52 +0900 Subject: [PATCH 198/259] Fix a comment --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 1d7de711..da985e32 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -179,7 +179,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o) { dTHX; - /* newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */ + /* newSVpvn(p, l) returns an undef if p == NULL */ *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l)); return 0; } From 8eaed95e027bcf66b61611871bc12e0cb110d859 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 20:44:51 +0900 Subject: [PATCH 199/259] Fix an use of execute() --- perl/t/06_stream_unpack2.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index 78ca8f7c..25af21d4 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -35,7 +35,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input); while( read($stream, $buff, $size) ) { note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - $up->execute($buff); + $up->execute($buff, 0); } ok $up->is_finished, 'is_finished'; my $data = $up->data; From 562de7926b87a027842e624fe385468145d09ed9 Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 21:37:49 +0900 Subject: [PATCH 200/259] More tests; some fails now :( --- perl/.gitignore | 1 + perl/Makefile.PL | 5 +++++ perl/lib/Data/MessagePack.pm | 12 ++++++++++++ perl/t/06_stream_unpack2.t | 2 +- perl/t/09_stddata.t | 35 +++++++++++++++++++++++++++++++++++ ruby/test/test_pack_unpack.rb | 4 ++-- 6 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 perl/t/09_stddata.t diff --git a/perl/.gitignore b/perl/.gitignore index b64dcdfe..3e0e73e5 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -6,6 +6,7 @@ MessagePack.o blib/ inc/ msgpack/ +t/std/ pack.o pm_to_blib unpack.o diff --git a/perl/Makefile.PL b/perl/Makefile.PL index e7b8c474..fafc3876 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -54,6 +54,11 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { for my $src (<../msgpack/*.h>) { File::Copy::copy($src, 'msgpack/') or die "copy failed: $!"; } + + mkdir 't/std'; + for my $data(<../test/*.{json,mpac}>) { + File::Copy::copy($data, 't/std') or die "copy failed: $!"; + } } requires 'Test::More' => 0.94; # done_testing diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 3511628c..ece00505 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -6,6 +6,18 @@ use 5.008001; our $VERSION = '0.23'; our $PreferInteger = 0; +{ + package + Data::MessagePack::Boolean; + use overload + 'bool' => sub { ${ $_[0] } }, + '0+' => sub { ${ $_[0] } }, + '""' => sub { ${ $_[0] } ? 'true' : 'false' }, + + fallback => 1, + ; +} + our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; sub true () { $true } diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index 25af21d4..78ca8f7c 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -35,7 +35,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input); while( read($stream, $buff, $size) ) { note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - $up->execute($buff, 0); + $up->execute($buff); } ok $up->is_finished, 'is_finished'; my $data = $up->data; diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t new file mode 100644 index 00000000..d035b4be --- /dev/null +++ b/perl/t/09_stddata.t @@ -0,0 +1,35 @@ +use strict; +use Test::More; +use Test::Requires qw(JSON); +use t::Util; + +use Data::MessagePack; + +sub slurp { + open my $fh, '<:raw', $_[0] or die "failed to open '$_[0]': $!"; + local $/; + return scalar <$fh>; +} + +my @data = @{ JSON::decode_json(slurp("t/std/cases.json")) }; + +my $mpac1 = slurp("t/std/cases.mpac"); +my $mpac2 = slurp("t/std/cases_compact.mpac"); + +my $mps = Data::MessagePack::Unpacker->new(); + +my $t = 1; +for my $mpac($mpac1, $mpac2) { + note "mpac", $t++; + + my $offset = 0; + my $i = 0; + while($offset < length($mpac)) { + $offset += $mps->execute($mpac, $offset); + is_deeply $mps->data, $data[$i], "data[$i]"; + $mps->reset; + $i++; + } +} + +done_testing; diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index 545e5939..f378c3c7 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -239,7 +239,7 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "gc mark" do - obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + obj = [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5] num = 4 raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new @@ -257,7 +257,7 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "streaming backward compatibility" do - obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + obj = [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5] num = 4 raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new From 599964ea5f1a064163ddda8940bb8c5b12ef022a Mon Sep 17 00:00:00 2001 From: gfx Date: Thu, 16 Sep 2010 21:45:06 +0900 Subject: [PATCH 201/259] Comments --- perl/t/09_stddata.t | 3 +++ 1 file changed, 3 insertions(+) diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index d035b4be..b6a612eb 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -1,3 +1,6 @@ +#!perl -w +# Testing standard dataset in msgpack/test/*.{json,mpac}. +# Don't edit msgpack/perl/t/std/*, which are just copies. use strict; use Test::More; use Test::Requires qw(JSON); From 5e602fb575b0941194be42504475208ca7d8d6ad Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:10:54 +0900 Subject: [PATCH 202/259] Fix tests --- perl/t/06_stream_unpack2.t | 44 ++++++++++++++++++-------------------- perl/t/09_stddata.t | 2 +- perl/t/10_splitted_bytes.t | 40 ++++++++++++++++++++++++++++++++++ perl/t/Util.pm | 2 ++ 4 files changed, 64 insertions(+), 24 deletions(-) create mode 100755 perl/t/10_splitted_bytes.t diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index 78ca8f7c..bb6fe93d 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -2,8 +2,16 @@ use strict; use warnings; use Data::MessagePack; use Test::More tests => 9; +use t::Util; + +my $input = [ + false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1, + 127,127,255,65535,4294967295,-32,-32,-128,-32768, + -2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","", + [0],[0],[0],[],[],[],{},{},{}, + {"a" => 97},{"a" => 97},{"a" => 97},[[]],[["a"]] +]; -my $input = [ 42, "foo", { x => [ (1) x 16 ] }, undef, 1 ]; my $packed = Data::MessagePack->pack($input); is_deeply(Data::MessagePack->unpack($packed), $input); @@ -16,30 +24,20 @@ is_deeply(Data::MessagePack->unpack($packed), $input); { my $up = Data::MessagePack::Unpacker->new(); - is $up->execute(substr($packed, 0, 3), 0), 3; - ok !$up->is_finished; - $up->execute($packed, 3); - ok $up->is_finished; - is_deeply $up->data, $input; -} + $packed x= 3; + my $offset = 0; + for my $i(1 .. 3) { + note "block $i (offset: $offset/".length($packed).")"; + note "starting 3 bytes: ", join " ", map { unpack 'H2', $_ } + split //, substr($packed, $offset, 3); -{ - my $up = Data::MessagePack::Unpacker->new(); - my $size = 8; - - note "packed size: ", length($packed); - open my $stream, '<:bytes :scalar', \$packed; - binmode $stream; - my $buff; - while( read($stream, $buff, $size) ) { - note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - - $up->execute($buff); + $offset = $up->execute($packed, $offset); + ok $up->is_finished, 'finished'; + my $data = $up->data; + is_deeply $data, $input; + $up->reset(); } - ok $up->is_finished, 'is_finished'; - my $data = $up->data; - note explain($data); - is_deeply $data, $input; } + diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index b6a612eb..976fc5d3 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -28,7 +28,7 @@ for my $mpac($mpac1, $mpac2) { my $offset = 0; my $i = 0; while($offset < length($mpac)) { - $offset += $mps->execute($mpac, $offset); + $offset = $mps->execute($mpac, $offset); is_deeply $mps->data, $data[$i], "data[$i]"; $mps->reset; $i++; diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t new file mode 100755 index 00000000..232d8707 --- /dev/null +++ b/perl/t/10_splitted_bytes.t @@ -0,0 +1,40 @@ +#!perl + +# This feature is not yet supported, but 0.23 (or former) caused SEGV in this code, +# so we put it here. + +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my $input = [ + false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1, + 127,127,255,65535,4294967295,-32,-32,-128,-32768, + -2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","", + [0],[0],[0],[],[],[],{},{},{}, + {"a" => 97},{"a" => 97},{"a" => 97},[[]],[["a"]] +]; + +my $packed = Data::MessagePack->pack($input); + +foreach my $size(1 .. 16) { + local $TODO = "Splitted byte streaming is not yet supported (bufer size: $size)"; + + my $up = Data::MessagePack::Unpacker->new(); + + open my $stream, '<:bytes :scalar', \$packed; + binmode $stream; + my $buff; + while( read($stream, $buff, $size) ) { + #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; + + $up->execute($buff); + } + ok $up->is_finished, 'is_finished'; + my $data = $up->data; + is_deeply $data, $input; +} + +done_testing; diff --git a/perl/t/Util.pm b/perl/t/Util.pm index c8debefb..ad69c4d5 100644 --- a/perl/t/Util.pm +++ b/perl/t/Util.pm @@ -1,6 +1,7 @@ package t::Util; use strict; use warnings; +use Data::MessagePack; sub import { my $pkg = caller(0); @@ -15,6 +16,7 @@ sub import { *{"$pkg\::false"} = sub () { Data::MessagePack::false() }; + *{"$pkg\::null"} = sub() { undef }; } 1; From d2962d8676060c8fdd5d474db7497910293d4f8a Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:25:23 +0900 Subject: [PATCH 203/259] Split the boolean class into an outer module --- perl/lib/Data/MessagePack.pm | 27 +++++++------ perl/lib/Data/MessagePack/Boolean.pm | 14 +++++++ perl/xs-src/MessagePack.c | 10 ++++- perl/xs-src/pack.c | 3 +- perl/xs-src/unpack.c | 58 ++++++++++++++++++++++++---- 5 files changed, 87 insertions(+), 25 deletions(-) create mode 100755 perl/lib/Data/MessagePack/Boolean.pm diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index ece00505..b1e0174d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -6,22 +6,21 @@ use 5.008001; our $VERSION = '0.23'; our $PreferInteger = 0; -{ - package - Data::MessagePack::Boolean; - use overload - 'bool' => sub { ${ $_[0] } }, - '0+' => sub { ${ $_[0] } }, - '""' => sub { ${ $_[0] } ? 'true' : 'false' }, - - fallback => 1, - ; +sub true () { + require Data::MessagePack::Boolean; + no warnings 'once', 'redefine'; + my $t = $Data::MessagePack::Boolean::true; + *true = sub (){ $t }; + return $t; } -our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; -our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; -sub true () { $true } -sub false () { $false } +sub false () { + require Data::MessagePack::Boolean; + no warnings 'once', 'redefine'; + my $f = $Data::MessagePack::Boolean::false; + *false = sub (){ $f }; + return $f; +} if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; diff --git a/perl/lib/Data/MessagePack/Boolean.pm b/perl/lib/Data/MessagePack/Boolean.pm new file mode 100755 index 00000000..2bb3ecad --- /dev/null +++ b/perl/lib/Data/MessagePack/Boolean.pm @@ -0,0 +1,14 @@ +package Data::MessagePack::Boolean; +use strict; +use overload + 'bool' => sub { ${ $_[0] } }, + '0+' => sub { ${ $_[0] } }, + '""' => sub { ${ $_[0] } ? 'true' : 'false' }, + + fallback => 1, +; + +our $true = do { bless \(my $dummy = 1) }; +our $false = do { bless \(my $dummy = 0) }; + +1; diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c index aba8ef91..69337f41 100644 --- a/perl/xs-src/MessagePack.c +++ b/perl/xs-src/MessagePack.c @@ -1,5 +1,9 @@ #include "xshelper.h" +#ifndef __cplusplus +#include +#endif + XS(xs_pack); XS(xs_unpack); XS(xs_unpacker_new); @@ -10,13 +14,15 @@ XS(xs_unpacker_data); XS(xs_unpacker_reset); XS(xs_unpacker_destroy); -void boot_Data__MessagePack_pack(void); +void init_Data__MessagePack_pack(pTHX_ bool const cloning); +void init_Data__MessagePack_unpack(pTHX_ bool const cloning); XS(boot_Data__MessagePack) { dXSARGS; PERL_UNUSED_VAR(items); - boot_Data__MessagePack_pack(); + init_Data__MessagePack_pack(aTHX_ false); + init_Data__MessagePack_unpack(aTHX_ false); newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c index 9a58ed05..862808eb 100644 --- a/perl/xs-src/pack.c +++ b/perl/xs-src/pack.c @@ -83,8 +83,7 @@ MGVTBL pref_int_vtbl = { #endif }; -void boot_Data__MessagePack_pack(void) { - dTHX; +void init_Data__MessagePack_pack(pTHX_ bool const cloning) { SV* var = get_sv("Data::MessagePack::PreferInteger", 0); sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0); SvSETMAGIC(var); diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index da985e32..6ebb48c2 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -2,6 +2,13 @@ #define NEED_sv_2pv_flags #include "xshelper.h" +#define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION +typedef struct { + SV* msgpack_true; + SV* msgpack_false; +} my_cxt_t; +START_MY_CXT + typedef struct { bool finished; bool incremented; @@ -22,13 +29,52 @@ typedef struct { #define msgpack_unpack_user unpack_user +void init_Data__MessagePack_unpack(pTHX_ bool const cloning) { + if(!cloning) { + MY_CXT_INIT; + MY_CXT.msgpack_true = NULL; + MY_CXT.msgpack_false = NULL; + } + else { + MY_CXT_CLONE; + MY_CXT.msgpack_true = NULL; + MY_CXT.msgpack_false = NULL; + } +} + + + /* ---------------------------------------------------------------------- */ /* utility functions */ -STATIC_INLINE SV * -get_bool (const char* const name) { +static SV* +load_bool(pTHX_ const char* const name) { + CV* const cv = get_cv(name, GV_ADD); + dSP; + PUSHMARK(SP); + call_sv((SV*)cv, G_SCALAR); + SPAGAIN; + SV* const sv = newSVsv(POPs); + PUTBACK; + return sv; +} + +static SV* +get_bool(bool const value) { dTHX; - return newSVsv(get_sv( name, GV_ADD )); + dMY_CXT; + if(value) { + if(!MY_CXT.msgpack_true) { + MY_CXT.msgpack_true = load_bool(aTHX_ "Data::MessagePack::true"); + } + return newSVsv(MY_CXT.msgpack_true); + } + else { + if(!MY_CXT.msgpack_false) { + MY_CXT.msgpack_false = load_bool(aTHX_ "Data::MessagePack::false"); + } + return newSVsv(MY_CXT.msgpack_false); + } } /* ---------------------------------------------------------------------- */ @@ -127,15 +173,13 @@ STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o) STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o) { - dTHX; - *o = get_bool("Data::MessagePack::true"); + *o = get_bool(true); return 0; } STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o) { - dTHX; - *o = get_bool("Data::MessagePack::false"); + *o = get_bool(false); return 0; } From eab7c877813e7c1dd7820307260221444773d116 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:37:17 +0900 Subject: [PATCH 204/259] Tidy PP --- perl/lib/Data/MessagePack/PP.pm | 119 ++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 9e322991..c7eaadf5 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -1,11 +1,8 @@ package Data::MessagePack::PP; - -use 5.008000; +use 5.008001; use strict; use Carp (); -our $VERSION = '0.15'; - # See also # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm @@ -25,49 +22,74 @@ BEGIN { # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE - # In really, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? - *pack_uint64 = $bo_is_le ? sub { - my @v = unpack( 'V2', pack( 'Q', $_[0] ) ); - return pack 'CN2', 0xcf, @v[1,0]; - } : sub { pack 'CQ', 0xcf, $_[0]; }; - *pack_int64 = $bo_is_le ? sub { - my @v = unpack( 'V2', pack( 'q', $_[0] ) ); - return pack 'CN2', 0xd3, @v[1,0]; - } : sub { pack 'Cq', 0xd3, $_[0]; }; - *pack_double = $bo_is_le ? sub { - my @v = unpack( 'V2', pack( 'd', $_[0] ) ); - return pack 'CN2', 0xcb, @v[1,0]; - } : sub { pack 'Cd', 0xcb, $_[0]; }; - *unpack_float = $bo_is_le ? sub { - my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) ); - return unpack( 'f', pack( 'n2', @v[1,0] ) ); - } : sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; - *unpack_double = $bo_is_le ? sub { - my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) ); - return unpack( 'd', pack( 'N2', @v[1,0] ) ); - } : sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = $bo_is_le ? sub { - my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); - return unpack( 'q', pack( 'N2', @v[1,0] ) ); - } : sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = $bo_is_le ? sub { - my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); - return unpack( 'Q', pack( 'N2', @v[1,0] ) ); - } : sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + + # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? + if($bo_is_le) { + *pack_uint64 = sub { + my @v = unpack( 'V2', pack( 'Q', $_[0] ) ); + return pack 'CN2', 0xcf, @v[1,0]; + }; + *pack_int64 = sub { + my @v = unpack( 'V2', pack( 'q', $_[0] ) ); + return pack 'CN2', 0xd3, @v[1,0]; + }; + *pack_double = sub { + my @v = unpack( 'V2', pack( 'd', $_[0] ) ); + return pack 'CN2', 0xcb, @v[1,0]; + }; + + *unpack_float = sub { + my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) ); + return unpack( 'f', pack( 'n2', @v[1,0] ) ); + }; + *unpack_double = sub { + my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) ); + return unpack( 'd', pack( 'N2', @v[1,0] ) ); + }; + + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? $v - 0x100000000 : 0; + }; + *unpack_int64 = sub { + my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); + return unpack( 'q', pack( 'N2', @v[1,0] ) ); + }; + *unpack_uint64 = sub { + my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); + return unpack( 'Q', pack( 'N2', @v[1,0] ) ); + }; + } + else { # big endian + *pack_uint64 = sub { return pack 'CQ', 0xcf, $_[0]; }; + *pack_int64 = sub { return pack 'Cq', 0xd3, $_[0]; }; + *pack_double = sub { return pack 'Cd', 0xcb, $_[0]; }; + + *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; + *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? $v - 0x100000000 : 0; + }; + *unpack_int64 = sub { pack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + } } else { *pack_uint64 = sub { return pack 'CQ>', 0xcf, $_[0]; }; *pack_int64 = sub { return pack 'Cq>', 0xd3, $_[0]; }; *pack_double = sub { return pack 'Cd>', 0xcb, $_[0]; }; + *unpack_float = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; *unpack_int16 = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); }; @@ -75,11 +97,6 @@ BEGIN { *unpack_int64 = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } - # for 5.8 etc. - unless ( defined &utf8::is_utf8 ) { - require Encode; - *utf8::is_utf8 = *Encode::is_utf8; - } } @@ -92,7 +109,7 @@ BEGIN { my $max_depth; -sub pack { +sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; $max_depth = defined $_[2] ? $_[2] : 512; # init return _pack( $_[1] ); @@ -209,7 +226,7 @@ sub _pack { my $p; # position variables for speed. -sub unpack { +sub unpack :method { $p = 0; # init _unpack( $_[1] ); } @@ -370,7 +387,7 @@ sub execute_limit { sub execute { my ( $self, $data, $offset, $limit ) = @_; - my $value = substr( $data, $offset, $limit ? $limit : length $data ); + my $value = substr( $data, $offset || 0, $limit ? $limit : length $data ); my $len = length $value; $p = 0; @@ -509,7 +526,7 @@ sub is_finished { } -sub reset { +sub reset :method { $_[0]->{ stack } = []; $_[0]->{ data } = undef; $_[0]->{ remain } = undef; From d5a17a3c25068573b20a15ccaa22f6b8926e8d0b Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:43:42 +0900 Subject: [PATCH 205/259] Fix stddata.t --- perl/t/09_stddata.t | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index 976fc5d3..a618787d 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -3,7 +3,6 @@ # Don't edit msgpack/perl/t/std/*, which are just copies. use strict; use Test::More; -use Test::Requires qw(JSON); use t::Util; use Data::MessagePack; @@ -14,7 +13,11 @@ sub slurp { return scalar <$fh>; } -my @data = @{ JSON::decode_json(slurp("t/std/cases.json")) }; +my @data = do { + my $json = slurp("t/std/cases.json"); + $json =~ s/:/=>/g; + @{ eval $json }; +}; my $mpac1 = slurp("t/std/cases.mpac"); my $mpac2 = slurp("t/std/cases_compact.mpac"); From 80058083b86919c02165e91719d7eca662b81db5 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:49:08 +0900 Subject: [PATCH 206/259] Tweaks --- perl/lib/Data/MessagePack/PP.pm | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index c7eaadf5..6a06c3c6 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -121,9 +121,7 @@ sub _pack { return CORE::pack( 'C', 0xc0 ) if ( not defined $value ); - my $b_obj = B::svref_2object( ref $value ? $value : \$value ); - - if ( $b_obj->isa('B::AV') ) { + if ( ref($value) eq 'ARRAY' ) { my $num = @$value; my $header = $num < 16 ? CORE::pack( 'C', 0x90 + $num ) @@ -137,7 +135,7 @@ sub _pack { return join( '', $header, map { _pack( $_ ) } @$value ); } - elsif ( $b_obj->isa('B::HV') ) { + elsif ( ref($value) eq 'HASH' ) { my $num = keys %$value; my $header = $num < 16 ? CORE::pack( 'C', 0x80 + $num ) @@ -151,10 +149,12 @@ sub _pack { return join( '', $header, map { _pack( $_ ) } %$value ); } - elsif ( blessed( $value ) and blessed( $value ) eq 'Data::MessagePack::Boolean' ) { - return CORE::pack( 'C', $$value ? 0xc3 : 0xc2 ); + elsif ( ref( $value ) eq 'Data::MessagePack::Boolean' ) { + return CORE::pack( 'C', ${$value} ? 0xc3 : 0xc2 ); } + + my $b_obj = B::svref_2object( \$value ); my $flags = $b_obj->FLAGS; if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) { @@ -175,7 +175,6 @@ sub _pack { } } - elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule if ( $Data::MessagePack::PreferInteger ) { @@ -204,11 +203,9 @@ sub _pack { return $header . $value; } - elsif ( $flags & ( B::SVf_NOK | B::SVp_NOK ) ) { # double only return pack_double( $value ); } - else { die "???"; } From 8512f9eda168b37b8d362484a13480ef3d06f24b Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:49:55 +0900 Subject: [PATCH 207/259] Add .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d740b181 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.o +*.so +ruby/Makefile From b71cc5d7ee99259950602bdcddce68a0fb1dc52b Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 13:59:52 +0900 Subject: [PATCH 208/259] chmod -x --- perl/lib/Data/MessagePack/Boolean.pm | 0 perl/t/10_splitted_bytes.t | 0 perl/t/50_leaktrace.t | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 perl/lib/Data/MessagePack/Boolean.pm mode change 100755 => 100644 perl/t/10_splitted_bytes.t mode change 100755 => 100644 perl/t/50_leaktrace.t diff --git a/perl/lib/Data/MessagePack/Boolean.pm b/perl/lib/Data/MessagePack/Boolean.pm old mode 100755 new mode 100644 diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t old mode 100755 new mode 100644 diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t old mode 100755 new mode 100644 From a10eb2a0d760f0a2f3aaa16d539d621c0ec41de4 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 14:02:12 +0900 Subject: [PATCH 209/259] Changelogging --- perl/Changes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/perl/Changes b/perl/Changes index 41203763..ce525818 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ + +0.24 + - Fixed a possible SEGV on streaming unpacking (gfx) + - Improve performance, esp. in unpacking (gfx) + 0.23 (NO FEATURE CHANGES) From 130d2064d583065c185e7d6d868c6c51b3731fa2 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 17 Sep 2010 15:28:24 +0900 Subject: [PATCH 210/259] perl: updated benchmark result! gfx++ # performance tuning --- perl/lib/Data/MessagePack.pm | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b1e0174d..7456c3a0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -113,23 +113,32 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) - + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- + -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- =head1 AUTHORS From 2c9d90d463429902226eca593fccebec440a57ef Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:08:15 +0900 Subject: [PATCH 211/259] perl: regen README --- perl/README | 28 ++++++++++++++++++---------- perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/perl/README b/perl/README index e46323da..6272a3b2 100644 --- a/perl/README +++ b/perl/README @@ -56,21 +56,29 @@ SPEED -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- AUTHORS Tokuhiro Matsuno diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 7456c3a0..609ea3cd 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -126,7 +126,7 @@ This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my storable 38399/s -- -73% -89% json 141940/s 270% -- -60% mp 355501/s 826% 150% -- - + -- deserialize JSON::XS: 2.3 Data::MessagePack: 0.24 From e8d8099563ced184991b188bfe7ed4189e24f334 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:08:34 +0900 Subject: [PATCH 212/259] Fix a macro redefinition --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 6ebb48c2..e89b22c5 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -142,7 +142,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -#define template_callback_uint64 template_callback_IV +#define template_callback_int64 template_callback_IV #endif /* IVSIZE */ From a0c18e4380a6c96688101e508acab43982dddfe1 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:16:33 +0900 Subject: [PATCH 213/259] Docs --- perl/lib/Data/MessagePack.pm | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 609ea3cd..52021090 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -45,7 +45,7 @@ Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); =head1 DESCRIPTION @@ -61,11 +61,11 @@ It enables to exchange structured objects between many languages like JSON. But =over 4 -=item PORTABILITY +=item PORTABLE -Messagepack is language independent binary serialize format. +The MessagePack format does not depend on language nor byte order. -=item SMALL SIZE +=item SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 @@ -76,6 +76,7 @@ The MessagePack format saves memory than JSON and Storable format. =item STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for networking such as RPC. +See L for details. =back @@ -105,13 +106,13 @@ unpack the $msgpackstr to a MessagePack format string. =item $Data::MessagePack::PreferInteger -Pack the string as int when the value looks like int(EXPERIMENTAL). +Packs a string as an integer, when it looks like an integer. =back =head1 SPEED -This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). +This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). -- serialize @@ -156,13 +157,15 @@ FURUHASHI Sadayuki hanekomu +gfx + =head1 LICENSE This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. - =head1 SEE ALSO -L is official web site for MessagePack format. +L is the official web site for the MessagePack format. +=cut From 7c8f8703a19d77e23e07b777054d888eff49774a Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:26:16 +0900 Subject: [PATCH 214/259] Add TODOs --- perl/lib/Data/MessagePack.pm | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 52021090..8c5cfac6 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -141,6 +141,25 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +=head1 TODO + +=over + +=item Error handling + +MessagePack cannot deal with complex scalars such as object references, +filehandles, and code references. We should report the errors more kindly. + +=item Streaming deserializer + +The current implementation of the streaming deserializer does not have internal +buffers while some other bindings (such as Ruby binding) does. This limitation +will astonish those who try to unpack byte streams with an arbitrary buffer size +(e.g. C<< while(read($socket, $buffer, $arbitrary_buffer_size)) { ... } >>). +We should implement the internal buffer for the unpacker. + +=back + =head1 AUTHORS Tokuhiro Matsuno From 845af014dce3c54c4294abefa8eb33b6bb39dd61 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:15:51 +0900 Subject: [PATCH 215/259] perl: gfx is a author. --- perl/lib/Data/MessagePack.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 8c5cfac6..953bdf85 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -166,6 +166,8 @@ Tokuhiro Matsuno Makamaka Hannyaharamitu +gfx + =head1 THANKS TO Jun Kuriyama @@ -176,8 +178,6 @@ FURUHASHI Sadayuki hanekomu -gfx - =head1 LICENSE This library is free software; you can redistribute it and/or modify From 446266776eb1a36fd562307eb6de4c64e6cc36d1 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:16:17 +0900 Subject: [PATCH 216/259] perl: regenerate README file --- perl/README | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/perl/README b/perl/README index 6272a3b2..31052789 100644 --- a/perl/README +++ b/perl/README @@ -2,7 +2,7 @@ NAME Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); DESCRIPTION @@ -14,10 +14,10 @@ ABOUT MESSAGEPACK FORMAT But unlike JSON, it is very fast and small. ADVANTAGES - PORTABILITY - Messagepack is language independent binary serialize format. + PORTABLE + The MessagePack format does not depend on language nor byte order. - SMALL SIZE + SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 @@ -26,7 +26,7 @@ ABOUT MESSAGEPACK FORMAT STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for - networking such as RPC. + networking such as RPC. See Data::MessagePack::Unpacker for details. If you want to get more information about the MessagePack format, please visit to . @@ -47,12 +47,11 @@ METHODS Configuration Variables $Data::MessagePack::PreferInteger - Pack the string as int when the value looks like int(EXPERIMENTAL). + Packs a string as an integer, when it looks like an integer. SPEED - This is the result of benchmark/serialize.pl and - benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu - SMP). + This is a result of benchmark/serialize.pl and benchmark/deserialize.pl + on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). -- serialize JSON::XS: 2.3 @@ -80,11 +79,27 @@ SPEED json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +TODO + Error handling + MessagePack cannot deal with complex scalars such as object + references, filehandles, and code references. We should report the + errors more kindly. + + Streaming deserializer + The current implementation of the streaming deserializer does not + have internal buffers while some other bindings (such as Ruby + binding) does. This limitation will astonish those who try to unpack + byte streams with an arbitrary buffer size (e.g. + "while(read($socket, $buffer, $arbitrary_buffer_size)) { ... }"). We + should implement the internal buffer for the unpacker. + AUTHORS Tokuhiro Matsuno Makamaka Hannyaharamitu + gfx + THANKS TO Jun Kuriyama @@ -99,5 +114,6 @@ LICENSE under the same terms as Perl itself. SEE ALSO - is official web site for MessagePack format. + is the official web site for the MessagePack + format. From 953aa95c648fef85f9c11c5ed251f3ddab988a83 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:16:26 +0900 Subject: [PATCH 217/259] perl: added failing test case for streaming unpacker with PP. --- perl/t/11_stream_unpack3.t | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 perl/t/11_stream_unpack3.t diff --git a/perl/t/11_stream_unpack3.t b/perl/t/11_stream_unpack3.t new file mode 100644 index 00000000..0eb8bff7 --- /dev/null +++ b/perl/t/11_stream_unpack3.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use Test::More; +use Data::MessagePack; + +my @data = ( [ 1, 2, 3 ], [ 4, 5, 6 ] ); + +# serialize +my $buffer = ''; +for my $d (@data) { + $buffer .= Data::MessagePack->pack($d); +} + +# deserialize +my $cb = sub { + my ($data) = @_; + + my $d = shift @data; + is_deeply $data, $d; +}; +my $unpacker = Data::MessagePack::Unpacker->new(); +my $nread = 0; +while (1) { + $nread = $unpacker->execute( $buffer, $nread ); + if ( $unpacker->is_finished ) { + my $ret = $unpacker->data; + $cb->( $ret ); + $unpacker->reset; + + $buffer = substr( $buffer, $nread ); + $nread = 0; + next if length($buffer) != 0; + } + last; +} +is scalar(@data), 0; + +done_testing; + From 2c9966a0a304dc4bdb5fc003100ec37b2ec3d70a Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 09:44:32 +0900 Subject: [PATCH 218/259] perl: fixed stream deserializer in pp. --- perl/lib/Data/MessagePack/PP.pm | 48 ++++++++++++++++++--------------- perl/t/03_stream_unpack.t | 4 +-- perl/t/09_stddata.t | 1 + perl/t/10_splitted_bytes.t | 6 +++-- perl/t/12_stream_unpack3.t | 23 ++++++++++++++++ 5 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 perl/t/12_stream_unpack3.t diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 6a06c3c6..0dd64272 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -370,7 +370,7 @@ package use strict; sub new { - bless { stack => [] }, shift; + bless { pos => 0 }, shift; } @@ -384,25 +384,30 @@ sub execute_limit { sub execute { my ( $self, $data, $offset, $limit ) = @_; - my $value = substr( $data, $offset || 0, $limit ? $limit : length $data ); + $offset ||= 0; + my $value = substr( $data, $offset, $limit ? $limit : length $data ); my $len = length $value; + $self->{data} .= $value; + local $self->{stack} = []; + $p = 0; - while ( $len > $p ) { - _count( $self, $value ) or last; + LOOP: while ( length($self->{data}) > $p ) { + _count( $self, $self->{data} ) or last; - if ( @{ $self->{stack} } > 0 ) { - pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0; + while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) { + pop @{ $self->{stack} }; + } + + if (@{$self->{stack}} == 0) { + $self->{is_finished}++; + last LOOP; } } + $self->{pos} = $p; - if ( $len == $p ) { - $self->{ data } .= substr( $value, 0, $p ); - $self->{ remain } = undef; - } - - return $p; + return $p + $offset; } @@ -424,7 +429,9 @@ sub _count { $num = $byte & ~0x90; } - push @{ $self->{stack} }, $num + 1; + if (defined($num) && $num > 0) { + push @{ $self->{stack} }, $num + 1; + } return 1; } @@ -443,7 +450,9 @@ sub _count { $num = $byte & ~0x80; } - push @{ $self->{stack} }, $num * 2 + 1; # a pair + if ($num > 0) { + push @{ $self->{stack} }, $num * 2 + 1; # a pair + } return 1; } @@ -511,22 +520,19 @@ sub _count { sub data { - my $data = Data::MessagePack->unpack( $_[0]->{ data } ); - $_[0]->reset; - return $data; + return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) ); } sub is_finished { my ( $self ) = @_; - ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1; + return $self->{is_finished}; } - sub reset :method { - $_[0]->{ stack } = []; $_[0]->{ data } = undef; - $_[0]->{ remain } = undef; + $_[0]->{ pos } = 0; + $_[0]->{ is_finished } = 0; } 1; diff --git a/perl/t/03_stream_unpack.t b/perl/t/03_stream_unpack.t index a4ab4eba..646fc249 100644 --- a/perl/t/03_stream_unpack.t +++ b/perl/t/03_stream_unpack.t @@ -37,7 +37,7 @@ for (my $i=0; $iexecute("\xc0", 0); # nil } - ok $up->is_finished; - is_deeply $up->data, [undef, undef, undef, undef, undef]; + ok $up->is_finished, 'finished'; + is_deeply $up->data, [undef, undef, undef, undef, undef], 'array, is_deeply'; } diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index a618787d..f98d696b 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -32,6 +32,7 @@ for my $mpac($mpac1, $mpac2) { my $i = 0; while($offset < length($mpac)) { $offset = $mps->execute($mpac, $offset); + ok $mps->is_finished, "data[$i] : is_finished"; is_deeply $mps->data, $data[$i], "data[$i]"; $mps->reset; $i++; diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t index 232d8707..15598f4e 100644 --- a/perl/t/10_splitted_bytes.t +++ b/perl/t/10_splitted_bytes.t @@ -27,12 +27,14 @@ foreach my $size(1 .. 16) { open my $stream, '<:bytes :scalar', \$packed; binmode $stream; my $buff; + my $done = 0; while( read($stream, $buff, $size) ) { #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - $up->execute($buff); + $done = $up->execute($buff); } - ok $up->is_finished, 'is_finished'; + is $done, length($packed); + ok $up->is_finished, "is_finished: $size"; my $data = $up->data; is_deeply $data, $input; } diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack3.t new file mode 100644 index 00000000..118acc30 --- /dev/null +++ b/perl/t/12_stream_unpack3.t @@ -0,0 +1,23 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my @input = ( + +[[]], + [[],[]], + [{"a" => 97},{"a" => 97}], + [{"a" => 97},{"a" => 97},{"a" => 97}], +); + +plan tests => @input * 2; + +for my $input (@input) { + my $packed = Data::MessagePack->pack($input); + my $up = Data::MessagePack::Unpacker->new(); + $up->execute($packed, 0); + ok $up->is_finished, 'finished'; + is_deeply($up->data, $input); +} + From 4767e450351dfbc807e5346ceaa4252bd99ae866 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 13:18:25 +0900 Subject: [PATCH 219/259] perl: fix a test name --- perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} (100%) diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack4.t similarity index 100% rename from perl/t/12_stream_unpack3.t rename to perl/t/12_stream_unpack4.t From 1f07721ec41147e02fa49aea19a3f6aa7b1eb723 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 13:33:27 +0900 Subject: [PATCH 220/259] perl: Scalar::Util is no longer used --- perl/lib/Data/MessagePack/PP.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 0dd64272..abb6e9a0 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -12,7 +12,6 @@ use Carp (); package Data::MessagePack; -use Scalar::Util qw( blessed ); use strict; use B (); From c707392a5a9307504595f6fb9f11930a6a514531 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:30:08 +0900 Subject: [PATCH 221/259] perl: fix int64_t unpacking in both XS and PP --- perl/lib/Data/MessagePack/PP.pm | 67 ++++++++++++++++++++------------- perl/t/data.pl | 30 ++++++++++++--- perl/xs-src/unpack.c | 32 +++++++++++----- 3 files changed, 89 insertions(+), 40 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index abb6e9a0..c3ce230c 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -16,12 +16,44 @@ use strict; use B (); BEGIN { + my $unpack_int64_slow; + my $unpack_uint64_slow; + + if(!eval { pack 'Q', 1 }) { # don't have quad types + $unpack_int64_slow = sub { + require Math::BigInt; + my $high = Math::BigInt->new( unpack_int32( $_[0], $_[1]) ); + my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + + return +($high << 32 | $low)->bstr; + }; + $unpack_uint64_slow = sub { + require Math::BigInt; + my $high = Math::BigInt->new( unpack_uint32( $_[0], $_[1]) ); + my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + return +($high << 32 | $low)->bstr; + }; + } + + *unpack_uint16 = sub { return unpack 'n', substr( $_[0], $_[1], 2 ) }; + *unpack_uint32 = sub { return unpack 'N', substr( $_[0], $_[1], 4 ) }; + # for pack and unpack compatibility if ( $] < 5.010 ) { # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? $v - 0x100000000 : 0; + }; + # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? if($bo_is_le) { *pack_uint64 = sub { @@ -46,20 +78,11 @@ BEGIN { return unpack( 'd', pack( 'N2', @v[1,0] ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { + *unpack_int64 = $unpack_int64_slow ||_sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); }; - *unpack_uint64 = sub { + *unpack_uint64 = $unpack_uint64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'Q', pack( 'N2', @v[1,0] ) ); }; @@ -71,17 +94,8 @@ BEGIN { *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + *unpack_int64 = $unpack_int64_slow || sub { pack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; } } else { @@ -93,8 +107,9 @@ BEGIN { *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; *unpack_int16 = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); }; *unpack_int32 = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); }; - *unpack_int64 = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; - *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; + + *unpack_int64 = $unpack_int64_slow || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } } @@ -283,11 +298,11 @@ sub _unpack { } elsif ( $byte == 0xcd ) { # uint16 $p += 2; - return CORE::unpack 'n', substr( $value, $p - 2, 2 ); + return unpack_uint16( $value, $p - 2 ); } elsif ( $byte == 0xce ) { # unit32 $p += 4; - return CORE::unpack 'N', substr( $value, $p - 4, 4 ); + return unpack_uint32( $value, $p - 4 ); } elsif ( $byte == 0xcf ) { # unit64 $p += 8; diff --git a/perl/t/data.pl b/perl/t/data.pl index 2f58d38c..8ffd25ae 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -5,14 +5,34 @@ no warnings; # i need this, i need this. '92 90 91 91 c0', [[], [[undef]]], '93 c0 c2 c3', [undef, false, true], 'ce 80 00 00 00', 2147483648, - '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], - '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], - '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], - '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', + [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', + ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, - '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', + [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], 'ce 00 ff ff ff' => ''.0xFFFFFF, 'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF, 'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF, + + 'd2 80 00 00 01' => '-2147483647', # int32_t + 'ce 80 00 00 01' => '2147483649', # uint32_t + + 'd2 ff ff ff ff' => '-1', # int32_t + 'ce ff ff ff ff' => '4294967295', # uint32_t + + 'd3 00 00 00 00 80 00 00 01' => '2147483649', # int64_t + 'cf 00 00 00 00 80 00 00 01' => '2147483649', # uint64_t + + 'd3 ff 00 ff ff ff ff ff ff' => '-71776119061217281', # int64_t + 'cf ff 00 ff ff ff ff ff ff' => '18374967954648334335', # uint64_t + + 'd3 ff ff ff ff ff ff ff ff' => '-1', # int64_t + 'cf ff ff ff ff ff ff ff ff' => '18446744073709551615', # uint64_t ) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index e89b22c5..fefb52e8 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -102,13 +102,6 @@ STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const return 0; } -STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) -{ - dTHX; - *o = newSVnv((NV)d); - return 0; -} - STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o) { dTHX; @@ -116,10 +109,31 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - *o = newSVnv((NV)d); + if((uint64_t)(NV)d == d) { + *o = newSVnv((NV)d); + } + else { + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); + *o = newSVpvn(tbuf, len); + } + return 0; +} + +static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +{ + dTHX; + if((uint64_t)(NV)d == (uint64_t)d) { + *o = newSVnv((NV)d); + } + else { + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); + *o = newSVpvn(tbuf, len); + } return 0; } From a86c1624a70b8f8a5012065019d9f1ba4b44595b Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:38:35 +0900 Subject: [PATCH 222/259] perl: More kind error messages in PP --- perl/lib/Data/MessagePack/PP.pm | 19 +++++++++++-------- perl/t/50_leaktrace.t | 6 +++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index c3ce230c..31c08339 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -113,6 +113,9 @@ BEGIN { } } +sub _unexpected { + Carp::confess("Unexpected " . sprintf(shift, @_) . " found"); +} # # PACK @@ -141,7 +144,7 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x90 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; if ( --$max_depth <= 0 ) { Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); @@ -155,7 +158,7 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x80 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; if ( --$max_depth <= 0 ) { Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); @@ -211,7 +214,7 @@ sub _pack { $num < 32 ? CORE::pack( 'C', 0xa0 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num ) - : die "" # don't arrivie here + : _unexpected_number($num) ; return $header . $value; @@ -221,7 +224,7 @@ sub _pack { return pack_double( $value ); } else { - die "???"; + _unexpected("data type %s", $b_obj); } } @@ -365,7 +368,7 @@ sub _unpack { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } } @@ -484,7 +487,7 @@ sub _count { : $byte == 0xcd ? 2 : $byte == 0xce ? 4 : $byte == 0xcf ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -493,7 +496,7 @@ sub _count { : $byte == 0xd1 ? 2 : $byte == 0xd2 ? 4 : $byte == 0xd3 ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -524,7 +527,7 @@ sub _count { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } return 0; diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t index 29485270..440ac901 100644 --- a/perl/t/50_leaktrace.t +++ b/perl/t/50_leaktrace.t @@ -2,8 +2,12 @@ use strict; use Test::Requires { 'Test::LeakTrace' => 0.13 }; use Test::More; - use Data::MessagePack; +BEGIN { + if($INC{'Data/MessagePack/PP.pm'}) { + plan skip_all => 'disabled in PP'; + } +} my $simple_data = "xyz"; my $complex_data = { From bab622de25042d11dc8c149da26e178c22398c56 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:46:10 +0900 Subject: [PATCH 223/259] perl: fix max depth checks in PP --- perl/lib/Data/MessagePack/PP.pm | 16 ++++++++-------- perl/t/12_stream_unpack4.t | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 31c08339..8a904a97 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -124,11 +124,11 @@ sub _unexpected { { no warnings 'recursion'; - my $max_depth; + our $_max_depth; sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; - $max_depth = defined $_[2] ? $_[2] : 512; # init + $_max_depth = defined $_[2] ? $_[2] : 512; # init return _pack( $_[1] ); } @@ -136,6 +136,12 @@ sub pack :method { sub _pack { my ( $value ) = @_; + local $_max_depth = $_max_depth - 1; + + if ( $_max_depth < 0 ) { + Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); + } + return CORE::pack( 'C', 0xc0 ) if ( not defined $value ); if ( ref($value) eq 'ARRAY' ) { @@ -146,9 +152,6 @@ sub _pack { : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } @$value ); } @@ -160,9 +163,6 @@ sub _pack { : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } %$value ); } diff --git a/perl/t/12_stream_unpack4.t b/perl/t/12_stream_unpack4.t index 118acc30..de76e81d 100644 --- a/perl/t/12_stream_unpack4.t +++ b/perl/t/12_stream_unpack4.t @@ -9,6 +9,7 @@ my @input = ( [[],[]], [{"a" => 97},{"a" => 97}], [{"a" => 97},{"a" => 97},{"a" => 97}], + [ map { +{ "foo $_" => "bar $_" } } 'aa' .. 'zz' ], ); plan tests => @input * 2; From e6f6aba2071c0a55b67dc935fa876af6b23e6c11 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:50:52 +0900 Subject: [PATCH 224/259] perl: add portability stuff --- perl/xs-src/unpack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index fefb52e8..a4929c04 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -1,5 +1,6 @@ #define NEED_newRV_noinc #define NEED_sv_2pv_flags +#define NEED_my_snprintf #include "xshelper.h" #define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION From 4902bed4098a4c98fc02555584150b1822c90b6b Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:51:17 +0900 Subject: [PATCH 225/259] perl: more kind testing messages --- perl/t/00_compile.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t index f91b29e7..d465f8dc 100644 --- a/perl/t/00_compile.t +++ b/perl/t/00_compile.t @@ -3,4 +3,5 @@ use warnings; use Test::More tests => 1; use_ok 'Data::MessagePack'; -diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' ); +diag ( "Testing Data::MessagePack/$Data::MessagePack::VERSION (", + $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS', ")" ); From 8935ecfdb86ad7a44c4bbf34acdd99cf80457cf0 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:56:33 +0900 Subject: [PATCH 226/259] perl: requires the latest version of Math::BigInt for PP --- perl/Makefile.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index fafc3876..586a0523 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -37,6 +37,7 @@ NOT_SUPPORT_C99 } else { print "configure PP version\n\n"; + requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken } clean_files qw{ From 1865898cd42c2745850e3fd502f32c4584968263 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:58:32 +0900 Subject: [PATCH 227/259] perl: Fix Makefile.PL --- perl/Makefile.PL | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 586a0523..b5d2701d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,3 +1,5 @@ +# Usage: Makefile.PL --pp # disable XS +# Makefile.PL -g # add -g to the compiler and disable optimization flags use inc::Module::Install; use Module::Install::XSUtil 0.32; use Config; @@ -21,8 +23,9 @@ if ( $] >= 5.008005 and want_xs() ) { if ( $has_c99 ) { use_xshelper(); cc_src_paths('xs-src'); - if ($ENV{DEBUG}) { - cc_append_to_ccflags '-g'; + + if($Module::Install::AUTHOR) { + postamble qq{test :: test_pp\n\n}; } } else { @@ -67,10 +70,6 @@ test_requires('Test::Requires'); test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' ); -if($Module::Install::AUTHOR) { - postamble qq{test :: test_pp\n\n}; -} - repository('http://github.com/msgpack/msgpack'); auto_include; WriteAll; From 63f6c86b46464cd5ff0c48c691bbfe2cee0a484d Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:05:22 +0900 Subject: [PATCH 228/259] perl: add a note about 64 bit integers --- perl/README | 6 ++++++ perl/lib/Data/MessagePack.pm | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/perl/README b/perl/README index 31052789..34769602 100644 --- a/perl/README +++ b/perl/README @@ -79,6 +79,12 @@ SPEED json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +CAVEAT + Unpacking 64 bit integers + This module can unpack 64 bit integers even if your perl does not + support them (i.e. where "perl -V:ivsize" is 4), but you cannot + calculate these values unless you use "Math::BigInt". + TODO Error handling MessagePack cannot deal with complex scalars such as object diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 953bdf85..29981781 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -141,6 +141,14 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +=head1 CAVEAT + +=head2 Unpacking 64 bit integers + +This module can unpack 64 bit integers even if your perl does not support them +(i.e. where C<< perl -V:ivsize >> is 4), but you cannot calculate these values +unless you use C. + =head1 TODO =over From 49379140c7c431c3a27f11dc7565da118dd38f16 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:14:29 +0900 Subject: [PATCH 229/259] perl: PERL_ONLY=1 disables XS --- perl/lib/Data/MessagePack.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 29981781..0f389b13 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -24,7 +24,7 @@ sub false () { if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; - if ( $backend !~ /\b pp \b/xms ) { + if ( $backend !~ /\b pp \b/xms or $ENV{PERL_ONLY} ) { eval { require XSLoader; XSLoader::load(__PACKAGE__, $VERSION); From cb85dcfcb8bdaf706ab515bce4bc1991c990716c Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:49:25 +0900 Subject: [PATCH 230/259] perl: tweaks for benchmarks --- perl/benchmark/data.pl | 6 ++++++ perl/benchmark/deserialize.pl | 8 ++------ perl/benchmark/serialize.pl | 7 +------ 3 files changed, 9 insertions(+), 12 deletions(-) create mode 100755 perl/benchmark/data.pl diff --git a/perl/benchmark/data.pl b/perl/benchmark/data.pl new file mode 100755 index 00000000..6908d1cc --- /dev/null +++ b/perl/benchmark/data.pl @@ -0,0 +1,6 @@ ++{ + "method" => "handleMessage", + "params" => [ "user1", "we were just talking", "foo\nbar\nbaz\nqux" ], + "id" => undef, + "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2), 1 .. 100 ], +}; diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 634a79ed..b1d7fdf2 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -7,12 +7,8 @@ use Storable; #$Data::MessagePack::PreferInteger = 1; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; +my $a = do 'benchmark/data.pl'; + my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); my $s = Storable::freeze($a); diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index e0509ffa..33746842 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -5,12 +5,7 @@ use JSON::XS; use Storable; use Benchmark ':all'; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; +my $a = do 'benchmark/data.pl'; print "-- serialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; From b40284955781a152221e6fe6e0d2568ef4fdc27a Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:54:22 +0900 Subject: [PATCH 231/259] perl: docs --- perl/lib/Data/MessagePack.pm | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 0f389b13..4e4064b0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -45,6 +45,8 @@ Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS + use Data::MessagePack; + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); @@ -55,7 +57,8 @@ This module converts Perl data structures to MessagePack and vice versa. =head1 ABOUT MESSAGEPACK FORMAT MessagePack is a binary-based efficient object serialization format. -It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. +It enables to exchange structured objects between many languages like JSON. +But unlike JSON, it is very fast and small. =head2 ADVANTAGES @@ -113,7 +116,7 @@ Packs a string as an integer, when it looks like an integer. =head1 SPEED This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). - +(You should benchmark them with B data if the speed matters, of course.) -- serialize JSON::XS: 2.3 @@ -195,4 +198,8 @@ it under the same terms as Perl itself. L is the official web site for the MessagePack format. +L + +L + =cut From c2bf2a817410b013aecc30d6a5abc0cad0423dff Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:16:51 +0900 Subject: [PATCH 232/259] perl: make pp benchmarks available --- perl/benchmark/deserialize.pl | 10 +++++----- perl/benchmark/serialize.pl | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index b1d7fdf2..faa2582f 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -1,25 +1,25 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; -use Benchmark ':all'; +use JSON; use Storable; +use Benchmark ':all'; #$Data::MessagePack::PreferInteger = 1; my $a = do 'benchmark/data.pl'; -my $j = JSON::XS::encode_json($a); +my $j = JSON::encode_json($a); my $m = Data::MessagePack->pack($a); my $s = Storable::freeze($a); print "-- deserialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::decode_json($j) }, + json => sub { JSON::decode_json($j) }, mp => sub { Data::MessagePack->unpack($m) }, storable => sub { Storable::thaw($s) }, } diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index 33746842..4982ff61 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -1,19 +1,19 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; +use JSON; use Storable; use Benchmark ':all'; my $a = do 'benchmark/data.pl'; print "-- serialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::encode_json($a) }, + json => sub { JSON::encode_json($a) }, storable => sub { Storable::freeze($a) }, mp => sub { Data::MessagePack->pack($a) }, } From 29707bd2ead5fa41dd0908fd7039124be1f6c5d8 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:19:16 +0900 Subject: [PATCH 233/259] perl: fix bootstrap --- perl/lib/Data/MessagePack.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 4e4064b0..154c5b17 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -23,8 +23,8 @@ sub false () { } if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate - my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; - if ( $backend !~ /\b pp \b/xms or $ENV{PERL_ONLY} ) { + my $backend = $ENV{PERL_DATA_MESSAGEPACK} || ($ENV{PERL_ONLY} ? 'pp' : ''); + if ( $backend !~ /\b pp \b/xms ) { eval { require XSLoader; XSLoader::load(__PACKAGE__, $VERSION); From 8d182f1d79808812bd095125eee1a43c7f7904f5 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:30:07 +0900 Subject: [PATCH 234/259] perl: Changelogging --- perl/Changes | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perl/Changes b/perl/Changes index ce525818..d338cf87 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,7 +1,8 @@ 0.24 - - Fixed a possible SEGV on streaming unpacking (gfx) - - Improve performance, esp. in unpacking (gfx) + - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) + - Fixed unpacking issues for 64 bit integers on 32 bit perls (gfx) + - Improved performance, esp. in unpacking (gfx) 0.23 From a1c01c6722a50db0aedcc73f2bb6cdd6f16c8f3a Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:15:31 +0900 Subject: [PATCH 235/259] perl: regen README --- perl/README | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/perl/README b/perl/README index 34769602..224ff08c 100644 --- a/perl/README +++ b/perl/README @@ -2,6 +2,8 @@ NAME Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS + use Data::MessagePack; + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); @@ -51,7 +53,8 @@ Configuration Variables SPEED This is a result of benchmark/serialize.pl and benchmark/deserialize.pl - on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). (You should + benchmark them with your data if the speed matters, of course.) -- serialize JSON::XS: 2.3 @@ -123,3 +126,7 @@ SEE ALSO is the official web site for the MessagePack format. + Data::MessagePack::Unpacker + + AnyEvent::MPRPC + From d6a825981d14079bf4fa74a0605ecec8e873f94b Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:16:08 +0900 Subject: [PATCH 236/259] perl: fix unpacking int64_t in PP (based on makamaka's patch) --- perl/lib/Data/MessagePack/PP.pm | 15 ++++++++++++--- perl/t/data.pl | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 8a904a97..44940dec 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -22,10 +22,19 @@ BEGIN { if(!eval { pack 'Q', 1 }) { # don't have quad types $unpack_int64_slow = sub { require Math::BigInt; - my $high = Math::BigInt->new( unpack_int32( $_[0], $_[1]) ); - my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + my $high = unpack_uint32( $_[0], $_[1] ); + my $low = unpack_uint32( $_[0], $_[1] + 4); - return +($high << 32 | $low)->bstr; + if($high < 0xF0000000) { # positive + $high = Math::BigInt->new( $high ); + $low = Math::BigInt->new( $low ); + return +($high << 32 | $low)->bstr; + } + else { # negative + $high = Math::BigInt->new( ~$high ); + $low = Math::BigInt->new( ~$low ); + return +( -($high << 32 | $low + 1) )->bstr; + } }; $unpack_uint64_slow = sub { require Math::BigInt; diff --git a/perl/t/data.pl b/perl/t/data.pl index 8ffd25ae..95eac419 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -35,4 +35,9 @@ no warnings; # i need this, i need this. 'd3 ff ff ff ff ff ff ff ff' => '-1', # int64_t 'cf ff ff ff ff ff ff ff ff' => '18446744073709551615', # uint64_t + + # int64_t + 'd3 00 00 00 10 00 00 00 00' => '68719476736', + 'd3 00 00 00 10 00 00 00 01' => '68719476737', + 'd3 10 00 00 00 00 00 00 00' => '1152921504606846976', ) From 5cd37e550533a4d12b3390ff161af83db9b21246 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:20:03 +0900 Subject: [PATCH 237/259] perl: always unpacking 64 bit ints as a string on 32 bit perls --- perl/xs-src/unpack.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index a4929c04..20b345fa 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -113,28 +113,18 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - if((uint64_t)(NV)d == d) { - *o = newSVnv((NV)d); - } - else { - char tbuf[64]; - STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); - *o = newSVpvn(tbuf, len); - } + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); + *o = newSVpvn(tbuf, len); return 0; } static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) { dTHX; - if((uint64_t)(NV)d == (uint64_t)d) { - *o = newSVnv((NV)d); - } - else { - char tbuf[64]; - STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); - *o = newSVpvn(tbuf, len); - } + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); + *o = newSVpvn(tbuf, len); return 0; } From afefbe4e564686ac3b9fac9e1d9782f0114c3383 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:21:25 +0900 Subject: [PATCH 238/259] perl: update .gitignore and MANIFEST.SKIP --- perl/.gitignore | 1 + perl/MANIFEST.SKIP | 1 + 2 files changed, 2 insertions(+) diff --git a/perl/.gitignore b/perl/.gitignore index 3e0e73e5..16568472 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -1,4 +1,5 @@ META.yml +MYMETA.yml Makefile Makefile.old MessagePack.bs diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 71a24e5c..1d2192f5 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -2,6 +2,7 @@ \bCVS\b ^MANIFEST\. ^Makefile$ +^MYMETA\.yml$ ~$ ^# \.old$ From 7f42ed86f29ad9eeab64c6816644bfb83c6eea11 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:47:32 +0900 Subject: [PATCH 239/259] More tests --- perl/t/02_unpack.t | 5 +++-- perl/t/data.pl | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t index 4da69e91..657387a9 100644 --- a/perl/t/02_unpack.t +++ b/perl/t/02_unpack.t @@ -5,13 +5,14 @@ no warnings 'uninitialized'; # i need this. i need this. sub unpackit { my $v = $_[0]; - $v =~ s/ //g; + $v =~ s/ +//g; $v = pack 'H*', $v; return Data::MessagePack->unpack($v); } sub pis ($$) { - is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0] + or diag( explain(unpackit($_[0])) ); } my @dat = do 't/data.pl'; diff --git a/perl/t/data.pl b/perl/t/data.pl index 95eac419..9bf07b7d 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -4,6 +4,11 @@ no warnings; # i need this, i need this. '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], '92 90 91 91 c0', [[], [[undef]]], '93 c0 c2 c3', [undef, false, true], + + '82 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # fix map + 'de 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 16 + 'df 00 00 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 32 + 'ce 80 00 00 00', 2147483648, '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], @@ -39,5 +44,8 @@ no warnings; # i need this, i need this. # int64_t 'd3 00 00 00 10 00 00 00 00' => '68719476736', 'd3 00 00 00 10 00 00 00 01' => '68719476737', + 'd3 00 00 10 00 00 00 00 00' => '17592186044416', + 'd3 00 10 00 00 00 00 00 00' => '4503599627370496', 'd3 10 00 00 00 00 00 00 00' => '1152921504606846976', + 'd3 11 00 00 00 00 00 00 00' => '1224979098644774912', ) From 6379d0fe0f1bc2d3b76480edbe9bd99947aae54e Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:53:54 +0900 Subject: [PATCH 240/259] perl: more tests for nil/true/false entities --- perl/t/07_break.t | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/perl/t/07_break.t b/perl/t/07_break.t index dd27ad25..cf3f1079 100644 --- a/perl/t/07_break.t +++ b/perl/t/07_break.t @@ -1,12 +1,23 @@ use Test::More; use Data::MessagePack; use t::Util; -no warnings 'uninitialized'; # i need this. i need this. -plan tests => 1; +plan tests => 4; -my $d = Data::MessagePack->unpack(Data::MessagePack->pack([{x => undef}])); -$d->[0]->{x} = 1; -ok delete $d->[0]->{x}; -$d->[0] = 4; +my $d = Data::MessagePack->unpack(Data::MessagePack->pack({ + nil => undef, + true => true, + false => false, + foo => [undef, true, false], +})); +$d->{nil} = 42; +is $d->{nil}, 42; + +$d->{true} = 43; +is $d->{true}, 43; + +$d->{false} = 44; +is $d->{false}, 44; + +is_deeply $d->{foo}, [undef, true, false]; From 53899cc492fc8ae4bc4095506d25e4d0eff48309 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 22:12:28 +0900 Subject: [PATCH 241/259] perl: tweaks for Makefile.PL --- perl/Makefile.PL | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index b5d2701d..219400f2 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -16,12 +16,12 @@ recursive_author_tests('xt'); if ( $] >= 5.008005 and want_xs() ) { - can_cc or die "This module requires a C compiler. Please retry with --pp"; - my $has_c99 = c99_available(); # msgpack C library requires C99. if ( $has_c99 ) { + requires_c99(); use_xshelper(); + cc_warnings; cc_src_paths('xs-src'); if($Module::Install::AUTHOR) { @@ -40,7 +40,7 @@ NOT_SUPPORT_C99 } else { print "configure PP version\n\n"; - requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken + requires 'Math::BigInt' => 1.89; # old versions of BigInt were broken } clean_files qw{ From f59178bc3369f43c3c63f02823bd01c1d4f8314a Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 09:30:26 +0900 Subject: [PATCH 242/259] Checking in changes prior to tagging of version 0.24. Changelog diff is: --- perl/lib/Data/MessagePack.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 154c5b17..5990bdbd 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.23'; +our $VERSION = '0.24'; our $PreferInteger = 0; sub true () { From 978bb5059f544b44cb277e0408103ca4a0c27d47 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 09:54:25 +0900 Subject: [PATCH 243/259] Checking in changes prior to tagging of version 0.25. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index d338cf8..3d455cb 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.25 + + (NO FEATURE CHANGES) + - oops. I failed releng. 0.24 - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index d338cf87..3d455cb3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.25 + + (NO FEATURE CHANGES) + - oops. I failed releng. 0.24 - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 5990bdbd..2cbbe797 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.24'; +our $VERSION = '0.25'; our $PreferInteger = 0; sub true () { From d973192b5e44ade6be1b290bc5020c749b43e585 Mon Sep 17 00:00:00 2001 From: makamaka Date: Mon, 20 Sep 2010 11:49:52 +0900 Subject: [PATCH 244/259] perl: modified a serious code typo in PP --- perl/lib/Data/MessagePack/PP.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 44940dec..23942225 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -87,7 +87,7 @@ BEGIN { return unpack( 'd', pack( 'N2', @v[1,0] ) ); }; - *unpack_int64 = $unpack_int64_slow ||_sub { + *unpack_int64 = $unpack_int64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); }; From 664eefdddb6633badeecbbd4dc654eb113b3814d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 23:40:12 +0900 Subject: [PATCH 245/259] Checking in changes prior to tagging of version 0.26. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 3d455cb..b717a82 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.26 + + - fixed a serious code typo in PP(makamaka) + 0.25 (NO FEATURE CHANGES) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 3d455cb3..b717a828 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.26 + + - fixed a serious code typo in PP(makamaka) + 0.25 (NO FEATURE CHANGES) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 2cbbe797..f6e68eb3 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.25'; +our $VERSION = '0.26'; our $PreferInteger = 0; sub true () { From ead8edc7cd3e797e5457c8ba63a2374fb8a87067 Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 22 Sep 2010 14:12:19 +0900 Subject: [PATCH 246/259] modified be unpack_(u)int64 in PP --- perl/lib/Data/MessagePack/PP.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 23942225..679dc00e 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -103,8 +103,8 @@ BEGIN { *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int64 = $unpack_int64_slow || sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + *unpack_int64 = $unpack_int64_slow || sub { unpack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { unpack 'Q', substr( $_[0], $_[1], 8 ); }; } } else { From 6d9a629b155c7a6f814a6973c6c24da308b295ce Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 22 Sep 2010 14:19:09 +0900 Subject: [PATCH 247/259] perl: modified trivial codes in PP::Unpacker --- perl/lib/Data/MessagePack/PP.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 679dc00e..8234b493 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -419,7 +419,7 @@ sub execute { $p = 0; - LOOP: while ( length($self->{data}) > $p ) { + while ( length($self->{data}) > $p ) { _count( $self, $self->{data} ) or last; while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) { @@ -428,7 +428,7 @@ sub execute { if (@{$self->{stack}} == 0) { $self->{is_finished}++; - last LOOP; + last; } } $self->{pos} = $p; @@ -455,7 +455,7 @@ sub _count { $num = $byte & ~0x90; } - if (defined($num) && $num > 0) { + if ( $num ) { push @{ $self->{stack} }, $num + 1; } @@ -476,7 +476,7 @@ sub _count { $num = $byte & ~0x80; } - if ($num > 0) { + if ( $num ) { push @{ $self->{stack} }, $num * 2 + 1; # a pair } From 80f7c54e4d14e4762714c6b26e1dcd994f2f0727 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:25:08 +0900 Subject: [PATCH 248/259] perl: make scopes --- perl/xs-src/unpack.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 20b345fa..95b5910f 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -31,6 +31,7 @@ typedef struct { #define msgpack_unpack_user unpack_user void init_Data__MessagePack_unpack(pTHX_ bool const cloning) { + // booleans are load on demand (lazy load). if(!cloning) { MY_CXT_INIT; MY_CXT.msgpack_true = NULL; @@ -52,11 +53,17 @@ static SV* load_bool(pTHX_ const char* const name) { CV* const cv = get_cv(name, GV_ADD); dSP; + ENTER; + SAVETMPS; PUSHMARK(SP); call_sv((SV*)cv, G_SCALAR); SPAGAIN; SV* const sv = newSVsv(POPs); PUTBACK; + FREETMPS; + LEAVE; + assert(sv); + assert(sv_isobject(sv)); return sv; } From 68b6fa46e61490c3f5fdca71b3e1dc90ebb4a02d Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:28:14 +0900 Subject: [PATCH 249/259] perl: add tests for boolean values --- perl/t/13_booleans.t | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 perl/t/13_booleans.t diff --git a/perl/t/13_booleans.t b/perl/t/13_booleans.t new file mode 100755 index 00000000..1ecbe646 --- /dev/null +++ b/perl/t/13_booleans.t @@ -0,0 +1,12 @@ +#!perl -w +use strict; +use Test::More tests => 6; +use Data::MessagePack; + +ok defined(Data::MessagePack::true()), 'true (1)'; +ok defined(Data::MessagePack::true()), 'true (2)'; +ok Data::MessagePack::true(), 'true is true'; + +ok defined(Data::MessagePack::false()), 'false (1)'; +ok defined(Data::MessagePack::false()), 'false (2)'; +ok !Data::MessagePack::false(), 'false is false'; From 0a8a6ed168dfd9338c2e97fd6b87b88ef39644ef Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:59:21 +0900 Subject: [PATCH 250/259] perl: cleanup PP --- perl/lib/Data/MessagePack/PP.pm | 54 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 8234b493..5dccc0bb 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -1,20 +1,17 @@ package Data::MessagePack::PP; use 5.008001; use strict; +use warnings; +no warnings 'recursion'; + use Carp (); +use B (); # See also # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack - -package - Data::MessagePack; - -use strict; -use B (); - BEGIN { my $unpack_int64_slow; my $unpack_uint64_slow; @@ -120,6 +117,18 @@ BEGIN { *unpack_int64 = $unpack_int64_slow || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } + + # fixin package symbols + no warnings 'once'; + sub pack :method; + sub unpack :method; + *Data::MessagePack::pack = \&pack; + *Data::MessagePack::unpack = \&unpack; + + @Data::MessagePack::Unpacker::ISA = qw(Data::MessagePack::PP::Unpacker); + + *true = \&Data::MessagePack::true; + *false = \&Data::MessagePack::false; } sub _unexpected { @@ -130,10 +139,7 @@ sub _unexpected { # PACK # -{ - no warnings 'recursion'; - - our $_max_depth; +our $_max_depth; sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; @@ -238,20 +244,19 @@ sub _pack { } -} # PACK - - # # UNPACK # -{ - - my $p; # position variables for speed. +my $p; # position variables for speed. sub unpack :method { $p = 0; # init - _unpack( $_[1] ); + my $data = _unpack( $_[1] ); + if($p < length($_[1])) { + Carp::croak("Data::MessagePack->unpack: extra bytes"); + } + return $data; } @@ -383,17 +388,12 @@ sub _unpack { } -} # UNPACK - - # # Data::MessagePack::Unpacker # package - Data::MessagePack::Unpacker; - -use strict; + Data::MessagePack::PP::Unpacker; sub new { bless { pos => 0 }, shift; @@ -404,10 +404,6 @@ sub execute_limit { execute( @_ ); } - -{ - my $p; - sub execute { my ( $self, $data, $offset, $limit ) = @_; $offset ||= 0; @@ -542,8 +538,6 @@ sub _count { return 0; } -} # execute - sub data { return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) ); From 3d905a7a4fe437433dd0746d9864928ec8f464b9 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 16:14:55 +0900 Subject: [PATCH 251/259] perl: add tests for 'extra bytes' exceptions --- perl/t/14_invalid_data.t | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 perl/t/14_invalid_data.t diff --git a/perl/t/14_invalid_data.t b/perl/t/14_invalid_data.t new file mode 100755 index 00000000..f5344857 --- /dev/null +++ b/perl/t/14_invalid_data.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my $nil = Data::MessagePack->pack(undef); + +my @data = do 't/data.pl'; +while(my($dump, $data) = splice @data, 0, 2) { + my $s = Data::MessagePack->pack($data); + eval { + Data::MessagePack->unpack($s . $nil); + }; + like $@, qr/extra bytes/, "dump $dump"; +} + +done_testing; From 142493076a7a268bdb38f02b06ac5bc34a0f3004 Mon Sep 17 00:00:00 2001 From: tanakh Date: Thu, 23 Sep 2010 00:04:34 +0900 Subject: [PATCH 252/259] haskell: TH support and refactoring --- haskell/msgpack.cabal | 10 +- haskell/src/Data/MessagePack.hs | 35 +-- haskell/src/Data/MessagePack/Derive.hs | 74 ++++++ haskell/src/Data/MessagePack/Iteratee.hs | 4 +- haskell/src/Data/MessagePack/Object.hs | 235 +++++++++++++++--- .../src/Data/MessagePack/{Put.hs => Pack.hs} | 78 +++--- .../Data/MessagePack/{Parser.hs => Unpack.hs} | 112 ++++++--- haskell/test/UserData.hs | 19 ++ 8 files changed, 422 insertions(+), 145 deletions(-) create mode 100644 haskell/src/Data/MessagePack/Derive.hs rename haskell/src/Data/MessagePack/{Put.hs => Pack.hs} (66%) rename haskell/src/Data/MessagePack/{Parser.hs => Unpack.hs} (70%) create mode 100644 haskell/test/UserData.hs diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index ccdb2f7f..9c67bdce 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,5 +1,5 @@ Name: msgpack -Version: 0.3.1.1 +Version: 0.4.0 Synopsis: A Haskell binding to MessagePack Description: A Haskell binding to MessagePack @@ -25,17 +25,19 @@ Library attoparsec >= 0.8.1 && < 0.8.2, binary >= 0.5.0 && < 0.5.1, data-binary-ieee754 >= 0.4 && < 0.5, - deepseq >= 1.1 && <1.2 + deepseq >= 1.1 && <1.2, + template-haskell >= 2.4 && < 2.5 Ghc-options: -Wall Hs-source-dirs: src Exposed-modules: Data.MessagePack + Data.MessagePack.Pack + Data.MessagePack.Unpack Data.MessagePack.Object - Data.MessagePack.Put - Data.MessagePack.Parser Data.MessagePack.Iteratee + Data.MessagePack.Derive Source-repository head Type: git diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs index 7137589f..b71190d6 100644 --- a/haskell/src/Data/MessagePack.hs +++ b/haskell/src/Data/MessagePack.hs @@ -13,14 +13,11 @@ -------------------------------------------------------------------- module Data.MessagePack( + module Data.MessagePack.Pack, + module Data.MessagePack.Unpack, module Data.MessagePack.Object, - module Data.MessagePack.Put, - module Data.MessagePack.Parser, module Data.MessagePack.Iteratee, - - -- * Simple functions of Pack and Unpack - pack, - unpack, + module Data.MessagePack.Derive, -- * Pack functions packToString, @@ -44,38 +41,18 @@ import qualified Data.Attoparsec as A import Data.Binary.Put import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as L -import Data.Functor.Identity import qualified Data.Iteratee as I import System.IO +import Data.MessagePack.Pack +import Data.MessagePack.Unpack import Data.MessagePack.Object -import Data.MessagePack.Put -import Data.MessagePack.Parser import Data.MessagePack.Iteratee +import Data.MessagePack.Derive bufferSize :: Int bufferSize = 4 * 1024 -class IsByteString s where - toBS :: s -> B.ByteString - -instance IsByteString B.ByteString where - toBS = id - -instance IsByteString L.ByteString where - toBS = B.concat . L.toChunks - --- | Pack Haskell data to MessagePack string. -pack :: ObjectPut a => a -> L.ByteString -pack = packToString . put - --- | Unpack MessagePack string to Haskell data. -unpack :: (ObjectGet a, IsByteString s) => s -> a -unpack bs = - runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) getI - --- TODO: tryUnpack - -- | Pack to ByteString. packToString :: Put -> L.ByteString packToString = runPut diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs new file mode 100644 index 00000000..cfdb6588 --- /dev/null +++ b/haskell/src/Data/MessagePack/Derive.hs @@ -0,0 +1,74 @@ +{-# Language TemplateHaskell #-} + +module Data.MessagePack.Derive ( + derivePack, + deriveUnpack, + deriveObject, + ) where + +import Control.Applicative +import Language.Haskell.TH + +import Data.MessagePack.Pack +import Data.MessagePack.Unpack + +deriveUnpack :: Name -> Q [Dec] +deriveUnpack typName = do + TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + + return + [ InstanceD [] (AppT (ConT ''Unpackable) (ConT name)) + [ FunD 'get [Clause [] (NormalB $ ch $ map body cons) []] + ]] + + where + body (NormalC conName elms) = + DoE + [ BindS (tupOrList $ map VarP names) (VarE 'get) + , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] + where + names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms + + tupOrList ls + | length ls <= 1 = ListP ls + | otherwise = TupP ls + + ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f) + +derivePack :: Name -> Q [Dec] +derivePack typName = do + TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + + return + [ InstanceD [] (AppT (ConT ''Packable) (ConT name)) + [ FunD 'put (map body cons) + ]] + + where + body (NormalC conName elms) = + Clause + [ ConP conName $ map VarP names ] + (NormalB $ AppE (VarE 'put) $ tupOrList $ map VarE names) [] + where + names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms + + tupOrList ls + | length ls <= 1 = ListE ls + | otherwise = TupE ls + +deriveObject :: Name -> Q [Dec] +deriveObject typName = do + g <- derivePack typName + p <- deriveUnpack typName + {- + TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + let o = InstanceD [] (AppT (ConT ''OBJECT) (ConT name)) + [ FunD 'toObject (map toObjectBody cons) ] + -} + return $ g ++ p -- ++ [o] +{- + where + toObjectBody (NormalC conName elms) = + Clause + [ ConP conP +-} diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs index 4258cf68..6bc08980 100644 --- a/haskell/src/Data/MessagePack/Iteratee.hs +++ b/haskell/src/Data/MessagePack/Iteratee.hs @@ -28,10 +28,10 @@ import qualified Data.ByteString as B import qualified Data.Iteratee as I import System.IO -import Data.MessagePack.Parser +import Data.MessagePack.Unpack -- | Deserialize a value -getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a +getI :: (Monad m, Unpackable a) => I.Iteratee B.ByteString m a getI = parserToIteratee get -- | Enumerator diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs index 87f24bd9..5111ebb6 100644 --- a/haskell/src/Data/MessagePack/Object.hs +++ b/haskell/src/Data/MessagePack/Object.hs @@ -1,6 +1,7 @@ {-# Language TypeSynonymInstances #-} {-# Language FlexibleInstances #-} {-# Language OverlappingInstances #-} +{-# Language IncoherentInstances #-} {-# Language DeriveDataTypeable #-} -------------------------------------------------------------------- @@ -23,16 +24,21 @@ module Data.MessagePack.Object( -- * Serialization to and from Object OBJECT(..), - Result, + -- Result, ) where import Control.DeepSeq +import Control.Exception import Control.Monad import Control.Monad.Trans.Error () +import qualified Data.Attoparsec as A import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as C8 import Data.Typeable +import Data.MessagePack.Pack +import Data.MessagePack.Unpack + -- | Object Representation of MessagePack data. data Object = ObjectNil @@ -55,70 +61,241 @@ instance NFData Object where ObjectArray a -> rnf a ObjectMap m -> rnf m +instance Unpackable Object where + get = + A.choice + [ liftM ObjectInteger get + , liftM (\() -> ObjectNil) get + , liftM ObjectBool get + , liftM ObjectDouble get + , liftM ObjectRAW get + , liftM ObjectArray get + , liftM ObjectMap get + ] + +instance Packable Object where + put obj = + case obj of + ObjectInteger n -> + put n + ObjectNil -> + put () + ObjectBool b -> + put b + ObjectDouble d -> + put d + ObjectRAW raw -> + put raw + ObjectArray arr -> + put arr + ObjectMap m -> + put m + -- | The class of types serializable to and from MessagePack object -class OBJECT a where +class (Unpackable a, Packable a) => OBJECT a where -- | Encode a value to MessagePack object toObject :: a -> Object + toObject = unpack . pack + -- | Decode a value from MessagePack object - fromObject :: Object -> Result a + fromObject :: Object -> a + fromObject a = + case tryFromObject a of + Left err -> + throw $ UnpackError err + Right ret -> + ret --- | A type for parser results -type Result a = Either String a + -- | Decode a value from MessagePack object + tryFromObject :: Object -> Either String a + tryFromObject = tryUnpack . pack instance OBJECT Object where toObject = id - fromObject = Right + tryFromObject = Right -fromObjectError :: String -fromObjectError = "fromObject: cannot cast" +tryFromObjectError :: Either String a +tryFromObjectError = Left "tryFromObject: cannot cast" instance OBJECT () where toObject = const ObjectNil - fromObject ObjectNil = Right () - fromObject _ = Left fromObjectError + tryFromObject ObjectNil = Right () + tryFromObject _ = tryFromObjectError instance OBJECT Int where toObject = ObjectInteger - fromObject (ObjectInteger n) = Right n - fromObject _ = Left fromObjectError + tryFromObject (ObjectInteger n) = Right n + tryFromObject _ = tryFromObjectError instance OBJECT Bool where toObject = ObjectBool - fromObject (ObjectBool b) = Right b - fromObject _ = Left fromObjectError + tryFromObject (ObjectBool b) = Right b + tryFromObject _ = tryFromObjectError instance OBJECT Double where toObject = ObjectDouble - fromObject (ObjectDouble d) = Right d - fromObject _ = Left fromObjectError + tryFromObject (ObjectDouble d) = Right d + tryFromObject _ = tryFromObjectError instance OBJECT B.ByteString where toObject = ObjectRAW - fromObject (ObjectRAW bs) = Right bs - fromObject _ = Left fromObjectError + tryFromObject (ObjectRAW bs) = Right bs + tryFromObject _ = tryFromObjectError instance OBJECT String where toObject = toObject . C8.pack - fromObject obj = liftM C8.unpack $ fromObject obj + tryFromObject obj = liftM C8.unpack $ tryFromObject obj instance OBJECT a => OBJECT [a] where toObject = ObjectArray . map toObject - fromObject (ObjectArray arr) = - mapM fromObject arr - fromObject _ = - Left fromObjectError + tryFromObject (ObjectArray arr) = + mapM tryFromObject arr + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2) => OBJECT (a1, a2) where + toObject (a1, a2) = ObjectArray [toObject a1, toObject a2] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + return (v1, v2) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3) => OBJECT (a1, a2, a3) where + toObject (a1, a2, a3) = ObjectArray [toObject a1, toObject a2, toObject a3] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + return (v1, v2, v3) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4) => OBJECT (a1, a2, a3, a4) where + toObject (a1, a2, a3, a4) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + return (v1, v2, v3, v4) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5) => OBJECT (a1, a2, a3, a4, a5) where + toObject (a1, a2, a3, a4, a5) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4, o5] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + v5 <- tryFromObject o5 + return (v1, v2, v3, v4, v5) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6) => OBJECT (a1, a2, a3, a4, a5, a6) where + toObject (a1, a2, a3, a4, a5, a6) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4, o5, o6] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + v5 <- tryFromObject o5 + v6 <- tryFromObject o6 + return (v1, v2, v3, v4, v5, v6) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7) => OBJECT (a1, a2, a3, a4, a5, a6, a7) where + toObject (a1, a2, a3, a4, a5, a6, a7) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4, o5, o6, o7] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + v5 <- tryFromObject o5 + v6 <- tryFromObject o6 + v7 <- tryFromObject o7 + return (v1, v2, v3, v4, v5, v6, v7) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7, OBJECT a8) => OBJECT (a1, a2, a3, a4, a5, a6, a7, a8) where + toObject (a1, a2, a3, a4, a5, a6, a7, a8) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7, toObject a8] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4, o5, o6, o7, o8] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + v5 <- tryFromObject o5 + v6 <- tryFromObject o6 + v7 <- tryFromObject o7 + v8 <- tryFromObject o8 + return (v1, v2, v3, v4, v5, v6, v7, v8) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError + +instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7, OBJECT a8, OBJECT a9) => OBJECT (a1, a2, a3, a4, a5, a6, a7, a8, a9) where + toObject (a1, a2, a3, a4, a5, a6, a7, a8, a9) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7, toObject a8, toObject a9] + tryFromObject (ObjectArray arr) = + case arr of + [o1, o2, o3, o4, o5, o6, o7, o8, o9] -> do + v1 <- tryFromObject o1 + v2 <- tryFromObject o2 + v3 <- tryFromObject o3 + v4 <- tryFromObject o4 + v5 <- tryFromObject o5 + v6 <- tryFromObject o6 + v7 <- tryFromObject o7 + v8 <- tryFromObject o8 + v9 <- tryFromObject o9 + return (v1, v2, v3, v4, v5, v6, v7, v8, v9) + _ -> + tryFromObjectError + tryFromObject _ = + tryFromObjectError instance (OBJECT a, OBJECT b) => OBJECT [(a, b)] where toObject = ObjectMap . map (\(a, b) -> (toObject a, toObject b)) - fromObject (ObjectMap mem) = do - mapM (\(a, b) -> liftM2 (,) (fromObject a) (fromObject b)) mem - fromObject _ = - Left fromObjectError + tryFromObject (ObjectMap mem) = do + mapM (\(a, b) -> liftM2 (,) (tryFromObject a) (tryFromObject b)) mem + tryFromObject _ = + tryFromObjectError instance OBJECT a => OBJECT (Maybe a) where toObject (Just a) = toObject a toObject Nothing = ObjectNil - fromObject ObjectNil = return Nothing - fromObject obj = liftM Just $ fromObject obj + tryFromObject ObjectNil = return Nothing + tryFromObject obj = liftM Just $ tryFromObject obj diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Pack.hs similarity index 66% rename from haskell/src/Data/MessagePack/Put.hs rename to haskell/src/Data/MessagePack/Pack.hs index 24ec3059..16243ad9 100644 --- a/haskell/src/Data/MessagePack/Put.hs +++ b/haskell/src/Data/MessagePack/Pack.hs @@ -5,7 +5,7 @@ -------------------------------------------------------------------- -- | --- Module : Data.MessagePack.Put +-- Module : Data.MessagePack.Pack -- Copyright : (c) Hideyuki Tanaka, 2009-2010 -- License : BSD3 -- @@ -13,13 +13,15 @@ -- Stability : experimental -- Portability: portable -- --- MessagePack Serializer using @Data.Binary.Put@ +-- MessagePack Serializer using @Data.Binary.Pack@ -- -------------------------------------------------------------------- -module Data.MessagePack.Put( +module Data.MessagePack.Pack ( -- * Serializable class - ObjectPut(..), + Packable(..), + -- * Simple function to pack a Haskell value + pack, ) where import Data.Binary.Put @@ -30,32 +32,16 @@ import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Lazy as L import qualified Data.Vector as V -import Data.MessagePack.Object - -- | Serializable class -class ObjectPut a where +class Packable a where -- | Serialize a value put :: a -> Put -instance ObjectPut Object where - put obj = - case obj of - ObjectInteger n -> - put n - ObjectNil -> - put () - ObjectBool b -> - put b - ObjectDouble d -> - put d - ObjectRAW raw -> - put raw - ObjectArray arr -> - put arr - ObjectMap m -> - put m +-- | Pack Haskell data to MessagePack string. +pack :: Packable a => a -> L.ByteString +pack = runPut . put -instance ObjectPut Int where +instance Packable Int where put n = case n of _ | n >= 0 && n <= 127 -> @@ -87,26 +73,26 @@ instance ObjectPut Int where putWord8 0xD3 putWord64be $ fromIntegral n -instance ObjectPut () where +instance Packable () where put _ = putWord8 0xC0 -instance ObjectPut Bool where +instance Packable Bool where put True = putWord8 0xC3 put False = putWord8 0xC2 -instance ObjectPut Double where +instance Packable Double where put d = do putWord8 0xCB putFloat64be d -instance ObjectPut String where +instance Packable String where put = putString length (putByteString . B8.pack) -instance ObjectPut B.ByteString where +instance Packable B.ByteString where put = putString B.length putByteString -instance ObjectPut L.ByteString where +instance Packable L.ByteString where put = putString (fromIntegral . L.length) putLazyByteString putString :: (s -> Int) -> (s -> Put) -> s -> Put @@ -122,41 +108,41 @@ putString lf pf str = do putWord32be $ fromIntegral len pf str -instance ObjectPut a => ObjectPut [a] where +instance Packable a => Packable [a] where put = putArray length (mapM_ put) -instance ObjectPut a => ObjectPut (V.Vector a) where +instance Packable a => Packable (V.Vector a) where put = putArray V.length (V.mapM_ put) -instance (ObjectPut a1, ObjectPut a2) => ObjectPut (a1, a2) where +instance (Packable a1, Packable a2) => Packable (a1, a2) where put = putArray (const 2) f where f (a1, a2) = put a1 >> put a2 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3) => ObjectPut (a1, a2, a3) where +instance (Packable a1, Packable a2, Packable a3) => Packable (a1, a2, a3) where put = putArray (const 3) f where f (a1, a2, a3) = put a1 >> put a2 >> put a3 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4) => ObjectPut (a1, a2, a3, a4) where +instance (Packable a1, Packable a2, Packable a3, Packable a4) => Packable (a1, a2, a3, a4) where put = putArray (const 4) f where f (a1, a2, a3, a4) = put a1 >> put a2 >> put a3 >> put a4 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5) => ObjectPut (a1, a2, a3, a4, a5) where +instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5) => Packable (a1, a2, a3, a4, a5) where put = putArray (const 5) f where f (a1, a2, a3, a4, a5) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6) => ObjectPut (a1, a2, a3, a4, a5, a6) where +instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6) => Packable (a1, a2, a3, a4, a5, a6) where put = putArray (const 6) f where f (a1, a2, a3, a4, a5, a6) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7) => ObjectPut (a1, a2, a3, a4, a5, a6, a7) where +instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7) => Packable (a1, a2, a3, a4, a5, a6, a7) where put = putArray (const 7) f where f (a1, a2, a3, a4, a5, a6, a7) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8) where +instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7, Packable a8) => Packable (a1, a2, a3, a4, a5, a6, a7, a8) where put = putArray (const 8) f where f (a1, a2, a3, a4, a5, a6, a7, a8) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 -instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8, ObjectPut a9) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8, a9) where +instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7, Packable a8, Packable a9) => Packable (a1, a2, a3, a4, a5, a6, a7, a8, a9) where put = putArray (const 9) f where f (a1, a2, a3, a4, a5, a6, a7, a8, a9) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 >> put a9 @@ -173,13 +159,13 @@ putArray lf pf arr = do putWord32be $ fromIntegral len pf arr -instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where +instance (Packable k, Packable v) => Packable [(k, v)] where put = putMap length (mapM_ putPair) -instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where +instance (Packable k, Packable v) => Packable (V.Vector (k, v)) where put = putMap V.length (V.mapM_ putPair) -putPair :: (ObjectPut a, ObjectPut b) => (a, b) -> Put +putPair :: (Packable a, Packable b) => (a, b) -> Put putPair (a, b) = put a >> put b putMap :: (a -> Int) -> (a -> Put) -> a -> Put @@ -194,3 +180,7 @@ putMap lf pf m = do putWord8 0xDF putWord32be $ fromIntegral len pf m + +instance Packable a => Packable (Maybe a) where + put Nothing = put () + put (Just a) = put a diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Unpack.hs similarity index 70% rename from haskell/src/Data/MessagePack/Parser.hs rename to haskell/src/Data/MessagePack/Unpack.hs index 200ad962..a0d618ec 100644 --- a/haskell/src/Data/MessagePack/Parser.hs +++ b/haskell/src/Data/MessagePack/Unpack.hs @@ -2,10 +2,11 @@ {-# Language IncoherentInstances #-} {-# Language OverlappingInstances #-} {-# Language TypeSynonymInstances #-} +{-# Language DeriveDataTypeable #-} -------------------------------------------------------------------- -- | --- Module : Data.MessagePack.Parser +-- Module : Data.MessagePack.Unpack -- Copyright : (c) Hideyuki Tanaka, 2009-2010 -- License : BSD3 -- @@ -17,11 +18,19 @@ -- -------------------------------------------------------------------- -module Data.MessagePack.Parser( +module Data.MessagePack.Unpack( -- * MessagePack deserializer - ObjectGet(..), + Unpackable(..), + -- * Simple function to unpack a Haskell value + unpack, + tryUnpack, + -- * Unpack exception + UnpackError(..), + -- * ByteString utils + IsByteString(..), ) where +import Control.Exception import Control.Monad import qualified Data.Attoparsec as A import Data.Binary.Get @@ -31,30 +40,53 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Lazy as L import Data.Int +import Data.Typeable import qualified Data.Vector as V import Data.Word import Text.Printf -import Data.MessagePack.Object - -- | Deserializable class -class ObjectGet a where +class Unpackable a where -- | Deserialize a value get :: A.Parser a -instance ObjectGet Object where - get = - A.choice - [ liftM ObjectInteger get - , liftM (\() -> ObjectNil) get - , liftM ObjectBool get - , liftM ObjectDouble get - , liftM ObjectRAW get - , liftM ObjectArray get - , liftM ObjectMap get - ] +class IsByteString s where + toBS :: s -> B.ByteString -instance ObjectGet Int where +instance IsByteString B.ByteString where + toBS = id + +instance IsByteString L.ByteString where + toBS = B.concat . L.toChunks + +-- | The exception of unpack +data UnpackError = + UnpackError String + deriving (Show, Typeable) + +instance Exception UnpackError + +-- | Unpack MessagePack string to Haskell data. +unpack :: (Unpackable a, IsByteString s) => s -> a +unpack bs = + case tryUnpack bs of + Left err -> + throw $ UnpackError err + Right ret -> + ret + +-- | Unpack MessagePack string to Haskell data. +tryUnpack :: (Unpackable a, IsByteString s) => s -> Either String a +tryUnpack bs = + case A.parse get (toBS bs) of + A.Fail _ _ err -> + Left err + A.Partial _ -> + Left "not enough input" + A.Done _ ret -> + Right ret + +instance Unpackable Int where get = do c <- A.anyWord8 case c of @@ -81,7 +113,7 @@ instance ObjectGet Int where _ -> fail $ printf "invlid integer tag: 0x%02X" c -instance ObjectGet () where +instance Unpackable () where get = do c <- A.anyWord8 case c of @@ -90,7 +122,7 @@ instance ObjectGet () where _ -> fail $ printf "invlid nil tag: 0x%02X" c -instance ObjectGet Bool where +instance Unpackable Bool where get = do c <- A.anyWord8 case c of @@ -101,7 +133,7 @@ instance ObjectGet Bool where _ -> fail $ printf "invlid bool tag: 0x%02X" c -instance ObjectGet Double where +instance Unpackable Double where get = do c <- A.anyWord8 case c of @@ -112,13 +144,13 @@ instance ObjectGet Double where _ -> fail $ printf "invlid double tag: 0x%02X" c -instance ObjectGet String where +instance Unpackable String where get = parseString (\n -> return . B8.unpack =<< A.take n) -instance ObjectGet B.ByteString where +instance Unpackable B.ByteString where get = parseString A.take -instance ObjectGet L.ByteString where +instance Unpackable L.ByteString where get = parseString (\n -> do bs <- A.take n; return $ L.fromChunks [bs]) parseString :: (Int -> A.Parser a) -> A.Parser a @@ -134,48 +166,48 @@ parseString aget = do _ -> fail $ printf "invlid raw tag: 0x%02X" c -instance ObjectGet a => ObjectGet [a] where +instance Unpackable a => Unpackable [a] where get = parseArray (flip replicateM get) -instance ObjectGet a => ObjectGet (V.Vector a) where +instance Unpackable a => Unpackable (V.Vector a) where get = parseArray (flip V.replicateM get) -instance (ObjectGet a1, ObjectGet a2) => ObjectGet (a1, a2) where +instance (Unpackable a1, Unpackable a2) => Unpackable (a1, a2) where get = parseArray f where f 2 = get >>= \a1 -> get >>= \a2 -> return (a1, a2) f n = fail $ printf "wrong tupple size: expected 2 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3) => ObjectGet (a1, a2, a3) where +instance (Unpackable a1, Unpackable a2, Unpackable a3) => Unpackable (a1, a2, a3) where get = parseArray f where f 3 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> return (a1, a2, a3) f n = fail $ printf "wrong tupple size: expected 3 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4) => ObjectGet (a1, a2, a3, a4) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4) => Unpackable (a1, a2, a3, a4) where get = parseArray f where f 4 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> return (a1, a2, a3, a4) f n = fail $ printf "wrong tupple size: expected 4 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5) => ObjectGet (a1, a2, a3, a4, a5) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5) => Unpackable (a1, a2, a3, a4, a5) where get = parseArray f where f 5 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> return (a1, a2, a3, a4, a5) f n = fail $ printf "wrong tupple size: expected 5 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6) => ObjectGet (a1, a2, a3, a4, a5, a6) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6) => Unpackable (a1, a2, a3, a4, a5, a6) where get = parseArray f where f 6 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> return (a1, a2, a3, a4, a5, a6) f n = fail $ printf "wrong tupple size: expected 6 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7) => ObjectGet (a1, a2, a3, a4, a5, a6, a7) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7) => Unpackable (a1, a2, a3, a4, a5, a6, a7) where get = parseArray f where f 7 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> return (a1, a2, a3, a4, a5, a6, a7) f n = fail $ printf "wrong tupple size: expected 7 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7, Unpackable a8) => Unpackable (a1, a2, a3, a4, a5, a6, a7, a8) where get = parseArray f where f 8 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> return (a1, a2, a3, a4, a5, a6, a7, a8) f n = fail $ printf "wrong tupple size: expected 8 but got " n -instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8, ObjectGet a9) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8, a9) where +instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7, Unpackable a8, Unpackable a9) => Unpackable (a1, a2, a3, a4, a5, a6, a7, a8, a9) where get = parseArray f where f 9 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> get >>= \a9 -> return (a1, a2, a3, a4, a5, a6, a7, a8, a9) f n = fail $ printf "wrong tupple size: expected 9 but got " n @@ -193,13 +225,13 @@ parseArray aget = do _ -> fail $ printf "invlid array tag: 0x%02X" c -instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where +instance (Unpackable k, Unpackable v) => Unpackable [(k, v)] where get = parseMap (flip replicateM parsePair) -instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where +instance (Unpackable k, Unpackable v) => Unpackable (V.Vector (k, v)) where get = parseMap (flip V.replicateM parsePair) -parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v) +parsePair :: (Unpackable k, Unpackable v) => A.Parser (k, v) parsePair = do a <- get b <- get @@ -218,6 +250,12 @@ parseMap aget = do _ -> fail $ printf "invlid map tag: 0x%02X" c +instance Unpackable a => Unpackable (Maybe a) where + get = + A.choice + [ liftM Just get + , liftM (\() -> Nothing) get ] + parseUint16 :: A.Parser Word16 parseUint16 = do b0 <- A.anyWord8 diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs new file mode 100644 index 00000000..8aced13f --- /dev/null +++ b/haskell/test/UserData.hs @@ -0,0 +1,19 @@ +{-# Language TemplateHaskell #-} + +import Data.MessagePack +import Data.MessagePack.Derive + +data T + = A Int String + | B Double + deriving (Show) + +$(deriveObject ''T) + +main = do + let bs = pack $ A 123 "hoge" + print bs + print (unpack bs :: T) + let cs = pack $ B 3.14 + print cs + print (unpack cs :: T) From 13b6708a099c1e3a6f498f9f4899b6b139c112d4 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 23 Sep 2010 15:24:12 +0900 Subject: [PATCH 253/259] java: append a code for generating a messageConvert method to annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 882 ++++++++++++++++-- .../annotation/TestMessagePackUnpackable.java | 243 ++++- 2 files changed, 1045 insertions(+), 80 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 52384448..a337ff94 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -11,6 +11,7 @@ import java.lang.reflect.ParameterizedType; import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -25,6 +26,7 @@ import javassist.CtNewMethod; import javassist.NotFoundException; import org.msgpack.MessageConvertable; +import org.msgpack.MessagePackObject; import org.msgpack.MessagePackable; import org.msgpack.MessageTypeException; import org.msgpack.MessageUnpackable; @@ -43,37 +45,6 @@ public class PackUnpackUtil { static final String KEYWORD_NEW = "new"; - static final String TYPE_NAME_VOID = void.class.getName(); - - static final String TYPE_NAME_OBJECT = Object.class.getName(); - - static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); - - static final String TYPE_NAME_PACKER = Packer.class.getName(); - - static final String TYPE_NAME_UNPACKER = Unpacker.class.getName(); - - static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class - .getName(); - - static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class - .getName(); - - static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class - .getName(); - - static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class - .getName(); - - static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class - .getName(); - - static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class - .getName(); - - static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class - .getName(); - static final String CHAR_NAME_SPACE = " "; static final String CHAR_NAME_COMMA = ","; @@ -92,15 +63,27 @@ public class PackUnpackUtil { static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{"; + static final String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]"; + + static final String CHAR_NAME_LEFT_SQUARE_BRACKET = "["; + static final String CHAR_NAME_DOT = "."; static final String CHAR_NAME_SEMICOLON = ";"; - static final String VARIABLE_NAME_PK = "pk"; + static final String VARIABLE_NAME_PK = "_$$_pk"; - static final String VARIABLE_NAME_OBJ = "obj"; + static final String VARIABLE_NAME_SIZE = "_$$_len"; - static final String VARIABLE_NAME_SIZE = "len"; + static final String VARIABLE_NAME_ARRAY = "_$$_ary"; + + static final String VARIABLE_NAME_LIST = "_$$_list"; + + static final String VARIABLE_NAME_MAP = "_$$_map"; + + static final String VARIABLE_NAME_ITER = "_$$_iter"; + + static final String VARIABLE_NAME_MPO1 = "_$$_mpo1"; static final String VARIABLE_NAME_I = "i"; @@ -110,6 +93,18 @@ public class PackUnpackUtil { static final String METHOD_NAME_PUT = "put"; + static final String METHOD_NAME_GET = "get"; + + static final String METHOD_NAME_SIZE = "size"; + + static final String METHOD_NAME_KEYSET = "keySet"; + + static final String METHOD_NAME_ITERATOR = "iterator"; + + static final String METHOD_NAME_HASNEXT = "hasNext"; + + static final String METHOD_NAME_NEXT = "next"; + static final String METHOD_NAME_MSGPACK = "messagePack"; static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; @@ -147,6 +142,8 @@ public class PackUnpackUtil { static final String METHOD_NAME_UNPACKARRAY = "unpackArray"; static final String METHOD_NAME_UNPACKMAP = "unpackMap"; + + static final String METHOD_NAME_ASARRAY = "asArray"; } public static class Enhancer { @@ -180,9 +177,9 @@ public class PackUnpackUtil { setInterfaces(enhCtClass); addConstructor(enhCtClass); Field[] fields = getDeclaredFields(origClass); - addMessagePackMethod(enhCtClass, origCtClass, fields); - addMessageUnpackMethod(enhCtClass, origCtClass, fields); - addMessageConvertMethod(enhCtClass, origCtClass, fields); + addMessagePackMethod(enhCtClass, fields); + addMessageUnpackMethod(enhCtClass, fields); + addMessageConvertMethod(enhCtClass, fields); return createClass(enhCtClass); } @@ -243,11 +240,11 @@ public class PackUnpackUtil { } private void setInterfaces(CtClass enhCtClass) throws NotFoundException { - CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE); + CtClass pacCtClass = pool.get(MessagePackable.class.getName()); enhCtClass.addInterface(pacCtClass); - CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE); + CtClass unpacCtClass = pool.get(MessageUnpackable.class.getName()); enhCtClass.addInterface(unpacCtClass); - CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE); + CtClass convCtClass = pool.get(MessageConvertable.class.getName()); enhCtClass.addInterface(convCtClass); } @@ -296,24 +293,23 @@ public class PackUnpackUtil { + field.getName()); } - private void addMessagePackMethod(CtClass enhCtClass, - CtClass origCtClass, Field[] fields) + private void addMessagePackMethod(CtClass enhCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.TYPE_NAME_VOID); + sb.append(void.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.METHOD_NAME_MSGPACK); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.TYPE_NAME_PACKER); + sb.append(Packer.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.VARIABLE_NAME_PK); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.KEYWORD_THROWS); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(IOException.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); sb.append(Constants.CHAR_NAME_SPACE); @@ -329,7 +325,7 @@ public class PackUnpackUtil { insertCodeOfMessagePack(sb, field); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - //System.out.println("messagePack method: " + sb.toString()); + // System.out.println("messagePack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } @@ -345,27 +341,26 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } - private void addMessageUnpackMethod(CtClass enhCtClass, - CtClass origCtClass, Field[] fields) + private void addMessageUnpackMethod(CtClass enhCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.TYPE_NAME_VOID); + sb.append(void.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.METHOD_NAME_MSGUNPACK); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.TYPE_NAME_UNPACKER); + sb.append(Unpacker.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.VARIABLE_NAME_PK); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.KEYWORD_THROWS); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION); + sb.append(MessageTypeException.class.getName()); sb.append(Constants.CHAR_NAME_COMMA); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(IOException.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); sb.append(Constants.CHAR_NAME_SPACE); @@ -380,7 +375,7 @@ public class PackUnpackUtil { insertCodeOfMessageUnpack(sb, field, field.getType()); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - //System.out.println("messageUnpack method: " + sb.toString()); + // System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } @@ -680,33 +675,761 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } - private void addMessageConvertMethod(CtClass enhCtClass, - CtClass origCtClass, Field[] fields) + private void addMessageConvertMethod(CtClass enhCtClass, Field[] fields) throws CannotCompileException { + // messageConvert(MessagePackObject obj) throws MessageTypeException StringBuilder sb = new StringBuilder(); - sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( - Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.METHOD_NAME_MSGCONVERT).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.TYPE_NAME_OBJECT).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_OBJ).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.KEYWORD_THROWS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( - Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( - Constants.CHAR_NAME_SPACE); - // TODO + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(void.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGCONVERT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_MPO1); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(MessageTypeException.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessagePackObjectArrayGet(sb); + insertCodeOfMesageConvertCalls(sb, fields); sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); //System.out.println("messageConvert method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } + private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) { + // MessagePackObject[] ary = obj.asArray(); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_MPO1); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ASARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMesageConvertCalls(StringBuilder sb, + Field[] fields) { + for (int i = 0; i < fields.length; ++i) { + insertCodeOfMessageConvertCall(sb, fields[i], fields[i] + .getType(), i, null); + } + } + + private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f, + Class c, int i, String name) { + if (c.isPrimitive()) { // primitive type + // f0 = objs[0].intValue(); + if (c.equals(boolean.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asBoolean"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(byte.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asByte"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(short.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asShort"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(int.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asInt"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(long.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asLong"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(float.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asFloat"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(double.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asDouble"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else { + throw new PackUnpackUtilException("fatal error: " + + c.getName()); + } + } else { // reference type + if (c.equals(Boolean.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Boolean.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asBoolean"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Byte.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Byte.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asByte"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Short.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Short.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asShort"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Integer.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Integer.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asInt"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Long.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Long.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asLong"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Float.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Float.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asFloat"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(Double.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Double.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asDouble"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else { + if (c.equals(String.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asString"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(byte[].class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asByteArray"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (c.equals(BigInteger.class)) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asBigInteger"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } else if (List.class.isAssignableFrom(c)) { + insertCodeOfMessageConvertCallForList(sb, f, c, i); + } else if (Map.class.isAssignableFrom(c)) { + insertCodeOfMessageConveretCallForMap(sb, f, c, i); + } else if (MessageConvertable.class.isAssignableFrom(c) + || (getCache(c.getName()) != null)) { + // TODO + // TODO + // TODO + // ((MessageConvertable)f_i).messageConvert(ary[i]); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageConvertable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_MSGCONVERT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } else { + throw new MessageTypeException("Type error: " + + c.getName()); + } + } + } + } + + private void insertCodeOfMessageConvertCallForList(StringBuilder sb, + Field field, Class type, int i) { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType = (Class) generic.getActualTypeArguments()[0]; + + // List list = ary[i].asList(); + sb.append(List.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_LIST); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asList"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // int size = list.size(); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_LIST); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_SIZE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(ArrayList.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + + // block begin + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append("_$$_val"); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_LIST); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_GET); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ADD); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageConvertCall(sb, null, genericType, -1, "_$$_val"); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + // block end + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageConveretCallForMap(StringBuilder sb, + Field f, Class c, int i) { + ParameterizedType generic = (ParameterizedType) f.getGenericType(); + Class genericType0 = (Class) generic.getActualTypeArguments()[0]; + Class genericType1 = (Class) generic.getActualTypeArguments()[1]; + + // Map map = ary[i].asMap(); + sb.append(Map.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_MAP); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_DOT); + sb.append("asMap"); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // int size = list.size(); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_MAP); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_SIZE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(HashMap.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Iterator.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ITER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_MAP); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_KEYSET); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ITERATOR); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ITER); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_HASNEXT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block map. + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append("_$$_key"); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_ITER); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_NEXT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append("_$$_val"); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessagePackObject.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_MAP); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_GET); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append("_$$_key"); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PUT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageConvertCall(sb, null, genericType0, -1, + "_$$_key"); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessageConvertCall(sb, null, genericType1, -1, + "_$$_val"); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + private Class createClass(CtClass enhCtClass) throws CannotCompileException { return enhCtClass.toClass(null, null); @@ -749,6 +1472,19 @@ public class PackUnpackUtil { } } + public static Object initEnhancedInstance(MessagePackObject obj, + Class origClass) { + Object ret = newEnhancedInstance(origClass); + ((MessageConvertable) ret).messageConvert(obj); + return ret; + } + + public static Object initEnhancedInstance(MessagePackObject obj, + Object origObj) { + ((MessageConvertable) origObj).messageConvert(obj); + return origObj; + } + @MessagePackUnpackable public static class Image { public String uri = ""; @@ -789,6 +1525,6 @@ public class PackUnpackUtil { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); - //System.out.println(src.equals(dst)); + // System.out.println(src.equals(dst)); } } diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index dbea7cb2..4b1e6664 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -12,6 +12,7 @@ import java.util.Map; import junit.framework.TestCase; import org.junit.Test; +import org.msgpack.MessagePackObject; import org.msgpack.MessageUnpackable; import org.msgpack.Packer; import org.msgpack.Unpacker; @@ -19,7 +20,7 @@ import org.msgpack.Unpacker; public class TestMessagePackUnpackable extends TestCase { @Test - public void testPrimitiveTypeFields() throws Exception { + public void testPrimitiveTypeFields01() throws Exception { PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil .newEnhancedInstance(PrimitiveTypeFieldsClass.class); src.f0 = (byte) 0; @@ -45,6 +46,36 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f6, dst.f6); } + @Test + public void testPrimitiveTypeFields02() throws Exception { + PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); + src.f0 = (byte) 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f6 = false; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil + .initEnhancedInstance(mpo, PrimitiveTypeFieldsClass.class); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + assertFalse(it.hasNext()); + } + @MessagePackUnpackable public static class PrimitiveTypeFieldsClass { public byte f0; @@ -60,7 +91,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testGeneralReferenceTypeFieldsClass() throws Exception { + public void testGeneralReferenceTypeFieldsClass01() throws Exception { GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); src.f0 = 0; @@ -93,6 +124,44 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f9[1], dst.f9[1]); } + @Test + public void testGeneralReferenceTypeFieldsClass02() throws Exception { + GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + src.f0 = 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = (long) 3; + src.f4 = (float) 4; + src.f5 = (double) 5; + src.f6 = false; + src.f7 = new BigInteger("7"); + src.f8 = "8"; + src.f9 = new byte[] { 0x01, 0x02 }; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .initEnhancedInstance(mpo, + GeneralReferenceTypeFieldsClass.class); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + assertEquals(src.f7, dst.f7); + assertEquals(src.f8, dst.f8); + assertEquals(src.f9[0], dst.f9[0]); + assertEquals(src.f9[1], dst.f9[1]); + assertFalse(it.hasNext()); + } + @MessagePackUnpackable public static class GeneralReferenceTypeFieldsClass { public Byte f0; @@ -110,7 +179,7 @@ public class TestMessagePackUnpackable extends TestCase { } } - public void testListTypes() throws Exception { + public void testListTypes01() throws Exception { SampleListTypes src = (SampleListTypes) PackUnpackUtil .newEnhancedInstance(SampleListTypes.class); src.f0 = new ArrayList(); @@ -139,6 +208,39 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f2.get(i), dst.f2.get(i)); } } + + public void testListTypes02() throws Exception { + SampleListTypes src = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + src.f0 = new ArrayList(); + src.f1 = new ArrayList(); + src.f1.add(1); + src.f1.add(2); + src.f1.add(3); + src.f2 = new ArrayList(); + src.f2.add("e1"); + src.f2.add("e2"); + src.f2.add("e3"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + SampleListTypes dst = (SampleListTypes) PackUnpackUtil + .initEnhancedInstance(mpo, SampleListTypes.class); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + for (int i = 0; i < src.f1.size(); ++i) { + assertEquals(src.f1.get(i), dst.f1.get(i)); + } + assertEquals(src.f2.size(), dst.f2.size()); + for (int i = 0; i < src.f2.size(); ++i) { + assertEquals(src.f2.get(i), dst.f2.get(i)); + } + assertFalse(it.hasNext()); + } @MessagePackUnpackable public static class SampleListTypes { @@ -150,7 +252,7 @@ public class TestMessagePackUnpackable extends TestCase { } } - public void testMapTypes() throws Exception { + public void testMapTypes01() throws Exception { SampleMapTypes src = (SampleMapTypes) PackUnpackUtil .newEnhancedInstance(SampleMapTypes.class); src.f0 = new HashMap(); @@ -189,6 +291,49 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f2.get(s2), dst.f2.get(d2)); } } + + public void testMapTypes02() throws Exception { + SampleMapTypes src = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + src.f0 = new HashMap(); + src.f1 = new HashMap(); + src.f1.put(1, 1); + src.f1.put(2, 2); + src.f1.put(3, 3); + src.f2 = new HashMap(); + src.f2.put("k1", 1); + src.f2.put("k2", 2); + src.f2.put("k3", 3); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil + .initEnhancedInstance(mpo, SampleMapTypes.class); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + Iterator srcf1 = src.f1.keySet().iterator(); + Iterator dstf1 = dst.f1.keySet().iterator(); + while (srcf1.hasNext()) { + Integer s1 = srcf1.next(); + Integer d1 = dstf1.next(); + assertEquals(s1, d1); + assertEquals(src.f1.get(s1), dst.f1.get(d1)); + } + assertEquals(src.f2.size(), dst.f2.size()); + Iterator srcf2 = src.f2.keySet().iterator(); + Iterator dstf2 = dst.f2.keySet().iterator(); + while (srcf2.hasNext()) { + String s2 = srcf2.next(); + String d2 = dstf2.next(); + assertEquals(s2, d2); + assertEquals(src.f2.get(s2), dst.f2.get(d2)); + } + assertFalse(it.hasNext()); + } @MessagePackUnpackable public static class SampleMapTypes { @@ -351,7 +496,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testFieldModifiers() throws Exception { + public void testFieldModifiers01() throws Exception { FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil .newEnhancedInstance(FieldModifiersClass.class); src.f0 = 0; @@ -372,6 +517,31 @@ public class TestMessagePackUnpackable extends TestCase { assertTrue(src.f4 != dst.f4); } + @Test + public void testFieldModifiers02() throws Exception { + FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil + .initEnhancedInstance(mpo, FieldModifiersClass.class); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + assertFalse(it.hasNext()); + } + @MessagePackUnpackable public static class FieldModifiersClass { public int f0; @@ -385,7 +555,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testNestedAnnotatedFieldClass() throws Exception { + public void testNestedAnnotatedFieldClass01() throws Exception { NestedClass src2 = (NestedClass) PackUnpackUtil .newEnhancedInstance(NestedClass.class); BaseClass src = (BaseClass) PackUnpackUtil @@ -407,6 +577,33 @@ public class TestMessagePackUnpackable extends TestCase { assertTrue(src2.f2 == dst.f1.f2); } + @Test + public void testNestedAnnotatedFieldClass02() throws Exception { + NestedClass src2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass src = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + src.f0 = 0; + src2.f2 = 2; + src.f1 = src2; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + NestedClass dst2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass dst = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + dst.f1 = dst2; + dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo, dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src2.f2 == dst.f1.f2); + assertFalse(it.hasNext()); + } + @MessagePackUnpackable public static class BaseClass { public int f0; @@ -425,7 +622,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testExtendedClass() throws Exception { + public void testExtendedClass01() throws Exception { SampleSubClass src = (SampleSubClass) PackUnpackUtil .newEnhancedInstance(SampleSubClass.class); src.f0 = 0; @@ -452,6 +649,38 @@ public class TestMessagePackUnpackable extends TestCase { assertTrue(src.f8 == dst.f8); assertTrue(src.f9 != dst.f9); } + + @Test + public void testExtendedClass02() throws Exception { + SampleSubClass src = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f8 = 8; + src.f9 = 9; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + Iterator it = pac.iterator(); + assertTrue(it.hasNext()); + MessagePackObject mpo = it.next(); + SampleSubClass dst = (SampleSubClass) PackUnpackUtil + .initEnhancedInstance(mpo, SampleSubClass.class); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + assertTrue(src.f5 == dst.f5); + assertTrue(src.f6 == dst.f6); + assertTrue(src.f8 == dst.f8); + assertTrue(src.f9 != dst.f9); + assertFalse(it.hasNext()); + } @MessagePackUnpackable public static class SampleSubClass extends SampleSuperClass { From df8a3e870ae9c350394ded3e2e73f9c0bca023f7 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 23 Sep 2010 16:18:23 +0900 Subject: [PATCH 254/259] java: refactor annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 678 ++++++------------ 1 file changed, 208 insertions(+), 470 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index a337ff94..ddf30b43 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -81,9 +81,13 @@ public class PackUnpackUtil { static final String VARIABLE_NAME_MAP = "_$$_map"; + static final String VARIABLE_NAME_KEY = "_$$_key"; + + static final String VARIABLE_NAME_VAL = "_$$_val"; + static final String VARIABLE_NAME_ITER = "_$$_iter"; - static final String VARIABLE_NAME_MPO1 = "_$$_mpo1"; + static final String VARIABLE_NAME_MPO = "_$$_mpo"; static final String VARIABLE_NAME_I = "i"; @@ -144,6 +148,26 @@ public class PackUnpackUtil { static final String METHOD_NAME_UNPACKMAP = "unpackMap"; static final String METHOD_NAME_ASARRAY = "asArray"; + + static final String METHOD_NAME_ASBOOLEAN = "asBoolean"; + + static final String METHOD_NAME_ASBYTE = "asByte"; + + static final String METHOD_NAME_ASSHORT = "asShort"; + + static final String METHOD_NAME_ASINT = "asInt"; + + static final String METHOD_NAME_ASFLOAT = "asFloat"; + + static final String METHOD_NAME_ASLONG = "asLong"; + + static final String METHOD_NAME_ASDOUBLE = "asDouble"; + + static final String METHOD_NAME_ASSTRING = "asString"; + + static final String METHOD_NAME_ASBYTEARRAY = "asByteArray"; + + static final String METHOD_NAME_ASBIGINTEGER = "asBigInteger"; } public static class Enhancer { @@ -322,7 +346,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); for (Field field : fields) { - insertCodeOfMessagePack(sb, field); + insertCodeOfMessagePackCall(sb, field); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); // System.out.println("messagePack method: " + sb.toString()); @@ -330,7 +354,7 @@ public class PackUnpackUtil { enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessagePack(StringBuilder sb, Field field) { + private void insertCodeOfMessagePackCall(StringBuilder sb, Field field) { sb.append(Constants.VARIABLE_NAME_PK); sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_PACK); @@ -371,20 +395,25 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); - for (Field field : fields) { - insertCodeOfMessageUnpack(sb, field, field.getType()); - } + insertCodeOfMessageUnpackCalls(sb, fields); sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); // System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessageUnpack(StringBuilder sb, Field field, - Class type) throws NotFoundException { + private void insertCodeOfMessageUnpackCalls(StringBuilder sb, + Field[] fields) throws NotFoundException { + for (Field field : fields) { + insertCodeOfMessageUnpackCall(sb, field, field.getType()); + } + } + + private void insertCodeOfMessageUnpackCall(StringBuilder sb, + Field field, Class type) throws NotFoundException { if (type.isPrimitive()) { // primitive type - insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type); } else if (type.equals(Boolean.class) || // Boolean type.equals(Byte.class) || // Byte type.equals(Double.class) || // Double @@ -393,28 +422,29 @@ public class PackUnpackUtil { type.equals(Long.class) || // Long type.equals(Short.class)) { // Short // reference type (wrapper type) - insertCodeOfMessageUnpackForWrapperTypes(sb, field, type); + insertCodeOfMessageUnpackCallForWrapperTypes(sb, field, type); } else if (type.equals(BigInteger.class) || // BigInteger type.equals(String.class) || // String type.equals(byte[].class)) { // byte[] // reference type (other type) - insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type); } else if (List.class.isAssignableFrom(type)) { // List - insertCodeOfMessageUnpackForListType(sb, field, type); + insertCodeOfMessageUnpackCallForListType(sb, field, type); } else if (Map.class.isAssignableFrom(type)) { // Map - insertCodeOfMessageUnpackForMapType(sb, field, type); + insertCodeOfMessageUnpackCallForMapType(sb, field, type); } else if (MessageUnpackable.class.isAssignableFrom(type) || (getCache(type.getName()) != null)) { // MessageUnpackable - insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type); + insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, field, + type); } else { throw new NotFoundException("unknown type: " + type.getName()); } } - private void insertCodeOfMessageUnpackForPrimitiveTypes( + private void insertCodeOfMessageUnpackCallForPrimitiveTypes( StringBuilder sb, Field field, Class type) throws NotFoundException { // insert a right variable @@ -461,8 +491,9 @@ public class PackUnpackUtil { } } - private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb, - Field field, Class type) throws NotFoundException { + private void insertCodeOfMessageUnpackCallForWrapperTypes( + StringBuilder sb, Field field, Class type) + throws NotFoundException { // insert a right variable if (field != null) { sb.append(field.getName()); @@ -503,7 +534,7 @@ public class PackUnpackUtil { } } - private void insertCodeOfMessageUnpackForListType(StringBuilder sb, + private void insertCodeOfMessageUnpackCallForListType(StringBuilder sb, Field field, Class type) throws NotFoundException { ParameterizedType generic = (ParameterizedType) field .getGenericType(); @@ -570,7 +601,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_ADD); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - insertCodeOfMessageUnpack(sb, null, genericType); + insertCodeOfMessageUnpackCall(sb, null, genericType); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); @@ -579,7 +610,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } - private void insertCodeOfMessageUnpackForMapType(StringBuilder sb, + private void insertCodeOfMessageUnpackCallForMapType(StringBuilder sb, Field field, Class type) throws NotFoundException { ParameterizedType generic = (ParameterizedType) field .getGenericType(); @@ -647,10 +678,10 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_PUT); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - insertCodeOfMessageUnpack(sb, null, genericType0); + insertCodeOfMessageUnpackCall(sb, null, genericType0); sb.append(Constants.CHAR_NAME_COMMA); sb.append(Constants.CHAR_NAME_SPACE); - insertCodeOfMessageUnpack(sb, null, genericType1); + insertCodeOfMessageUnpackCall(sb, null, genericType1); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); @@ -659,7 +690,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } - private void insertCodeOfMessageUnpackForMsgUnpackableType( + private void insertCodeOfMessageUnpackCallForMsgUnpackableType( StringBuilder sb, Field field, Class type) { // insert a right variable // ignore sb.append(Constants.VARIABLE_NAME_PK); @@ -687,7 +718,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); sb.append(MessagePackObject.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_MPO1); + sb.append(Constants.VARIABLE_NAME_MPO); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.KEYWORD_THROWS); @@ -699,7 +730,7 @@ public class PackUnpackUtil { insertCodeOfMessagePackObjectArrayGet(sb); insertCodeOfMesageConvertCalls(sb, fields); sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - //System.out.println("messageConvert method: " + sb.toString()); + // System.out.println("messageConvert method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } @@ -714,7 +745,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_EQUAL); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_MPO1); + sb.append(Constants.VARIABLE_NAME_MPO); sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_ASARRAY); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); @@ -732,455 +763,161 @@ public class PackUnpackUtil { } private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f, - Class c, int i, String name) { + Class c, int i, String v) { if (c.isPrimitive()) { // primitive type - // f0 = objs[0].intValue(); - if (c.equals(boolean.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asBoolean"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(byte.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asByte"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(short.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asShort"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(int.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asInt"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(long.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asLong"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(float.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asFloat"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(double.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asDouble"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else { - throw new PackUnpackUtilException("fatal error: " - + c.getName()); - } + insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v); } else { // reference type - if (c.equals(Boolean.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Boolean.class.getName()); + if (c.equals(Boolean.class) || c.equals(Byte.class) + || c.equals(Short.class) || c.equals(Integer.class) + || c.equals(Float.class) || c.equals(Long.class) + || c.equals(Double.class)) { + // wrapper type + insertCodeOfMessageConvertCallForWrapTypes(sb, f, c, i, v); + } else if (c.equals(String.class) || c.equals(byte[].class) + || c.equals(BigInteger.class)) { + insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v); + } else if (List.class.isAssignableFrom(c)) { + insertCodeOfMessageConvertCallForList(sb, f, c, i); + } else if (Map.class.isAssignableFrom(c)) { + insertCodeOfMessageConveretCallForMap(sb, f, c, i); + } else if (MessageConvertable.class.isAssignableFrom(c) + || (getCache(c.getName()) != null)) { + // TODO + // TODO + // TODO + // ((MessageConvertable)f_i).messageConvert(ary[i]); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageConvertable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_DOT); - sb.append("asBoolean"); + sb.append(Constants.METHOD_NAME_MSGCONVERT); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Byte.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Byte.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asByte"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Short.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Short.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asShort"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Integer.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Integer.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asInt"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Long.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Long.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asLong"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Float.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Float.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asFloat"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(Double.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - } - sb.append(Constants.KEYWORD_NEW); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Double.class.getName()); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - if (f != null) { - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asDouble"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } } else { - if (c.equals(String.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asString"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(byte[].class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asByteArray"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (c.equals(BigInteger.class)) { - if (f != null) { - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.CHAR_NAME_EQUAL); - sb.append(Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - } else { - sb.append(name); - } - sb.append(Constants.CHAR_NAME_DOT); - sb.append("asBigInteger"); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - if (f != null) { - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } - } else if (List.class.isAssignableFrom(c)) { - insertCodeOfMessageConvertCallForList(sb, f, c, i); - } else if (Map.class.isAssignableFrom(c)) { - insertCodeOfMessageConveretCallForMap(sb, f, c, i); - } else if (MessageConvertable.class.isAssignableFrom(c) - || (getCache(c.getName()) != null)) { - // TODO - // TODO - // TODO - // ((MessageConvertable)f_i).messageConvert(ary[i]); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(MessageConvertable.class.getName()); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_DOT); - sb.append(Constants.METHOD_NAME_MSGCONVERT); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); - } else { - throw new MessageTypeException("Type error: " - + c.getName()); - } + throw new MessageTypeException("Type error: " + c.getName()); } } } + private void insertCodeOfMessageConvertCallForPrimTypes( + StringBuilder sb, Field f, Class c, int i, String name) { + // f0 = objs[0].intValue(); + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(name); + } + sb.append(Constants.CHAR_NAME_DOT); + if (c.equals(boolean.class)) { + sb.append(Constants.METHOD_NAME_ASBOOLEAN); + } else if (c.equals(byte.class)) { + sb.append(Constants.METHOD_NAME_ASBYTE); + } else if (c.equals(short.class)) { + sb.append(Constants.METHOD_NAME_ASSHORT); + } else if (c.equals(int.class)) { + sb.append(Constants.METHOD_NAME_ASINT); + } else if (c.equals(float.class)) { + sb.append(Constants.METHOD_NAME_ASFLOAT); + } else if (c.equals(long.class)) { + sb.append(Constants.METHOD_NAME_ASLONG); + } else if (c.equals(double.class)) { + sb.append(Constants.METHOD_NAME_ASDOUBLE); + } else if (c.equals(String.class)) { + sb.append(Constants.METHOD_NAME_ASSTRING); + } else if (c.equals(byte[].class)) { + sb.append(Constants.METHOD_NAME_ASBYTEARRAY); + } else if (c.equals(BigInteger.class)) { + sb.append(Constants.METHOD_NAME_ASBIGINTEGER); + } else { + throw new MessageTypeException("Type error: " + c.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + + private void insertCodeOfMessageConvertCallForWrapTypes( + StringBuilder sb, Field f, Class c, int i, String v) { + if (f != null) { + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + if (c.equals(Boolean.class)) { + sb.append(Boolean.class.getName()); + } else if (c.equals(Byte.class)) { + sb.append(Byte.class.getName()); + } else if (c.equals(Short.class)) { + sb.append(Short.class.getName()); + } else if (c.equals(Integer.class)) { + sb.append(Integer.class.getName()); + } else if (c.equals(Float.class)) { + sb.append(Float.class.getName()); + } else if (c.equals(Long.class)) { + sb.append(Long.class.getName()); + } else if (c.equals(Double.class)) { + sb.append(Double.class.getName()); + } else { + throw new MessageTypeException("Type error: " + c.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + if (f != null) { + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + } else { + sb.append(v); + } + sb.append(Constants.CHAR_NAME_DOT); + if (c.equals(Boolean.class)) { + sb.append(Constants.METHOD_NAME_ASBOOLEAN); + } else if (c.equals(Byte.class)) { + sb.append(Constants.METHOD_NAME_ASBYTE); + } else if (c.equals(Short.class)) { + sb.append(Constants.METHOD_NAME_ASSHORT); + } else if (c.equals(Integer.class)) { + sb.append(Constants.METHOD_NAME_ASINT); + } else if (c.equals(Float.class)) { + sb.append(Constants.METHOD_NAME_ASFLOAT); + } else if (c.equals(Long.class)) { + sb.append(Constants.METHOD_NAME_ASLONG); + } else if (c.equals(Double.class)) { + sb.append(Constants.METHOD_NAME_ASDOUBLE); + } else { + throw new MessageTypeException("Type error: " + c.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (f != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + private void insertCodeOfMessageConvertCallForList(StringBuilder sb, Field field, Class type, int i) { ParameterizedType generic = (ParameterizedType) field @@ -1264,7 +1001,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); sb.append(MessagePackObject.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); - sb.append("_$$_val"); + sb.append(Constants.VARIABLE_NAME_VAL); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_EQUAL); sb.append(Constants.CHAR_NAME_SPACE); @@ -1284,7 +1021,8 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_ADD); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - insertCodeOfMessageConvertCall(sb, null, genericType, -1, "_$$_val"); + insertCodeOfMessageConvertCall(sb, null, genericType, -1, + Constants.VARIABLE_NAME_VAL); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); @@ -1380,7 +1118,7 @@ public class PackUnpackUtil { // block map. sb.append(MessagePackObject.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); - sb.append("_$$_key"); + sb.append(Constants.VARIABLE_NAME_KEY); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_EQUAL); sb.append(Constants.CHAR_NAME_SPACE); @@ -1396,7 +1134,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); sb.append(MessagePackObject.class.getName()); sb.append(Constants.CHAR_NAME_SPACE); - sb.append("_$$_val"); + sb.append(Constants.VARIABLE_NAME_VAL); sb.append(Constants.CHAR_NAME_SPACE); sb.append(Constants.CHAR_NAME_EQUAL); sb.append(Constants.CHAR_NAME_SPACE); @@ -1407,7 +1145,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_DOT); sb.append(Constants.METHOD_NAME_GET); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append("_$$_key"); + sb.append(Constants.VARIABLE_NAME_KEY); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); @@ -1417,11 +1155,11 @@ public class PackUnpackUtil { sb.append(Constants.METHOD_NAME_PUT); sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); insertCodeOfMessageConvertCall(sb, null, genericType0, -1, - "_$$_key"); + Constants.VARIABLE_NAME_KEY); sb.append(Constants.CHAR_NAME_COMMA); sb.append(Constants.CHAR_NAME_SPACE); insertCodeOfMessageConvertCall(sb, null, genericType1, -1, - "_$$_val"); + Constants.VARIABLE_NAME_VAL); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); From e121f344077bad630fbe4c249c5c80f6a3932a35 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 23 Sep 2010 17:26:31 +0900 Subject: [PATCH 255/259] java: refactor annotation-utilities and edit those test programs --- .../util/annotation/PackUnpackUtil.java | 182 +++++++++++++----- .../annotation/TestMessagePackUnpackable.java | 17 +- 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index ddf30b43..6e6a6651 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -41,10 +41,14 @@ public class PackUnpackUtil { static final String KEYWORD_FOR = "for"; + static final String KEYWORD_IF = "if"; + static final String KEYWORD_THROWS = "throws"; static final String KEYWORD_NEW = "new"; + static final String KEYWORD_NULL = "null"; + static final String CHAR_NAME_SPACE = " "; static final String CHAR_NAME_COMMA = ","; @@ -409,42 +413,43 @@ public class PackUnpackUtil { } } - private void insertCodeOfMessageUnpackCall(StringBuilder sb, - Field field, Class type) throws NotFoundException { - if (type.isPrimitive()) { + private void insertCodeOfMessageUnpackCall(StringBuilder sb, Field f, + Class c) throws NotFoundException { + if (c.isPrimitive()) { // primitive type - insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type); - } else if (type.equals(Boolean.class) || // Boolean - type.equals(Byte.class) || // Byte - type.equals(Double.class) || // Double - type.equals(Float.class) || // Float - type.equals(Integer.class) || // Integer - type.equals(Long.class) || // Long - type.equals(Short.class)) { // Short + insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c); + } else if (c.equals(Boolean.class) || // Boolean + c.equals(Byte.class) || // Byte + c.equals(Double.class) || // Double + c.equals(Float.class) || // Float + c.equals(Integer.class) || // Integer + c.equals(Long.class) || // Long + c.equals(Short.class)) { // Short // reference type (wrapper type) - insertCodeOfMessageUnpackCallForWrapperTypes(sb, field, type); - } else if (type.equals(BigInteger.class) || // BigInteger - type.equals(String.class) || // String - type.equals(byte[].class)) { // byte[] + insertCodeOfMessageUnpackCallForWrapTypes(sb, f, c); + } else if (c.equals(BigInteger.class) || // BigInteger + c.equals(String.class) || // String + c.equals(byte[].class)) { // byte[] // reference type (other type) - insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type); - } else if (List.class.isAssignableFrom(type)) { + insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c); + } else if (List.class.isAssignableFrom(c)) { // List - insertCodeOfMessageUnpackCallForListType(sb, field, type); - } else if (Map.class.isAssignableFrom(type)) { + insertCodeOfMessageUnpackCallForListType(sb, f, c); + } else if (Map.class.isAssignableFrom(c)) { // Map - insertCodeOfMessageUnpackCallForMapType(sb, field, type); - } else if (MessageUnpackable.class.isAssignableFrom(type) - || (getCache(type.getName()) != null)) { + insertCodeOfMessageUnpackCallForMapType(sb, f, c); + } else if (getCache(c.getName()) != null) { + // cached + insertCodeOfMessageUnpackCallForEnhancedType(sb, f, c); + } else if (MessageUnpackable.class.isAssignableFrom(c)) { // MessageUnpackable - insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, field, - type); + insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c); } else { - throw new NotFoundException("unknown type: " + type.getName()); + throw new NotFoundException("unknown type: " + c.getName()); } } - private void insertCodeOfMessageUnpackCallForPrimitiveTypes( + private void insertCodeOfMessageUnpackCallForPrimTypes( StringBuilder sb, Field field, Class type) throws NotFoundException { // insert a right variable @@ -491,7 +496,7 @@ public class PackUnpackUtil { } } - private void insertCodeOfMessageUnpackCallForWrapperTypes( + private void insertCodeOfMessageUnpackCallForWrapTypes( StringBuilder sb, Field field, Class type) throws NotFoundException { // insert a right variable @@ -690,8 +695,42 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } + private void insertCodeOfMessageUnpackCallForEnhancedType( + StringBuilder sb, Field f, Class c) { + c = this.getCache(c.getName()); + insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c); + } + private void insertCodeOfMessageUnpackCallForMsgUnpackableType( - StringBuilder sb, Field field, Class type) { + StringBuilder sb, Field f, Class c) { + // if (fi == null) { fi = new Foo_$$_Enhanced(); } + sb.append(Constants.KEYWORD_IF); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NULL); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(c.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + // insert a right variable // ignore sb.append(Constants.VARIABLE_NAME_PK); sb.append(Constants.CHAR_NAME_DOT); @@ -700,7 +739,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); sb.append(MessageUnpackable.class.getName()); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(field.getName()); + sb.append(f.getName()); sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); sb.append(Constants.CHAR_NAME_SEMICOLON); sb.append(Constants.CHAR_NAME_SPACE); @@ -779,35 +818,72 @@ public class PackUnpackUtil { } else if (List.class.isAssignableFrom(c)) { insertCodeOfMessageConvertCallForList(sb, f, c, i); } else if (Map.class.isAssignableFrom(c)) { - insertCodeOfMessageConveretCallForMap(sb, f, c, i); - } else if (MessageConvertable.class.isAssignableFrom(c) - || (getCache(c.getName()) != null)) { - // TODO - // TODO - // TODO - // ((MessageConvertable)f_i).messageConvert(ary[i]); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(MessageConvertable.class.getName()); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(f.getName()); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_DOT); - sb.append(Constants.METHOD_NAME_MSGCONVERT); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - sb.append(Constants.VARIABLE_NAME_ARRAY); - sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); - sb.append(i); - sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - sb.append(Constants.CHAR_NAME_SEMICOLON); - sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessageConvertCallForMap(sb, f, c, i); + } else if ((getCache(c.getName()) != null)) { + insertCodeOfMessageConvertCallForEnhancedType(sb, f, c, i); + } else if (MessageConvertable.class.isAssignableFrom(c)) { + insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i); } else { throw new MessageTypeException("Type error: " + c.getName()); } } } + private void insertCodeOfMessageConvertCallForEnhancedType( + StringBuilder sb, Field f, Class c, int i) { + c = getCache(c.getName()); + insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i); + } + + private void insertCodeOfMessageConvertCallForMsgConvtblType( + StringBuilder sb, Field f, Class c, int i) { + // if (fi == null) { fi = new Foo_$$_Enhanced(); } + sb.append(Constants.KEYWORD_IF); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NULL); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(c.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // ((MessageConvertable)f_i).messageConvert(ary[i]); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageConvertable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(f.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_MSGCONVERT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_ARRAY); + sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET); + sb.append(i); + sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + private void insertCodeOfMessageConvertCallForPrimTypes( StringBuilder sb, Field f, Class c, int i, String name) { // f0 = objs[0].intValue(); @@ -1031,7 +1107,7 @@ public class PackUnpackUtil { sb.append(Constants.CHAR_NAME_SPACE); } - private void insertCodeOfMessageConveretCallForMap(StringBuilder sb, + private void insertCodeOfMessageConvertCallForMap(StringBuilder sb, Field f, Class c, int i) { ParameterizedType generic = (ParameterizedType) f.getGenericType(); Class genericType0 = (Class) generic.getActualTypeArguments()[0]; diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 4b1e6664..08c4fa82 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -208,7 +208,7 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f2.get(i), dst.f2.get(i)); } } - + public void testListTypes02() throws Exception { SampleListTypes src = (SampleListTypes) PackUnpackUtil .newEnhancedInstance(SampleListTypes.class); @@ -291,7 +291,7 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f2.get(s2), dst.f2.get(d2)); } } - + public void testMapTypes02() throws Exception { SampleMapTypes src = (SampleMapTypes) PackUnpackUtil .newEnhancedInstance(SampleMapTypes.class); @@ -565,11 +565,8 @@ public class TestMessagePackUnpackable extends TestCase { src.f1 = src2; ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(src); - NestedClass dst2 = (NestedClass) PackUnpackUtil - .newEnhancedInstance(NestedClass.class); BaseClass dst = (BaseClass) PackUnpackUtil .newEnhancedInstance(BaseClass.class); - dst.f1 = dst2; ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); @@ -593,12 +590,8 @@ public class TestMessagePackUnpackable extends TestCase { Iterator it = pac.iterator(); assertTrue(it.hasNext()); MessagePackObject mpo = it.next(); - NestedClass dst2 = (NestedClass) PackUnpackUtil - .newEnhancedInstance(NestedClass.class); - BaseClass dst = (BaseClass) PackUnpackUtil - .newEnhancedInstance(BaseClass.class); - dst.f1 = dst2; - dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo, dst); + BaseClass dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo, + BaseClass.class); assertTrue(src.f0 == dst.f0); assertTrue(src2.f2 == dst.f1.f2); assertFalse(it.hasNext()); @@ -649,7 +642,7 @@ public class TestMessagePackUnpackable extends TestCase { assertTrue(src.f8 == dst.f8); assertTrue(src.f9 != dst.f9); } - + @Test public void testExtendedClass02() throws Exception { SampleSubClass src = (SampleSubClass) PackUnpackUtil From 22ddd91b1f73adc8b0cc402e5346504df4f4a935 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 23 Sep 2010 20:38:54 +0900 Subject: [PATCH 256/259] java: add several API to annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 6e6a6651..e509cd8e 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -193,12 +193,12 @@ public class PackUnpackUtil { classCache.putIfAbsent(origName, enhClass); } - protected Class generate(Class origClass) + protected Class generate(Class origClass, boolean packUnpackable) throws NotFoundException, CannotCompileException { String origName = origClass.getName(); String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; CtClass origCtClass = pool.get(origName); - checkClassValidation(origClass); + checkClassValidation(origClass, packUnpackable); checkDefaultConstructorValidation(origClass); CtClass enhCtClass = pool.makeClass(enhName); setSuperclass(enhCtClass, origCtClass); @@ -211,7 +211,7 @@ public class PackUnpackUtil { return createClass(enhCtClass); } - private void checkClassValidation(Class origClass) { + private void checkClassValidation(Class origClass, boolean packUnpackable) { // not public, abstract, final int mod = origClass.getModifiers(); if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) @@ -223,7 +223,9 @@ public class PackUnpackUtil { throwClassValidationException(origClass); } // annotation - checkPackUnpackAnnotation(origClass); + if (!packUnpackable) { + checkPackUnpackAnnotation(origClass); + } } private static void throwClassValidationException(Class origClass) { @@ -1251,18 +1253,17 @@ public class PackUnpackUtil { } private static Enhancer enhancer; - - public static Class getEnhancedClass(Class origClass) { + + public static void registerEnhancedClass(Class origClass, boolean packUnpackable) { if (enhancer == null) { enhancer = new Enhancer(); } - String origName = origClass.getName(); Class enhClass = enhancer.getCache(origName); if (enhClass == null) { // generate a class object related to the original class try { - enhClass = enhancer.generate(origClass); + enhClass = enhancer.generate(origClass, packUnpackable); } catch (NotFoundException e) { throw new PackUnpackUtilException(e.getMessage(), e); } catch (CannotCompileException e) { @@ -1270,33 +1271,57 @@ public class PackUnpackUtil { } // set the generated class to the cache enhancer.setCache(origName, enhClass); + } + } + + public static void registerEnhancedClass(Class origClass) { + registerEnhancedClass(origClass, false); + } + + public static boolean isRegistered(Class origClass) { + if (enhancer == null) { + enhancer = new Enhancer(); } - return enhClass; + return enhancer.getCache(origClass.getName()) != null; } - public static Object newEnhancedInstance(Class origClass) { + public static Class getEnhancedClass(Class origClass, boolean packUnpackable) { + if (!isRegistered(origClass)) { + registerEnhancedClass(origClass, packUnpackable); + } + return enhancer.getCache(origClass.getName()); + } + + public static Class getEnhancedClass(Class origClass) { + return getEnhancedClass(origClass, false); + } + + public static Object newEnhancedInstance(Class origClass, boolean packUnpackable) { try { - Class enhClass = getEnhancedClass(origClass); + Class enhClass = getEnhancedClass(origClass, packUnpackable); // create a new object of the generated class return enhClass.newInstance(); } catch (InstantiationException e) { throw new PackUnpackUtilException(e.getMessage(), e); } catch (IllegalAccessException e) { throw new PackUnpackUtilException(e.getMessage(), e); - } + } } - + + public static Object newEnhancedInstance(Class origClass) { + return newEnhancedInstance(origClass, false); + } + public static Object initEnhancedInstance(MessagePackObject obj, - Class origClass) { - Object ret = newEnhancedInstance(origClass); + Class origClass, boolean packUnpackable) { + Object ret = newEnhancedInstance(origClass, packUnpackable); ((MessageConvertable) ret).messageConvert(obj); return ret; } public static Object initEnhancedInstance(MessagePackObject obj, - Object origObj) { - ((MessageConvertable) origObj).messageConvert(obj); - return origObj; + Class origClass) { + return initEnhancedInstance(obj, origClass, false); } @MessagePackUnpackable @@ -1326,7 +1351,6 @@ public class PackUnpackUtil { } public static void main(final String[] args) throws Exception { - PackUnpackUtil.getEnhancedClass(Image.class); Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); src.title = "msgpack"; src.uri = "http://msgpack.org/"; From 34c008adce9ffd716dfcac11b35490b6b1244f3f Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 23 Sep 2010 20:40:50 +0900 Subject: [PATCH 257/259] java: refactor annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index e509cd8e..0aeb6d87 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -211,7 +211,8 @@ public class PackUnpackUtil { return createClass(enhCtClass); } - private void checkClassValidation(Class origClass, boolean packUnpackable) { + private void checkClassValidation(Class origClass, + boolean packUnpackable) { // not public, abstract, final int mod = origClass.getModifiers(); if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) @@ -1253,8 +1254,9 @@ public class PackUnpackUtil { } private static Enhancer enhancer; - - public static void registerEnhancedClass(Class origClass, boolean packUnpackable) { + + public static void registerEnhancedClass(Class origClass, + boolean packUnpackable) { if (enhancer == null) { enhancer = new Enhancer(); } @@ -1271,13 +1273,13 @@ public class PackUnpackUtil { } // set the generated class to the cache enhancer.setCache(origName, enhClass); - } + } } - + public static void registerEnhancedClass(Class origClass) { registerEnhancedClass(origClass, false); } - + public static boolean isRegistered(Class origClass) { if (enhancer == null) { enhancer = new Enhancer(); @@ -1285,18 +1287,20 @@ public class PackUnpackUtil { return enhancer.getCache(origClass.getName()) != null; } - public static Class getEnhancedClass(Class origClass, boolean packUnpackable) { + public static Class getEnhancedClass(Class origClass, + boolean packUnpackable) { if (!isRegistered(origClass)) { registerEnhancedClass(origClass, packUnpackable); } return enhancer.getCache(origClass.getName()); } - + public static Class getEnhancedClass(Class origClass) { return getEnhancedClass(origClass, false); } - public static Object newEnhancedInstance(Class origClass, boolean packUnpackable) { + public static Object newEnhancedInstance(Class origClass, + boolean packUnpackable) { try { Class enhClass = getEnhancedClass(origClass, packUnpackable); // create a new object of the generated class @@ -1305,13 +1309,13 @@ public class PackUnpackUtil { throw new PackUnpackUtilException(e.getMessage(), e); } catch (IllegalAccessException e) { throw new PackUnpackUtilException(e.getMessage(), e); - } + } } - + public static Object newEnhancedInstance(Class origClass) { return newEnhancedInstance(origClass, false); } - + public static Object initEnhancedInstance(MessagePackObject obj, Class origClass, boolean packUnpackable) { Object ret = newEnhancedInstance(origClass, packUnpackable); From 93bed9c5df6d4fe7a0defdaeb2f158e27d4feb1d Mon Sep 17 00:00:00 2001 From: tanakh Date: Fri, 24 Sep 2010 01:24:13 +0900 Subject: [PATCH 258/259] haskell: finish template-haskell deriving implement --- haskell/msgpack.cabal | 4 ++ haskell/src/Data/MessagePack/Derive.hs | 62 ++++++++++++++++++-------- haskell/test/Test.hs | 2 +- haskell/test/UserData.hs | 26 +++++++++++ 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 9c67bdce..99502732 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -15,6 +15,10 @@ Stability: Experimental Cabal-Version: >= 1.6 Build-Type: Simple +Extra-source-files: + test/Test.hs + test/UserData.hs + Library Build-depends: base >=4 && <5, transformers >= 0.2.1 && < 0.2.2, diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs index cfdb6588..e9984730 100644 --- a/haskell/src/Data/MessagePack/Derive.hs +++ b/haskell/src/Data/MessagePack/Derive.hs @@ -11,10 +11,11 @@ import Language.Haskell.TH import Data.MessagePack.Pack import Data.MessagePack.Unpack +import Data.MessagePack.Object deriveUnpack :: Name -> Q [Dec] deriveUnpack typName = do - TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + TyConI (DataD _ name _ cons _) <- reify typName return [ InstanceD [] (AppT (ConT ''Unpackable) (ConT name)) @@ -24,20 +25,19 @@ deriveUnpack typName = do where body (NormalC conName elms) = DoE - [ BindS (tupOrList $ map VarP names) (VarE 'get) + [ BindS (tupOrListP $ map VarP names) (VarE 'get) , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] where names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms - tupOrList ls - | length ls <= 1 = ListP ls - | otherwise = TupP ls + body (RecC conName elms) = + body (NormalC conName $ map (\(_, b, c) -> (b, c)) elms) ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f) derivePack :: Name -> Q [Dec] derivePack typName = do - TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + TyConI (DataD _ name _ cons _) <- reify typName return [ InstanceD [] (AppT (ConT ''Packable) (ConT name)) @@ -48,27 +48,53 @@ derivePack typName = do body (NormalC conName elms) = Clause [ ConP conName $ map VarP names ] - (NormalB $ AppE (VarE 'put) $ tupOrList $ map VarE names) [] + (NormalB $ AppE (VarE 'put) $ tupOrListE $ map VarE names) [] where names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms - tupOrList ls - | length ls <= 1 = ListE ls - | otherwise = TupE ls + body (RecC conName elms) = + body (NormalC conName $ map (\(_, b, c) -> (b, c)) elms) deriveObject :: Name -> Q [Dec] deriveObject typName = do g <- derivePack typName p <- deriveUnpack typName - {- - TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName + + TyConI (DataD _ name _ cons _) <- reify typName let o = InstanceD [] (AppT (ConT ''OBJECT) (ConT name)) - [ FunD 'toObject (map toObjectBody cons) ] - -} - return $ g ++ p -- ++ [o] -{- + [ FunD 'toObject (map toObjectBody cons), + FunD 'tryFromObject [Clause [ VarP oname ] + (NormalB $ ch $ map tryFromObjectBody cons) []]] + + return $ g ++ p ++ [o] where toObjectBody (NormalC conName elms) = Clause - [ ConP conP --} + [ ConP conName $ map VarP names ] + (NormalB $ AppE (VarE 'toObject) $ tupOrListE $ map VarE names) [] + where + names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms + toObjectBody (RecC conName elms) = + toObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms) + + tryFromObjectBody (NormalC conName elms) = + DoE + [ BindS (tupOrListP $ map VarP names) (AppE (VarE 'tryFromObject) (VarE oname)) + , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] + where + names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms + tryFromObjectBody (RecC conName elms) = + tryFromObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms) + + oname = mkName "o" + ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f) + +tupOrListP :: [Pat] -> Pat +tupOrListP ls + | length ls <= 1 = ListP ls + | otherwise = TupP ls + +tupOrListE :: [Exp] -> Exp +tupOrListE ls + | length ls <= 1 = ListE ls + | otherwise = TupE ls diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs index a73ac9ab..43af2efc 100644 --- a/haskell/test/Test.hs +++ b/haskell/test/Test.hs @@ -7,7 +7,7 @@ import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as L import Data.MessagePack -mid :: (ObjectGet a, ObjectPut a) => a -> a +mid :: (Packable a, Unpackable a) => a -> a mid = unpack . pack prop_mid_int a = a == mid a diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs index 8aced13f..73647ff1 100644 --- a/haskell/test/UserData.hs +++ b/haskell/test/UserData.hs @@ -10,6 +10,13 @@ data T $(deriveObject ''T) +data U + = C { c1 :: Int, c2 :: String } + | D { d1 :: Double } + deriving (Show) + +$(deriveObject ''U) + main = do let bs = pack $ A 123 "hoge" print bs @@ -17,3 +24,22 @@ main = do let cs = pack $ B 3.14 print cs print (unpack cs :: T) + let oa = toObject $ A 123 "hoge" + print oa + print (fromObject oa :: T) + let ob = toObject $ B 3.14 + print ob + print (fromObject ob :: T) + + let ds = pack $ C 123 "hoge" + print ds + print (unpack ds :: U) + let es = pack $ D 3.14 + print es + print (unpack es :: U) + let oc = toObject $ C 123 "hoge" + print oc + print (fromObject oc :: U) + let od = toObject $ D 3.14 + print od + print (fromObject od :: U) From 894ff716647eeb63b8a04e279faa09092ac9c1c7 Mon Sep 17 00:00:00 2001 From: tanakh Date: Fri, 24 Sep 2010 03:49:31 +0900 Subject: [PATCH 259/259] haskell: fix for empty constructor --- haskell/msgpack.cabal | 2 +- haskell/src/Data/MessagePack/Derive.hs | 28 ++++++++------ haskell/test/UserData.hs | 52 +++++++++++++------------- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 99502732..98133a9e 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,5 +1,5 @@ Name: msgpack -Version: 0.4.0 +Version: 0.4.0.1 Synopsis: A Haskell binding to MessagePack Description: A Haskell binding to MessagePack diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs index e9984730..74943e9d 100644 --- a/haskell/src/Data/MessagePack/Derive.hs +++ b/haskell/src/Data/MessagePack/Derive.hs @@ -7,6 +7,7 @@ module Data.MessagePack.Derive ( ) where import Control.Applicative +import Control.Monad import Language.Haskell.TH import Data.MessagePack.Pack @@ -24,9 +25,9 @@ deriveUnpack typName = do where body (NormalC conName elms) = - DoE - [ BindS (tupOrListP $ map VarP names) (VarE 'get) - , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] + DoE $ + tupOrListP (map VarP names) (VarE 'get) ++ + [ NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] where names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms @@ -78,9 +79,9 @@ deriveObject typName = do toObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms) tryFromObjectBody (NormalC conName elms) = - DoE - [ BindS (tupOrListP $ map VarP names) (AppE (VarE 'tryFromObject) (VarE oname)) - , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] + DoE $ + tupOrListP (map VarP names) (AppE (VarE 'tryFromObject) (VarE oname)) ++ + [ NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ] where names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms tryFromObjectBody (RecC conName elms) = @@ -89,12 +90,17 @@ deriveObject typName = do oname = mkName "o" ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f) -tupOrListP :: [Pat] -> Pat -tupOrListP ls - | length ls <= 1 = ListP ls - | otherwise = TupP ls +tupOrListP :: [Pat] -> Exp -> [Stmt] +tupOrListP ls e + | length ls == 0 = + let lsname = mkName "ls" in + [ BindS (VarP lsname) e + , NoBindS $ AppE (VarE 'guard) $ AppE (VarE 'null) $ SigE (VarE lsname) (AppT ListT (ConT ''())) ] + | length ls == 1 = [ BindS (ListP ls) e ] + | otherwise = [ BindS (TupP ls) e ] tupOrListE :: [Exp] -> Exp tupOrListE ls - | length ls <= 1 = ListE ls + | length ls == 0 = SigE (ListE []) (AppT ListT (ConT ''())) + | length ls == 1 = ListE ls | otherwise = TupE ls diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs index 73647ff1..5e5d0ea0 100644 --- a/haskell/test/UserData.hs +++ b/haskell/test/UserData.hs @@ -6,40 +6,38 @@ import Data.MessagePack.Derive data T = A Int String | B Double - deriving (Show) + deriving (Show, Eq) $(deriveObject ''T) data U = C { c1 :: Int, c2 :: String } | D { d1 :: Double } - deriving (Show) + deriving (Show, Eq) $(deriveObject ''U) -main = do - let bs = pack $ A 123 "hoge" - print bs - print (unpack bs :: T) - let cs = pack $ B 3.14 - print cs - print (unpack cs :: T) - let oa = toObject $ A 123 "hoge" - print oa - print (fromObject oa :: T) - let ob = toObject $ B 3.14 - print ob - print (fromObject ob :: T) +data V + = E String | F + deriving (Show, Eq) - let ds = pack $ C 123 "hoge" - print ds - print (unpack ds :: U) - let es = pack $ D 3.14 - print es - print (unpack es :: U) - let oc = toObject $ C 123 "hoge" - print oc - print (fromObject oc :: U) - let od = toObject $ D 3.14 - print od - print (fromObject od :: U) +$(deriveObject ''V) + +test :: (OBJECT a, Show a, Eq a) => a -> IO () +test v = do + let bs = pack v + print bs + print (unpack bs == v) + + let oa = toObject v + print oa + print (fromObject oa == v) + +main = do + test $ A 123 "hoge" + test $ B 3.14 + test $ C 123 "hoge" + test $ D 3.14 + test $ E "hello" + test $ F + return () \ No newline at end of file