From 18c712cd99d1317d69f628fe58fd6ff9d10865c5 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Wed, 25 Aug 2010 21:35:52 +0900 Subject: [PATCH 01/50] object serialization with reflection and with dynamic code generation --- java/pom.xml | 12 + .../org/msgpack/tmp/DynamicCodeGenerator.java | 209 ++++++++++++++++++ .../src/main/java/org/msgpack/tmp/Image1.java | 47 ++++ .../src/main/java/org/msgpack/tmp/Image2.java | 85 +++++++ .../src/main/java/org/msgpack/tmp/Image3.java | 38 ++++ .../msgpack/tmp/ImagePackUnpackPerfTest.java | 134 +++++++++++ 6 files changed, 525 insertions(+) create mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java create mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java create mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java diff --git a/java/pom.xml b/java/pom.xml index 44806d51..6591a727 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,6 +29,12 @@ 4.8.1 test + + + jboss + javassist + 3.7.ga + @@ -101,6 +107,12 @@ MessagePack Maven2 Repository http://msgpack.org/maven2 + + + repo2.maven.org + repo2.maven.org Maven2 Repository + http://repo2.maven.org/maven2 + diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java new file mode 100644 index 00000000..9e4336d3 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java @@ -0,0 +1,209 @@ +package org.msgpack.tmp; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.util.HashMap; + +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; + +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class DynamicCodeGenerator { + + private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced"; + + private static final String SPACE = " "; + + private static final String MODIFIER_PUBLIC = "public"; + + private static final String METHOD_NAME_MESSAGEPACK = "messagePack"; + + private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack"; + + private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); + + private static final String PACKER_OBJECT_NAME = "pk"; + + private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class + .getName(); + + private static final String UNPACKER_OBJECT_NAME = "pk"; + + private HashMap> classMap; + + private HashMap objectCacheMap; + + private ClassPool pool; + + public DynamicCodeGenerator() { + classMap = new HashMap>(); + objectCacheMap = new HashMap(); + pool = ClassPool.getDefault(); + } + + public Object newEnhancedInstance(Class targetClass) throws Exception { + String targetClassName = targetClass.getName(); + //Class enhancedClass = classMap.get(targetClassName); + Object enhancedObject = objectCacheMap.get(targetClassName); + //if (enhancedClass == null) { + if (enhancedObject == null) { + CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); +// System.out.println("enhanced class name: " +// + enhancedCtClass.getName()); + addSuperclass(enhancedCtClass, targetClassName); + addConstructor(enhancedCtClass); + createMessagePackMethod(enhancedCtClass, targetClass); + createMessageUnpackMethod(enhancedCtClass, targetClass); + Class enhancedClass = loadEnhancedClass(enhancedCtClass); + //classMap.put(targetClassName, enhancedClass); + enhancedObject = enhancedClass.newInstance(); + objectCacheMap.put(targetClassName, enhancedObject); + } + //return newEnhancedInstance0(enhancedClass); + return enhancedObject; + } + + private CtClass createEnhancedCtClass(final String targetClassName) + throws Exception { + return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX); + } + + private void addSuperclass(CtClass enhancedCtClass, + final String targetClassName) throws Exception { + CtClass targetCtClass = pool.get(targetClassName); + enhancedCtClass.setSuperclass(targetCtClass); + } + + private void addConstructor(CtClass enhancedCtClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append(MODIFIER_PUBLIC).append(SPACE).append( + // enhancedCtClass.getName()).append("(").append(")") + "Image3_$$_Enhanced").append("(").append(")").append(SPACE) + .append("{ super(); }"); // TODO +// System.out.println("cons: " + sb.toString()); + CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(), + enhancedCtClass); + enhancedCtClass.addConstructor(newCtConstructor); + } + + private void createMessagePackMethod(CtClass enhancedCtClass, + Class targetClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE) + .append(METHOD_NAME_MESSAGEPACK).append("(").append( + PACKER_CLASS_TYPE_NAME).append(SPACE).append( + PACKER_OBJECT_NAME).append(")").append(SPACE).append( + "throws").append(SPACE).append("java.io.IOException") + .append(SPACE).append("{"); + Field[] fields = targetClass.getFields(); + sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append( + "(").append(fields.length).append(")").append(";"); + for (Field field : fields) { + insertCodeOfMessagePack(field, sb); + } + sb.append("}"); +// System.out.println("messagePack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); + enhancedCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessagePack(Field field, StringBuilder sb) { + Class type = field.getType(); + if (type.equals(int.class)) { + sb.append(PACKER_OBJECT_NAME).append(".").append("pack") + .append("(").append(field.getName()).append(")") + .append(";"); + } else if (type.equals(String.class)) { + sb.append(PACKER_OBJECT_NAME).append(".").append("pack") + .append("(").append(field.getName()).append(")") + .append(";"); + } else { + throw new UnsupportedOperationException(); + } + } + + private void createMessageUnpackMethod(CtClass enhancedCtClass, + Class targetClass) throws Exception { + StringBuilder sb = new StringBuilder(); + sb + .append(MODIFIER_PUBLIC) + .append(SPACE) + .append("void") + .append(SPACE) + .append(METHOD_NAME_MESSAGEUNPACK) + .append("(") + .append(UNPACKER_CLASS_TYPE_NAME) + .append(SPACE) + .append(UNPACKER_OBJECT_NAME) + .append(")") + .append(SPACE) + .append("throws") + .append(SPACE) + .append("org.msgpack.MessageTypeException, java.io.IOException") + .append(SPACE).append("{"); + Field[] fields = targetClass.getFields(); + sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";"); + // TODO + for (Field field : fields) { + insertCodeOfMessageUnpack(field, sb); + } + sb.append("}"); +// System.out.println("messageUnpack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); + enhancedCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) { + Class type = field.getType(); + if (type.equals(int.class)) { + sb.append(field.getName()).append(SPACE).append("=").append(SPACE) + .append(PACKER_OBJECT_NAME).append(".").append("unpackInt") + .append("(").append(")").append(";"); + } else if (type.equals(String.class)) { + sb.append(field.getName()).append(SPACE).append("=").append(SPACE) + .append(PACKER_OBJECT_NAME).append(".").append( + "unpackString").append("(").append(")").append(";"); + } else { + throw new UnsupportedOperationException(); + } + } + + private Class loadEnhancedClass(CtClass enhancedCtClass) + throws Exception { + return enhancedCtClass.toClass(null, null); + } + + private Object newEnhancedInstance0(Class enhancedClass) { + try { + return enhancedClass.newInstance(); + } catch (Exception e) { + throw new UnsupportedOperationException(); + } + } + + public static void main(String[] args) throws Exception { + DynamicCodeGenerator gen = new DynamicCodeGenerator(); + Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + +// System.out.println(dst.equals(src)); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java new file mode 100644 index 00000000..f1819ed6 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image1.java @@ -0,0 +1,47 @@ +package org.msgpack.tmp; + +import org.msgpack.*; +import java.io.*; + +public class Image1 implements MessagePackable, MessageUnpackable { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + pk.packArray(5); + pk.pack(uri); + pk.pack(title); + pk.pack(width); + pk.pack(height); + pk.pack(size); + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + int length = pac.unpackArray(); + if (length != 5) { + throw new MessageTypeException(); + } + uri = pac.unpackString(); + title = pac.unpackString(); + width = pac.unpackInt(); + height = pac.unpackInt(); + size = pac.unpackInt(); + } + + public boolean equals(Image1 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image1.class) { + return false; + } + return equals((Image1) obj); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java new file mode 100644 index 00000000..9c357bf3 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image2.java @@ -0,0 +1,85 @@ +package org.msgpack.tmp; + +import java.io.IOException; +import java.lang.reflect.Field; + +import org.msgpack.MessageTypeException; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class Image2 { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + messagePackWithReflection(pk); + } + + public void messagePackWithReflection(Packer pk) throws IOException { + Class cl = this.getClass(); + Field[] fields = cl.getFields(); + pk.packArray(fields.length); + for (Field field : fields) { + try { + Object obj = field.get(this); + pk.pack(obj); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + messageUnpackWithReflection(pac); + } + + public void messageUnpackWithReflection(Unpacker pac) throws IOException, + MessageTypeException { + Class cl = this.getClass(); + Field[] fields = cl.getFields(); + int length = pac.unpackArray(); + if (length != fields.length) { + throw new MessageTypeException(); + } + for (Field field : fields) { + try { + field.set(this, unpack(pac, field.getType())); + } catch (IOException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + + public static Object unpack(Unpacker pk, Class cl) throws IOException { + if (cl == int.class) { + return pk.unpackInt(); + } else if (cl == String.class) { + return pk.unpackString(); + } else { + throw new UnsupportedOperationException(); + } + } + + public boolean equals(Image2 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image2.class) { + return false; + } + return equals((Image2) obj); + } +} diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java new file mode 100644 index 00000000..a28ff53f --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/Image3.java @@ -0,0 +1,38 @@ +package org.msgpack.tmp; + +import java.io.IOException; +import org.msgpack.MessagePackable; +import org.msgpack.MessageTypeException; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class Image3 implements MessagePackable, MessageUnpackable { + public String uri = ""; + public String title = ""; + public int width = 0; + public int height = 0; + public int size = 0; + + public void messagePack(Packer pk) throws IOException { + // empty + } + + public void messageUnpack(Unpacker pac) throws IOException, + MessageTypeException { + // empty + } + + public boolean equals(Image3 obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image3.class) { + return false; + } + return equals((Image3) obj); + } +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java new file mode 100644 index 00000000..1f27c1b1 --- /dev/null +++ b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java @@ -0,0 +1,134 @@ +package org.msgpack.tmp; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class ImagePackUnpackPerfTest { + + private static int BIG_LOOP = 5; + + private static int SMALL_LOOP = 50000; + + private static DynamicCodeGenerator gen; + + private static void doGc() { + try { + Thread.sleep(50L); + } catch (InterruptedException ie) { + System.err + .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); + } + System.gc(); + try { // longer sleep afterwards (not needed by GC, but may help with + // scheduling) + Thread.sleep(200L); + } catch (InterruptedException ie) { + System.err + .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); + } + } + + public void doIt(int versionID) { + try { + doIt0(versionID); + } catch (Exception e) { + e.printStackTrace(); + } + try { + for (int j = 0; j < BIG_LOOP; ++j) { + long t = System.currentTimeMillis(); + doIt0(versionID); + t = System.currentTimeMillis() - t; + System.out.println("time: " + t); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void doIt0(int versionID) throws Exception { + for (int i = 0; i < SMALL_LOOP; ++i) { + switch (versionID) { + case 0: + doCurrentVersion(); + break; + case 1: + doWithReflection(); + break; + case 2: + doWithDynamicCodeGeneration(); + break; + default: + throw new RuntimeException(); + } + } + } + + public void doCurrentVersion() throws Exception { + Image1 src = new Image1(); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image1 dst = new Image1(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public void doWithReflection() throws Exception { + Image2 src = new Image2(); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image2 dst = new Image2(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public void doWithDynamicCodeGeneration() throws Exception { + Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + src.messagePack(new Packer(out)); + Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + dst.messageUnpack(pac); + } + + public static void main(String[] args) { + ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest(); + + doGc(); + System.out.println("test current version"); + test.doIt(0); + + doGc(); + System.out.println("test with reflection"); + test.doIt(1); + + doGc(); + System.out.println("test with dynamic codegen"); + gen = new DynamicCodeGenerator(); + test.doIt(2); + } +} From ff0e1bbbc070e95be3631459815b0f765c3aab8d Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Thu, 26 Aug 2010 17:53:29 +0900 Subject: [PATCH 02/50] change the part for creating a new constructor within DynamicCodeGenerator --- .../org/msgpack/tmp/DynamicCodeGenerator.java | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java index 9e4336d3..78238b4f 100644 --- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java +++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java @@ -29,6 +29,8 @@ public class DynamicCodeGenerator { private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); + private static final String VOID_TYPE_NAME = "void"; + private static final String PACKER_OBJECT_NAME = "pk"; private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class @@ -37,7 +39,7 @@ public class DynamicCodeGenerator { private static final String UNPACKER_OBJECT_NAME = "pk"; private HashMap> classMap; - + private HashMap objectCacheMap; private ClassPool pool; @@ -50,23 +52,23 @@ public class DynamicCodeGenerator { public Object newEnhancedInstance(Class targetClass) throws Exception { String targetClassName = targetClass.getName(); - //Class enhancedClass = classMap.get(targetClassName); + // Class enhancedClass = classMap.get(targetClassName); Object enhancedObject = objectCacheMap.get(targetClassName); - //if (enhancedClass == null) { + // if (enhancedClass == null) { if (enhancedObject == null) { CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); -// System.out.println("enhanced class name: " -// + enhancedCtClass.getName()); + // System.out.println("enhanced class name: " + // + enhancedCtClass.getName()); addSuperclass(enhancedCtClass, targetClassName); addConstructor(enhancedCtClass); createMessagePackMethod(enhancedCtClass, targetClass); createMessageUnpackMethod(enhancedCtClass, targetClass); Class enhancedClass = loadEnhancedClass(enhancedCtClass); - //classMap.put(targetClassName, enhancedClass); + // classMap.put(targetClassName, enhancedClass); enhancedObject = enhancedClass.newInstance(); objectCacheMap.put(targetClassName, enhancedObject); } - //return newEnhancedInstance0(enhancedClass); + // return newEnhancedInstance0(enhancedClass); return enhancedObject; } @@ -82,24 +84,18 @@ public class DynamicCodeGenerator { } private void addConstructor(CtClass enhancedCtClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append( - // enhancedCtClass.getName()).append("(").append(")") - "Image3_$$_Enhanced").append("(").append(")").append(SPACE) - .append("{ super(); }"); // TODO -// System.out.println("cons: " + sb.toString()); - CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(), - enhancedCtClass); + CtConstructor newCtConstructor = CtNewConstructor + .defaultConstructor(enhancedCtClass); enhancedCtClass.addConstructor(newCtConstructor); } private void createMessagePackMethod(CtClass enhancedCtClass, Class targetClass) throws Exception { StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE) - .append(METHOD_NAME_MESSAGEPACK).append("(").append( - PACKER_CLASS_TYPE_NAME).append(SPACE).append( - PACKER_OBJECT_NAME).append(")").append(SPACE).append( + sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append( + SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append( + PACKER_CLASS_TYPE_NAME).append(SPACE) + .append(PACKER_OBJECT_NAME).append(")").append(SPACE).append( "throws").append(SPACE).append("java.io.IOException") .append(SPACE).append("{"); Field[] fields = targetClass.getFields(); @@ -109,7 +105,7 @@ public class DynamicCodeGenerator { insertCodeOfMessagePack(field, sb); } sb.append("}"); -// System.out.println("messagePack method: " + sb.toString()); + // System.out.println("messagePack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); enhancedCtClass.addMethod(newCtMethod); } @@ -135,7 +131,7 @@ public class DynamicCodeGenerator { sb .append(MODIFIER_PUBLIC) .append(SPACE) - .append("void") + .append(VOID_TYPE_NAME) .append(SPACE) .append(METHOD_NAME_MESSAGEUNPACK) .append("(") @@ -149,13 +145,14 @@ public class DynamicCodeGenerator { .append("org.msgpack.MessageTypeException, java.io.IOException") .append(SPACE).append("{"); Field[] fields = targetClass.getFields(); - sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";"); + sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()") + .append(";"); // TODO for (Field field : fields) { insertCodeOfMessageUnpack(field, sb); } sb.append("}"); -// System.out.println("messageUnpack method: " + sb.toString()); + // System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); enhancedCtClass.addMethod(newCtMethod); } From a9566b31be76ebc8aaf9149c1a09d3720c3f2ec9 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Sun, 12 Sep 2010 22:21:33 +0900 Subject: [PATCH 03/50] add annotation utilities for Java --- .../org/msgpack/tmp/DynamicCodeGenerator.java | 206 -------- .../src/main/java/org/msgpack/tmp/Image1.java | 47 -- .../src/main/java/org/msgpack/tmp/Image2.java | 85 --- .../src/main/java/org/msgpack/tmp/Image3.java | 38 -- .../msgpack/tmp/ImagePackUnpackPerfTest.java | 134 ----- .../util/annotation/MessagePackOptional.java | 12 + .../util/annotation/MessagePackRequired.java | 12 + .../annotation/MessagePackUnpackable.java | 11 + .../util/annotation/PackUnpackUtil.java | 498 ++++++++++++++++++ .../annotation/PackUnpackUtilException.java | 12 + 10 files changed, 545 insertions(+), 510 deletions(-) delete mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java delete mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java delete mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java deleted file mode 100644 index 78238b4f..00000000 --- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java +++ /dev/null @@ -1,206 +0,0 @@ -package org.msgpack.tmp; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; -import java.util.HashMap; - -import javassist.ClassPool; -import javassist.CtClass; -import javassist.CtConstructor; -import javassist.CtMethod; -import javassist.CtNewConstructor; -import javassist.CtNewMethod; - -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class DynamicCodeGenerator { - - private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced"; - - private static final String SPACE = " "; - - private static final String MODIFIER_PUBLIC = "public"; - - private static final String METHOD_NAME_MESSAGEPACK = "messagePack"; - - private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack"; - - private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName(); - - private static final String VOID_TYPE_NAME = "void"; - - private static final String PACKER_OBJECT_NAME = "pk"; - - private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class - .getName(); - - private static final String UNPACKER_OBJECT_NAME = "pk"; - - private HashMap> classMap; - - private HashMap objectCacheMap; - - private ClassPool pool; - - public DynamicCodeGenerator() { - classMap = new HashMap>(); - objectCacheMap = new HashMap(); - pool = ClassPool.getDefault(); - } - - public Object newEnhancedInstance(Class targetClass) throws Exception { - String targetClassName = targetClass.getName(); - // Class enhancedClass = classMap.get(targetClassName); - Object enhancedObject = objectCacheMap.get(targetClassName); - // if (enhancedClass == null) { - if (enhancedObject == null) { - CtClass enhancedCtClass = createEnhancedCtClass(targetClassName); - // System.out.println("enhanced class name: " - // + enhancedCtClass.getName()); - addSuperclass(enhancedCtClass, targetClassName); - addConstructor(enhancedCtClass); - createMessagePackMethod(enhancedCtClass, targetClass); - createMessageUnpackMethod(enhancedCtClass, targetClass); - Class enhancedClass = loadEnhancedClass(enhancedCtClass); - // classMap.put(targetClassName, enhancedClass); - enhancedObject = enhancedClass.newInstance(); - objectCacheMap.put(targetClassName, enhancedObject); - } - // return newEnhancedInstance0(enhancedClass); - return enhancedObject; - } - - private CtClass createEnhancedCtClass(final String targetClassName) - throws Exception { - return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX); - } - - private void addSuperclass(CtClass enhancedCtClass, - final String targetClassName) throws Exception { - CtClass targetCtClass = pool.get(targetClassName); - enhancedCtClass.setSuperclass(targetCtClass); - } - - private void addConstructor(CtClass enhancedCtClass) throws Exception { - CtConstructor newCtConstructor = CtNewConstructor - .defaultConstructor(enhancedCtClass); - enhancedCtClass.addConstructor(newCtConstructor); - } - - private void createMessagePackMethod(CtClass enhancedCtClass, - Class targetClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append( - SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append( - PACKER_CLASS_TYPE_NAME).append(SPACE) - .append(PACKER_OBJECT_NAME).append(")").append(SPACE).append( - "throws").append(SPACE).append("java.io.IOException") - .append(SPACE).append("{"); - Field[] fields = targetClass.getFields(); - sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append( - "(").append(fields.length).append(")").append(";"); - for (Field field : fields) { - insertCodeOfMessagePack(field, sb); - } - sb.append("}"); - // System.out.println("messagePack method: " + sb.toString()); - CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); - enhancedCtClass.addMethod(newCtMethod); - } - - private void insertCodeOfMessagePack(Field field, StringBuilder sb) { - Class type = field.getType(); - if (type.equals(int.class)) { - sb.append(PACKER_OBJECT_NAME).append(".").append("pack") - .append("(").append(field.getName()).append(")") - .append(";"); - } else if (type.equals(String.class)) { - sb.append(PACKER_OBJECT_NAME).append(".").append("pack") - .append("(").append(field.getName()).append(")") - .append(";"); - } else { - throw new UnsupportedOperationException(); - } - } - - private void createMessageUnpackMethod(CtClass enhancedCtClass, - Class targetClass) throws Exception { - StringBuilder sb = new StringBuilder(); - sb - .append(MODIFIER_PUBLIC) - .append(SPACE) - .append(VOID_TYPE_NAME) - .append(SPACE) - .append(METHOD_NAME_MESSAGEUNPACK) - .append("(") - .append(UNPACKER_CLASS_TYPE_NAME) - .append(SPACE) - .append(UNPACKER_OBJECT_NAME) - .append(")") - .append(SPACE) - .append("throws") - .append(SPACE) - .append("org.msgpack.MessageTypeException, java.io.IOException") - .append(SPACE).append("{"); - Field[] fields = targetClass.getFields(); - sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()") - .append(";"); - // TODO - for (Field field : fields) { - insertCodeOfMessageUnpack(field, sb); - } - sb.append("}"); - // System.out.println("messageUnpack method: " + sb.toString()); - CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass); - enhancedCtClass.addMethod(newCtMethod); - } - - private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) { - Class type = field.getType(); - if (type.equals(int.class)) { - sb.append(field.getName()).append(SPACE).append("=").append(SPACE) - .append(PACKER_OBJECT_NAME).append(".").append("unpackInt") - .append("(").append(")").append(";"); - } else if (type.equals(String.class)) { - sb.append(field.getName()).append(SPACE).append("=").append(SPACE) - .append(PACKER_OBJECT_NAME).append(".").append( - "unpackString").append("(").append(")").append(";"); - } else { - throw new UnsupportedOperationException(); - } - } - - private Class loadEnhancedClass(CtClass enhancedCtClass) - throws Exception { - return enhancedCtClass.toClass(null, null); - } - - private Object newEnhancedInstance0(Class enhancedClass) { - try { - return enhancedClass.newInstance(); - } catch (Exception e) { - throw new UnsupportedOperationException(); - } - } - - public static void main(String[] args) throws Exception { - DynamicCodeGenerator gen = new DynamicCodeGenerator(); - Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - -// System.out.println(dst.equals(src)); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java deleted file mode 100644 index f1819ed6..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image1.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.msgpack.tmp; - -import org.msgpack.*; -import java.io.*; - -public class Image1 implements MessagePackable, MessageUnpackable { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - pk.packArray(5); - pk.pack(uri); - pk.pack(title); - pk.pack(width); - pk.pack(height); - pk.pack(size); - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - int length = pac.unpackArray(); - if (length != 5) { - throw new MessageTypeException(); - } - uri = pac.unpackString(); - title = pac.unpackString(); - width = pac.unpackInt(); - height = pac.unpackInt(); - size = pac.unpackInt(); - } - - public boolean equals(Image1 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image1.class) { - return false; - } - return equals((Image1) obj); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java deleted file mode 100644 index 9c357bf3..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image2.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.msgpack.tmp; - -import java.io.IOException; -import java.lang.reflect.Field; - -import org.msgpack.MessageTypeException; -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class Image2 { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - messagePackWithReflection(pk); - } - - public void messagePackWithReflection(Packer pk) throws IOException { - Class cl = this.getClass(); - Field[] fields = cl.getFields(); - pk.packArray(fields.length); - for (Field field : fields) { - try { - Object obj = field.get(this); - pk.pack(obj); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - } - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - messageUnpackWithReflection(pac); - } - - public void messageUnpackWithReflection(Unpacker pac) throws IOException, - MessageTypeException { - Class cl = this.getClass(); - Field[] fields = cl.getFields(); - int length = pac.unpackArray(); - if (length != fields.length) { - throw new MessageTypeException(); - } - for (Field field : fields) { - try { - field.set(this, unpack(pac, field.getType())); - } catch (IOException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - } - } - - public static Object unpack(Unpacker pk, Class cl) throws IOException { - if (cl == int.class) { - return pk.unpackInt(); - } else if (cl == String.class) { - return pk.unpackString(); - } else { - throw new UnsupportedOperationException(); - } - } - - public boolean equals(Image2 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image2.class) { - return false; - } - return equals((Image2) obj); - } -} diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java deleted file mode 100644 index a28ff53f..00000000 --- a/java/src/main/java/org/msgpack/tmp/Image3.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.msgpack.tmp; - -import java.io.IOException; -import org.msgpack.MessagePackable; -import org.msgpack.MessageTypeException; -import org.msgpack.MessageUnpackable; -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class Image3 implements MessagePackable, MessageUnpackable { - public String uri = ""; - public String title = ""; - public int width = 0; - public int height = 0; - public int size = 0; - - public void messagePack(Packer pk) throws IOException { - // empty - } - - public void messageUnpack(Unpacker pac) throws IOException, - MessageTypeException { - // empty - } - - public boolean equals(Image3 obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image3.class) { - return false; - } - return equals((Image3) obj); - } -} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java deleted file mode 100644 index 1f27c1b1..00000000 --- a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.msgpack.tmp; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; - -import org.msgpack.Packer; -import org.msgpack.Unpacker; - -public class ImagePackUnpackPerfTest { - - private static int BIG_LOOP = 5; - - private static int SMALL_LOOP = 50000; - - private static DynamicCodeGenerator gen; - - private static void doGc() { - try { - Thread.sleep(50L); - } catch (InterruptedException ie) { - System.err - .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); - } - System.gc(); - try { // longer sleep afterwards (not needed by GC, but may help with - // scheduling) - Thread.sleep(200L); - } catch (InterruptedException ie) { - System.err - .println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()"); - } - } - - public void doIt(int versionID) { - try { - doIt0(versionID); - } catch (Exception e) { - e.printStackTrace(); - } - try { - for (int j = 0; j < BIG_LOOP; ++j) { - long t = System.currentTimeMillis(); - doIt0(versionID); - t = System.currentTimeMillis() - t; - System.out.println("time: " + t); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void doIt0(int versionID) throws Exception { - for (int i = 0; i < SMALL_LOOP; ++i) { - switch (versionID) { - case 0: - doCurrentVersion(); - break; - case 1: - doWithReflection(); - break; - case 2: - doWithDynamicCodeGeneration(); - break; - default: - throw new RuntimeException(); - } - } - } - - public void doCurrentVersion() throws Exception { - Image1 src = new Image1(); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image1 dst = new Image1(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public void doWithReflection() throws Exception { - Image2 src = new Image2(); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image2 dst = new Image2(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public void doWithDynamicCodeGeneration() throws Exception { - Image3 src = (Image3) gen.newEnhancedInstance(Image3.class); - src.title = "msgpack"; - src.uri = "http://msgpack.org/"; - src.width = 2560; - src.height = 1600; - src.size = 4096000; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - src.messagePack(new Packer(out)); - Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker pac = new Unpacker(in); - dst.messageUnpack(pac); - } - - public static void main(String[] args) { - ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest(); - - doGc(); - System.out.println("test current version"); - test.doIt(0); - - doGc(); - System.out.println("test with reflection"); - test.doIt(1); - - doGc(); - System.out.println("test with dynamic codegen"); - gen = new DynamicCodeGenerator(); - test.doIt(2); - } -} diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java new file mode 100644 index 00000000..a565292d --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackOptional { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java new file mode 100644 index 00000000..03e50cf3 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackRequired { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java new file mode 100644 index 00000000..473b5412 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java @@ -0,0 +1,11 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackUnpackable { +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java new file mode 100644 index 00000000..37b9e0d5 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -0,0 +1,498 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtField; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; +import javassist.NotFoundException; + +import org.msgpack.MessageConvertable; +import org.msgpack.MessagePackable; +import org.msgpack.MessageTypeException; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class PackUnpackUtil { + static class Constants { + static final String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced"; + + static final String KEYWORD_MODIFIER_PUBLIC = "public"; + + static final String KEYWORD_THROWS = "throws"; + + static final String TYPE_NAME_VOID = void.class.getName(); + + static final String TYPE_NAME_BOOLEAN = boolean.class.getName(); + + static final String TYPE_NAME_BYTE = byte.class.getName(); + + static final String TYPE_NAME_DOUBLE = double.class.getName(); + + static final String TYPE_NAME_FLOAT = float.class.getName(); + + static final String TYPE_NAME_INT = int.class.getName(); + + static final String TYPE_NAME_LONG = long.class.getName(); + + static final String TYPE_NAME_SHORT = short.class.getName(); + + static final String TYPE_NAME_OBJECT = Object.class.getName(); + + static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName(); + + static final String TYPE_NAME_STRING = String.class.getName(); + + static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName(); + + static final String TYPE_NAME_BYTE2 = Byte.class.getName(); + + static final String TYPE_NAME_DOUBLE2 = Double.class.getName(); + + static final String TYPE_NAME_FLOAT2 = Float.class.getName(); + + static final String TYPE_NAME_INT2 = Integer.class.getName(); + + static final String TYPE_NAME_LONG2 = Long.class.getName(); + + static final String TYPE_NAME_SHORT2 = Short.class.getName(); + + static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + + static final String TYPE_NAME_LIST = List.class.getName(); + + static final String TYPE_NAME_MAP = Map.class.getName(); + + static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); + + static final String TYPE_NAME_PACKER = Packer.class.getName(); + + static final String TYPE_NAME_UNPACKER = Unpacker.class.getName(); + + static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class + .getName(); + + static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class + .getName(); + + static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class + .getName(); + + static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class + .getName(); + + static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class + .getName(); + + static final String CHAR_NAME_SPACE = " "; + + static final String CHAR_NAME_COMMA = ","; + + static final String CHAR_NAME_EQUAL = "="; + + static final String CHAR_NAME_RIGHT_PARENTHESIS = ")"; + + static final String CHAR_NAME_LEFT_PARENTHESIS = "("; + + static final String CHAR_NAME_RIGHT_CURLY_BRACHET = "}"; + + static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{"; + + static final String CHAR_NAME_DOT = "."; + + static final String CHAR_NAME_SEMICOLON = ";"; + + static final String VARIABLE_NAME_PK = "pk"; + + static final String VARIABLE_NAME_OBJ = "obj"; + + static final String METHOD_NAME_MSGPACK = "messagePack"; + + static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; + + static final String METHOD_NAME_MSGCONVERT = "messageConvert"; + + static final String METHOD_NAME_PACK = "pack"; + + static final String METHOD_NAME_PACKARRAY = "packArray"; + + static final String METHOD_NAME_UNPACK = "unpack"; + + static final String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean"; + + static final String METHOD_NAME_UNPACKBYTE = "unpackByte"; + + static final String METHOD_NAME_UNPACKDOUBLE = "unpackDouble"; + + static final String METHOD_NAME_UNPACKFLOAT = "unpackFloat"; + + static final String METHOD_NAME_UNPACKINT = "unpackInt"; + + static final String METHOD_NAME_UNPACKLONG = "unpackLong"; + + static final String METHOD_NAME_UNPACKSHORT = "unpackShort"; + + static final String METHOD_NAME_UNPACKSTRING = "unpackString"; + + static final String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger"; + + static final String METHOD_NAME_UNPACKOBJECT = "unpackObject"; + + static final String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray"; + + static final String METHOD_NAME_UNPACKARRAY = "unpackArray"; + + static final String METHOD_NAME_UNPACKMAP = "unpackMap"; + } + + public static class Enhancer { + + private ConcurrentHashMap> classCache; + + private ClassPool pool; + + protected Enhancer() { + classCache = new ConcurrentHashMap>(); + pool = ClassPool.getDefault(); + } + + protected Class getCache(String origName) { + return classCache.get(origName); + } + + protected void setCache(String origName, Class enhClass) { + classCache.putIfAbsent(origName, enhClass); + } + + protected Class generate(Class origClass) + throws NotFoundException, CannotCompileException { + String origName = origClass.getName(); + String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; + CtClass origCtClass = pool.get(origName); + checkPackUnpackAnnotation(origCtClass); + CtClass enhCtClass = pool.makeClass(enhName); + setSuperclass(enhCtClass, origCtClass); + setInterfaces(enhCtClass); + addConstructor(enhCtClass); + addMessagePackMethod(enhCtClass, origCtClass); + addMessageUnpackMethod(enhCtClass, origCtClass); + addMessageConvertMethod(enhCtClass, origCtClass); + return createClass(enhCtClass); + } + + private void checkPackUnpackAnnotation(CtClass origCtClass) { + try { + Object[] objs = origCtClass.getAnnotations(); + for (Object obj : objs) { + if (obj instanceof MessagePackUnpackable) { + return; + } + } + throw new PackUnpackUtilException( + "Not annotated with this class: " + + origCtClass.getName()); + } catch (ClassNotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + } + + private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) + throws CannotCompileException { + enhCtClass.setSuperclass(origCtClass); + } + + private void setInterfaces(CtClass enhCtClass) throws NotFoundException { + CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE); + enhCtClass.addInterface(pacCtClass); + CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE); + enhCtClass.addInterface(unpacCtClass); + CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE); + enhCtClass.addInterface(convCtClass); + } + + private void addConstructor(CtClass enhCtClass) + throws CannotCompileException { + CtConstructor newCtCons = CtNewConstructor + .defaultConstructor(enhCtClass); + enhCtClass.addConstructor(newCtCons); + } + + private void addMessagePackMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException, + NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGPACK).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_PACKER).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_IOEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + CtField[] fields = origCtClass.getDeclaredFields(); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append( + Constants.METHOD_NAME_PACKARRAY).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length) + .append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + for (CtField field : fields) { + insertCodeOfMessagePack(field, sb); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messagePack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessagePack(CtField field, StringBuilder sb) + throws NotFoundException { + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK) + .append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + field.getName()).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + } + + private void addMessageUnpackMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException, + NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGUNPACK).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_UNPACKER).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( + Constants.CHAR_NAME_COMMA).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_IOEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + CtField[] fields = origCtClass.getFields(); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT).append( + Constants.METHOD_NAME_UNPACKARRAY).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + for (CtField field : fields) { + insertCodeOfMessageUnpack(field, sb); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messageUnpack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) + throws NotFoundException { + CtClass type = field.getType(); + sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) + .append(Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT); + if (type.equals(CtClass.booleanType)) { // boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(CtClass.byteType)) { // byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(CtClass.doubleType)) { // double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(CtClass.floatType)) { // float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(CtClass.intType)) { // int + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(CtClass.longType)) { // long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(CtClass.shortType)) { // short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else if (type + .equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger + sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER); + } else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String + sb.append(Constants.METHOD_NAME_UNPACKSTRING); + } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[] + sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY); + } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map + sb.append(Constants.METHOD_NAME_UNPACKMAP); + } else if (type.subtypeOf(pool + .get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable + sb.append(Constants.METHOD_NAME_UNPACKOBJECT); + } else { + throw new NotFoundException("unknown type: " + + type.getName()); + } + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + } + + private void addMessageConvertMethod(CtClass enhCtClass, + CtClass origCtClass) throws CannotCompileException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGCONVERT).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_OBJECT).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_OBJ).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + // TODO + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + System.out.println("messageConvert method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private Class createClass(CtClass enhCtClass) + throws CannotCompileException { + return enhCtClass.toClass(null, null); + } + } + + private static Enhancer enhancer; + + public static Class getEnhancedClass(Class origClass) { + if (enhancer == null) { + enhancer = new Enhancer(); + } + + String origName = origClass.getName(); + Class enhClass = enhancer.getCache(origName); + if (enhClass == null) { + // generate a class object related to the original class + try { + enhClass = enhancer.generate(origClass); + } catch (NotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (CannotCompileException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + // set the generated class to the cache + enhancer.setCache(origName, enhClass); + } + return enhClass; + } + + public static Object newEnhancedInstance(Class origClass) { + try { + Class enhClass = getEnhancedClass(origClass); + // create a new object of the generated class + return enhClass.newInstance(); + } catch (InstantiationException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + } + + public static void main(final String[] args) throws Exception { + @MessagePackUnpackable + class Image { + public String uri = ""; + + public String title = ""; + + public int width = 0; + + public int height = 0; + + public int size = 0; + + public boolean equals(Image obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image.class) { + return false; + } + return equals((Image) obj); + } + } + + PackUnpackUtil.getEnhancedClass(Image.class); + Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + Image dst = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + System.out.println(src.equals(dst)); + } +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java new file mode 100644 index 00000000..df3af058 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +public class PackUnpackUtilException extends RuntimeException { + + public PackUnpackUtilException(String reason) { + super(reason); + } + + public PackUnpackUtilException(String reason, Throwable t) { + super(reason, t); + } +} From 599b200ca5dda7957b47fb5156199a75a50a57ae Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Sun, 12 Sep 2010 23:04:23 +0900 Subject: [PATCH 04/50] change the version of javassist 3.12.1.GA --- java/pom.xml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 6591a727..70da6a6f 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,11 +29,11 @@ 4.8.1 test - - jboss + javassist javassist - 3.7.ga + 3.12.1.GA + compile @@ -107,11 +107,12 @@ MessagePack Maven2 Repository http://msgpack.org/maven2 - - repo2.maven.org - repo2.maven.org Maven2 Repository - http://repo2.maven.org/maven2 + repository.jboss.org + https://repository.jboss.org/nexus/content/groups/public/ + + false + From 95b820305ae55823ee5b64476acfa5c2a7f44beb Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 13:34:11 +0900 Subject: [PATCH 05/50] add a new test program for annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 116 +++++++- .../annotation/TestMessagePackUnpackable.java | 280 ++++++++++++++++++ 2 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 37b9e0d5..5efef792 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -3,6 +3,7 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.reflect.Modifier; import java.math.BigInteger; import java.util.List; import java.util.Map; @@ -124,6 +125,8 @@ public class PackUnpackUtil { static final String VARIABLE_NAME_OBJ = "obj"; + static final String METHOD_NAME_VALUEOF = "valueOf"; + static final String METHOD_NAME_MSGPACK = "messagePack"; static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; @@ -187,7 +190,8 @@ public class PackUnpackUtil { String origName = origClass.getName(); String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; CtClass origCtClass = pool.get(origName); - checkPackUnpackAnnotation(origCtClass); + checkClassValidation(origCtClass); + checkDefaultConstructorValidation(origCtClass); CtClass enhCtClass = pool.makeClass(enhName); setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); @@ -198,6 +202,28 @@ public class PackUnpackUtil { return createClass(enhCtClass); } + private void checkClassValidation(CtClass origCtClass) { + // not public, abstract, final + int mod = origCtClass.getModifiers(); + if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod) + || Modifier.isFinal(mod)) { + throwClassValidationException(origCtClass); + } + // interface, enum + if (origCtClass.isInterface() || origCtClass.isEnum()) { + throwClassValidationException(origCtClass); + } + // annotation + checkPackUnpackAnnotation(origCtClass); + } + + private static void throwClassValidationException(CtClass origCtClass) { + throw new PackUnpackUtilException( + "it must be a public class and have @" + + MessagePackUnpackable.class.getName() + ": " + + origCtClass.getName()); + } + private void checkPackUnpackAnnotation(CtClass origCtClass) { try { Object[] objs = origCtClass.getAnnotations(); @@ -206,14 +232,32 @@ public class PackUnpackUtil { return; } } - throw new PackUnpackUtilException( - "Not annotated with this class: " - + origCtClass.getName()); + throwClassValidationException(origCtClass); } catch (ClassNotFoundException e) { throw new PackUnpackUtilException(e.getMessage(), e); } } + private void checkDefaultConstructorValidation(CtClass origCtClass) { + CtConstructor cons = null; + try { + cons = origCtClass.getDeclaredConstructor(new CtClass[0]); + } catch (NotFoundException e) { + throwConstructoValidationException(origCtClass); + } + int mod = cons.getModifiers(); + if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) { + throwConstructoValidationException(origCtClass); + } + } + + private static void throwConstructoValidationException( + CtClass origCtClass) { + throw new PackUnpackUtilException( + "it must have a public zero-argument constructor: " + + origCtClass.getName()); + } + private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) throws CannotCompileException { enhCtClass.setSuperclass(origCtClass); @@ -328,9 +372,43 @@ public class PackUnpackUtil { CtClass type = field.getType(); sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT); + Constants.CHAR_NAME_SPACE); + insertValueOfMethodAndLeftParenthesis(sb, type); + sb.append(Constants.VARIABLE_NAME_PK).append( + Constants.CHAR_NAME_DOT); + insertUnpackMethod(sb, type); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS); + insertValueOfMethodAndRightParenthesis(sb, type); + sb.append(Constants.CHAR_NAME_SEMICOLON).append( + Constants.CHAR_NAME_SPACE); + + } + + private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + ) { + sb.append(type.getName()).append(Constants.CHAR_NAME_DOT) + .append(Constants.METHOD_NAME_VALUEOF).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS); + } else { + return; + } + } + } + + private void insertUnpackMethod(StringBuilder sb, CtClass type) + throws NotFoundException { if (type.equals(CtClass.booleanType)) { // boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); } else if (type.equals(CtClass.byteType)) { // byte @@ -379,10 +457,26 @@ public class PackUnpackUtil { + type.getName()); } } - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); + } + + private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + ) { + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + } else { + return; + } + } } private void addMessageConvertMethod(CtClass enhCtClass, diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java new file mode 100644 index 00000000..1ec7d64e --- /dev/null +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -0,0 +1,280 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class TestMessagePackUnpackable extends TestCase { + + @Test + public void testGeneralPrimitiveTypeFieldsClass() throws Exception { + GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + src.f0 = (byte) 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f6 = false; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + } + + @MessagePackUnpackable + public static class GeneralPrimitiveTypeFieldsClass { + public byte f0; + public short f1; + public int f2; + public long f3; + public float f4; + public double f5; + public boolean f6; + + public GeneralPrimitiveTypeFieldsClass() { + } + } + + @Test + public void testGeneralReferenceTypeFieldsClass() throws Exception { + GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + src.f0 = 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = (long) 3; + src.f4 = (float) 4; + src.f5 = (double) 5; + src.f6 = false; + src.f7 = new BigInteger("7"); + src.f8 = "8"; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + assertEquals(src.f7, dst.f7); + assertEquals(src.f8, dst.f8); + } + + @MessagePackUnpackable + public static class GeneralReferenceTypeFieldsClass { + public Byte f0; + public Short f1; + public Integer f2; + public Long f3; + public Float f4; + public Double f5; + public Boolean f6; + public BigInteger f7; + public String f8; + + public GeneralReferenceTypeFieldsClass() { + } + } + + @Test + public void testPublicDefaultConstructorClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public static class NoDefaultConstructorClass { + public NoDefaultConstructorClass(int i) { + } + } + + @MessagePackUnpackable + public static class PrivateDefaultConstructorClass { + private PrivateDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class ProtectedDefaultConstructorClass { + protected ProtectedDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class PackageDefaultConstructorClass { + PackageDefaultConstructorClass() { + } + } + + @Test + public void testPublicModifierClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + private static class PrivateModifierClass { + } + + @MessagePackUnpackable + protected static class ProtectedModifierClass { + protected ProtectedModifierClass() { + } + } + + @MessagePackUnpackable + static class PackageModifierClass { + } + + @Test + public void testFinalAndAbstractModifierClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public final static class FinalModifierClass { + } + + @MessagePackUnpackable + public abstract static class AbstractModifierClass { + } + + @Test + public void testInterfaceAndEnum() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(SampleInterface.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(SampleEnum.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public interface SampleInterface { + } + + @MessagePackUnpackable + public enum SampleEnum { + } + + @Test + public void testFinalFieldClass() throws Exception { + + } + + @Test + public void testPrivateFieldClass() throws Exception { + + } + + @Test + public void testProtectedFieldClass() throws Exception { + + } + + @Test + public void testNonModifierFieldClass() throws Exception { + + } + + @Test + public void testNestedAnnotatedFieldClass() throws Exception { + + } + + @Test + public void testSuperClass() throws Exception { + + } +} From c7f8b94ccdde64512aced14930ebdc0e14825e66 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 22:26:04 +0900 Subject: [PATCH 06/50] fixed several bugs of a verify error within annotation-utilities --- .../util/annotation/PackUnpackUtil.java | 170 ++++++++++++++---- .../annotation/TestMessagePackUnpackable.java | 160 ++++++++++++++--- 2 files changed, 270 insertions(+), 60 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index 5efef792..a58b4bd5 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -5,6 +5,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Modifier; import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -196,9 +197,10 @@ public class PackUnpackUtil { setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); addConstructor(enhCtClass); - addMessagePackMethod(enhCtClass, origCtClass); - addMessageUnpackMethod(enhCtClass, origCtClass); - addMessageConvertMethod(enhCtClass, origCtClass); + CtField[] fields = getDeclaredFields(origCtClass); + addMessagePackMethod(enhCtClass, origCtClass, fields); + addMessageUnpackMethod(enhCtClass, origCtClass, fields); + addMessageConvertMethod(enhCtClass, origCtClass, fields); return createClass(enhCtClass); } @@ -279,9 +281,54 @@ public class PackUnpackUtil { enhCtClass.addConstructor(newCtCons); } + private CtField[] getDeclaredFields(CtClass origCtClass) { + ArrayList allFields = new ArrayList(); + try { + CtClass nextCtClass = origCtClass; + while (!nextCtClass + .equals(pool.get(Constants.TYPE_NAME_OBJECT))) { + CtField[] fields = nextCtClass.getDeclaredFields(); + for (CtField field : fields) { + try { + checkFieldValidation(field, allFields); + allFields.add(field); + } catch (PackUnpackUtilException e) { // ignore + } + } + nextCtClass = nextCtClass.getSuperclass(); + } + + } catch (NotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + return allFields.toArray(new CtField[0]); + } + + private void checkFieldValidation(CtField field, + ArrayList allFields) { + // check modifiers (public or protected) + int mod = field.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isStatic(mod) || Modifier.isFinal(mod) + || Modifier.isTransient(mod)) { + throwFieldValidationException(field); + } + // check same name + for (CtField f : allFields) { + if (f.getName().equals(field.getName())) { + throwFieldValidationException(field); + } + } + } + + private static void throwFieldValidationException(CtField field) { + throw new PackUnpackUtilException("it must be a public field: " + + field.getName()); + } + private void addMessagePackMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException, - NotFoundException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -299,7 +346,6 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE).append( Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( Constants.CHAR_NAME_SPACE); - CtField[] fields = origCtClass.getDeclaredFields(); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT).append( Constants.METHOD_NAME_PACKARRAY).append( @@ -328,8 +374,8 @@ public class PackUnpackUtil { } private void addMessageUnpackMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException, - NotFoundException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -350,7 +396,6 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE).append( Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( Constants.CHAR_NAME_SPACE); - CtField[] fields = origCtClass.getFields(); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT).append( Constants.METHOD_NAME_UNPACKARRAY).append( @@ -370,21 +415,53 @@ public class PackUnpackUtil { private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) throws NotFoundException { CtClass type = field.getType(); - sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) - .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); + insertRightVariable(sb, field, type); insertValueOfMethodAndLeftParenthesis(sb, type); sb.append(Constants.VARIABLE_NAME_PK).append( Constants.CHAR_NAME_DOT); insertUnpackMethod(sb, type); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertUnpackMethodArgumenet(sb, field, type); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); insertValueOfMethodAndRightParenthesis(sb, type); sb.append(Constants.CHAR_NAME_SEMICOLON).append( Constants.CHAR_NAME_SPACE); } + private void insertRightVariable(StringBuilder sb, CtField field, + CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) + .append(Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE); + } else { // reference type + if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean + || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte + || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double + || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float + || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer + || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long + || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short + || type + .equals(pool + .get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger + || type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String + || type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[] + || type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List + || type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map + ) { + sb.append(field.getName()) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_EQUAL).append( + Constants.CHAR_NAME_SPACE); + } else { // MessageUnpackable + return; + } + } + + } + private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, CtClass type) throws NotFoundException { if (type.isPrimitive()) { // primitive type @@ -424,6 +501,7 @@ public class PackUnpackUtil { } else if (type.equals(CtClass.shortType)) { // short sb.append(Constants.METHOD_NAME_UNPACKSHORT); } else { // reference type + Class c = null; if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte @@ -450,8 +528,9 @@ public class PackUnpackUtil { } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map sb.append(Constants.METHOD_NAME_UNPACKMAP); } else if (type.subtypeOf(pool - .get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable - sb.append(Constants.METHOD_NAME_UNPACKOBJECT); + .get(Constants.TYPE_NAME_MSGUNPACKABLE)) + || ((c = getCache(type.getName())) != null)) { // MessageUnpackable + sb.append(Constants.METHOD_NAME_UNPACK); } else { throw new NotFoundException("unknown type: " + type.getName()); @@ -459,6 +538,22 @@ public class PackUnpackUtil { } } + private void insertUnpackMethodArgumenet(StringBuilder sb, + CtField field, CtClass type) throws NotFoundException { + if (type.isPrimitive()) { // primitive type + return; + } else { // reference type + Class c = null; + if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE)) + || ((c = getCache(type.getName())) != null)) { + sb.append("(org.msgpack.MessageUnpackable)"); + sb.append(field.getName()); + } else { + return; + } + } + } + private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, CtClass type) throws NotFoundException { if (type.isPrimitive()) { // primitive type @@ -480,7 +575,8 @@ public class PackUnpackUtil { } private void addMessageConvertMethod(CtClass enhCtClass, - CtClass origCtClass) throws CannotCompileException { + CtClass origCtClass, CtField[] fields) + throws CannotCompileException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) @@ -547,33 +643,33 @@ public class PackUnpackUtil { } } - public static void main(final String[] args) throws Exception { - @MessagePackUnpackable - class Image { - public String uri = ""; + @MessagePackUnpackable + public static class Image { + public String uri = ""; - public String title = ""; + public String title = ""; - public int width = 0; + public int width = 0; - public int height = 0; + public int height = 0; - public int size = 0; + public int size = 0; - public boolean equals(Image obj) { - return uri.equals(obj.uri) && title.equals(obj.title) - && width == obj.width && height == obj.height - && size == obj.size; - } - - public boolean equals(Object obj) { - if (obj.getClass() != Image.class) { - return false; - } - return equals((Image) obj); - } + public boolean equals(Image obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; } + public boolean equals(Object obj) { + if (obj.getClass() != Image.class) { + return false; + } + return equals((Image) obj); + } + } + + public static void main(final String[] args) throws Exception { PackUnpackUtil.getEnhancedClass(Image.class); Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); src.title = "msgpack"; diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 1ec7d64e..4f6e74ef 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -3,6 +3,8 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.math.BigInteger; +import java.util.List; +import java.util.Map; import junit.framework.TestCase; @@ -14,7 +16,7 @@ import org.msgpack.Unpacker; public class TestMessagePackUnpackable extends TestCase { @Test - public void testGeneralPrimitiveTypeFieldsClass() throws Exception { + public void testGeneralPrimitiveTypeFields() throws Exception { GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); src.f0 = (byte) 0; @@ -101,8 +103,21 @@ public class TestMessagePackUnpackable extends TestCase { } } + public void testListAndMap() throws Exception { + // TODO + } + + @MessagePackUnpackable + public static class ListAndMapClass { + public List f0; + public Map f1; + + public ListAndMapClass() { + } + } + @Test - public void testPublicDefaultConstructorClass() throws Exception { + public void testDefaultConstructorModifiers() throws Exception { try { PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class); fail(); @@ -111,21 +126,24 @@ public class TestMessagePackUnpackable extends TestCase { } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(PrivateDefaultConstructorClass.class); fail(); } catch (PackUnpackUtilException e) { assertTrue(true); } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(ProtectedDefaultConstructorClass.class); assertTrue(true); } catch (PackUnpackUtilException e) { fail(); } assertTrue(true); try { - PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class); + PackUnpackUtil + .newEnhancedInstance(PackageDefaultConstructorClass.class); fail(); } catch (PackUnpackUtilException e) { assertTrue(true); @@ -158,7 +176,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testPublicModifierClass() throws Exception { + public void testClassModifiers() throws Exception { try { PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class); fail(); @@ -197,7 +215,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testFinalAndAbstractModifierClass() throws Exception { + public void testFinalClassAndAbstractClass() throws Exception { try { PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class); fail(); @@ -223,7 +241,7 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testInterfaceAndEnum() throws Exception { + public void testInterfaceAndEnumType() throws Exception { try { PackUnpackUtil.newEnhancedInstance(SampleInterface.class); fail(); @@ -249,32 +267,128 @@ public class TestMessagePackUnpackable extends TestCase { } @Test - public void testFinalFieldClass() throws Exception { - + public void testFieldModifiers() throws Exception { + FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); } - @Test - public void testPrivateFieldClass() throws Exception { - - } - - @Test - public void testProtectedFieldClass() throws Exception { - - } - - @Test - public void testNonModifierFieldClass() throws Exception { + @MessagePackUnpackable + public static class FieldModifiersClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + public FieldModifiersClass() { + } } @Test public void testNestedAnnotatedFieldClass() throws Exception { + NestedClass src2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass src = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + src.f0 = 0; + src2.f2 = 2; + src.f1 = src2; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + NestedClass dst2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass dst = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + dst.f1 = dst2; + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src2.f2 == dst.f1.f2); + } + @MessagePackUnpackable + public static class BaseClass { + public int f0; + public NestedClass f1; + + public BaseClass() { + } + } + + @MessagePackUnpackable + public static class NestedClass { + public int f2; + + public NestedClass() { + } } @Test - public void testSuperClass() throws Exception { + public void testExtendedClass() throws Exception { + SampleSubClass src = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f8 = 8; + src.f9 = 9; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleSubClass dst = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + assertTrue(src.f5 == dst.f5); + assertTrue(src.f6 == dst.f6); + assertTrue(src.f8 == dst.f8); + assertTrue(src.f9 != dst.f9); + } + @MessagePackUnpackable + public static class SampleSubClass extends SampleSuperClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + + public SampleSubClass() { + } + } + + public static class SampleSuperClass { + public int f5; + public final int f6 = 2; + private int f7; + protected int f8; + int f9; + + public SampleSuperClass() { + } } } From 56ece4db0f03f086c2d4608472de4b20e6a83a19 Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Tue, 14 Sep 2010 23:23:09 +0900 Subject: [PATCH 07/50] fixed a bug the program for packing and unpacking byte[] in annotation-utility --- .../java/org/msgpack/util/annotation/PackUnpackUtil.java | 3 ++- .../msgpack/util/annotation/TestMessagePackUnpackable.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index a58b4bd5..f92c51a0 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -71,7 +71,8 @@ public class PackUnpackUtil { static final String TYPE_NAME_SHORT2 = Short.class.getName(); - static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + //static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); + static final String TYPE_NAME_BYTEARRAY = "byte[]"; static final String TYPE_NAME_LIST = List.class.getName(); diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 4f6e74ef..1cde65f2 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -69,6 +69,7 @@ public class TestMessagePackUnpackable extends TestCase { src.f6 = false; src.f7 = new BigInteger("7"); src.f8 = "8"; + src.f9 = new byte[] { 0x01, 0x02 }; ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(src); GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil @@ -85,6 +86,8 @@ public class TestMessagePackUnpackable extends TestCase { assertEquals(src.f6, dst.f6); assertEquals(src.f7, dst.f7); assertEquals(src.f8, dst.f8); + assertEquals(src.f9[0], dst.f9[0]); + assertEquals(src.f9[1], dst.f9[1]); } @MessagePackUnpackable @@ -98,13 +101,13 @@ public class TestMessagePackUnpackable extends TestCase { public Boolean f6; public BigInteger f7; public String f8; + public byte[] f9; public GeneralReferenceTypeFieldsClass() { } } public void testListAndMap() throws Exception { - // TODO } @MessagePackUnpackable From 9eeb702ca536f0e97e53202b177d9db5ac9e322f Mon Sep 17 00:00:00 2001 From: Muga Nishizawa Date: Wed, 15 Sep 2010 22:28:46 +0900 Subject: [PATCH 08/50] change an annotation-utility in Java. it allows users to pack and unpack a List object. --- .../annotation/MessagePackUnpackable.java | 2 +- .../util/annotation/PackUnpackUtil.java | 705 ++++++++++-------- .../annotation/TestMessagePackUnpackable.java | 105 ++- 3 files changed, 499 insertions(+), 313 deletions(-) diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java index 473b5412..77f6e6de 100644 --- a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java @@ -6,6 +6,6 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Target(ElementType.TYPE) -@Retention(RetentionPolicy.CLASS) +@Retention(RetentionPolicy.RUNTIME) public @interface MessagePackUnpackable { } diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java index f92c51a0..52384448 100644 --- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -3,9 +3,14 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; import java.math.BigInteger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -14,7 +19,6 @@ import javassist.CannotCompileException; import javassist.ClassPool; import javassist.CtClass; import javassist.CtConstructor; -import javassist.CtField; import javassist.CtMethod; import javassist.CtNewConstructor; import javassist.CtNewMethod; @@ -33,51 +37,16 @@ public class PackUnpackUtil { static final String KEYWORD_MODIFIER_PUBLIC = "public"; + static final String KEYWORD_FOR = "for"; + static final String KEYWORD_THROWS = "throws"; + static final String KEYWORD_NEW = "new"; + static final String TYPE_NAME_VOID = void.class.getName(); - static final String TYPE_NAME_BOOLEAN = boolean.class.getName(); - - static final String TYPE_NAME_BYTE = byte.class.getName(); - - static final String TYPE_NAME_DOUBLE = double.class.getName(); - - static final String TYPE_NAME_FLOAT = float.class.getName(); - - static final String TYPE_NAME_INT = int.class.getName(); - - static final String TYPE_NAME_LONG = long.class.getName(); - - static final String TYPE_NAME_SHORT = short.class.getName(); - static final String TYPE_NAME_OBJECT = Object.class.getName(); - static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName(); - - static final String TYPE_NAME_STRING = String.class.getName(); - - static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName(); - - static final String TYPE_NAME_BYTE2 = Byte.class.getName(); - - static final String TYPE_NAME_DOUBLE2 = Double.class.getName(); - - static final String TYPE_NAME_FLOAT2 = Float.class.getName(); - - static final String TYPE_NAME_INT2 = Integer.class.getName(); - - static final String TYPE_NAME_LONG2 = Long.class.getName(); - - static final String TYPE_NAME_SHORT2 = Short.class.getName(); - - //static final String TYPE_NAME_BYTEARRAY = byte[].class.getName(); - static final String TYPE_NAME_BYTEARRAY = "byte[]"; - - static final String TYPE_NAME_LIST = List.class.getName(); - - static final String TYPE_NAME_MAP = Map.class.getName(); - static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); static final String TYPE_NAME_PACKER = Packer.class.getName(); @@ -111,6 +80,10 @@ public class PackUnpackUtil { static final String CHAR_NAME_EQUAL = "="; + static final String CHAR_NAME_PLUS = "+"; + + static final String CHAR_NAME_LESSTHAN = "<"; + static final String CHAR_NAME_RIGHT_PARENTHESIS = ")"; static final String CHAR_NAME_LEFT_PARENTHESIS = "("; @@ -127,8 +100,16 @@ public class PackUnpackUtil { static final String VARIABLE_NAME_OBJ = "obj"; + static final String VARIABLE_NAME_SIZE = "len"; + + static final String VARIABLE_NAME_I = "i"; + static final String METHOD_NAME_VALUEOF = "valueOf"; + static final String METHOD_NAME_ADD = "add"; + + static final String METHOD_NAME_PUT = "put"; + static final String METHOD_NAME_MSGPACK = "messagePack"; static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; @@ -192,73 +173,68 @@ public class PackUnpackUtil { String origName = origClass.getName(); String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; CtClass origCtClass = pool.get(origName); - checkClassValidation(origCtClass); - checkDefaultConstructorValidation(origCtClass); + checkClassValidation(origClass); + checkDefaultConstructorValidation(origClass); CtClass enhCtClass = pool.makeClass(enhName); setSuperclass(enhCtClass, origCtClass); setInterfaces(enhCtClass); addConstructor(enhCtClass); - CtField[] fields = getDeclaredFields(origCtClass); + Field[] fields = getDeclaredFields(origClass); addMessagePackMethod(enhCtClass, origCtClass, fields); addMessageUnpackMethod(enhCtClass, origCtClass, fields); addMessageConvertMethod(enhCtClass, origCtClass, fields); return createClass(enhCtClass); } - private void checkClassValidation(CtClass origCtClass) { + private void checkClassValidation(Class origClass) { // not public, abstract, final - int mod = origCtClass.getModifiers(); - if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod) - || Modifier.isFinal(mod)) { - throwClassValidationException(origCtClass); + int mod = origClass.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isAbstract(mod) || Modifier.isFinal(mod)) { + throwClassValidationException(origClass); } // interface, enum - if (origCtClass.isInterface() || origCtClass.isEnum()) { - throwClassValidationException(origCtClass); + if (origClass.isInterface() || origClass.isEnum()) { + throwClassValidationException(origClass); } // annotation - checkPackUnpackAnnotation(origCtClass); + checkPackUnpackAnnotation(origClass); } - private static void throwClassValidationException(CtClass origCtClass) { + private static void throwClassValidationException(Class origClass) { throw new PackUnpackUtilException( "it must be a public class and have @" + MessagePackUnpackable.class.getName() + ": " - + origCtClass.getName()); + + origClass.getName()); } - private void checkPackUnpackAnnotation(CtClass origCtClass) { - try { - Object[] objs = origCtClass.getAnnotations(); - for (Object obj : objs) { - if (obj instanceof MessagePackUnpackable) { - return; - } - } - throwClassValidationException(origCtClass); - } catch (ClassNotFoundException e) { - throw new PackUnpackUtilException(e.getMessage(), e); + private void checkPackUnpackAnnotation(Class origClass) { + Annotation anno = origClass + .getAnnotation(MessagePackUnpackable.class); + if (anno == null) { + throwClassValidationException(origClass); } } - private void checkDefaultConstructorValidation(CtClass origCtClass) { - CtConstructor cons = null; + private void checkDefaultConstructorValidation(Class origClass) { + Constructor cons = null; try { - cons = origCtClass.getDeclaredConstructor(new CtClass[0]); - } catch (NotFoundException e) { - throwConstructoValidationException(origCtClass); + cons = origClass.getDeclaredConstructor(new Class[0]); + } catch (Exception e1) { + throwConstructoValidationException(origClass); } + int mod = cons.getModifiers(); if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) { - throwConstructoValidationException(origCtClass); + throwConstructoValidationException(origClass); } } private static void throwConstructoValidationException( - CtClass origCtClass) { + Class origClass) { throw new PackUnpackUtilException( "it must have a public zero-argument constructor: " - + origCtClass.getName()); + + origClass.getName()); } private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) @@ -282,301 +258,430 @@ public class PackUnpackUtil { enhCtClass.addConstructor(newCtCons); } - private CtField[] getDeclaredFields(CtClass origCtClass) { - ArrayList allFields = new ArrayList(); - try { - CtClass nextCtClass = origCtClass; - while (!nextCtClass - .equals(pool.get(Constants.TYPE_NAME_OBJECT))) { - CtField[] fields = nextCtClass.getDeclaredFields(); - for (CtField field : fields) { - try { - checkFieldValidation(field, allFields); - allFields.add(field); - } catch (PackUnpackUtilException e) { // ignore - } + private Field[] getDeclaredFields(Class origClass) { + ArrayList allFields = new ArrayList(); + Class nextClass = origClass; + while (nextClass != Object.class) { + Field[] fields = nextClass.getDeclaredFields(); + for (Field field : fields) { + try { + checkFieldValidation(field, allFields); + allFields.add(field); + } catch (PackUnpackUtilException e) { // ignore } - nextCtClass = nextCtClass.getSuperclass(); } - - } catch (NotFoundException e) { - throw new PackUnpackUtilException(e.getMessage(), e); + nextClass = nextClass.getSuperclass(); } - return allFields.toArray(new CtField[0]); + return allFields.toArray(new Field[0]); } - private void checkFieldValidation(CtField field, - ArrayList allFields) { + private void checkFieldValidation(Field field, List fields) { // check modifiers (public or protected) int mod = field.getModifiers(); if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) || Modifier.isStatic(mod) || Modifier.isFinal(mod) - || Modifier.isTransient(mod)) { + || Modifier.isTransient(mod) || field.isSynthetic()) { throwFieldValidationException(field); } // check same name - for (CtField f : allFields) { + for (Field f : fields) { if (f.getName().equals(field.getName())) { throwFieldValidationException(field); } } } - private static void throwFieldValidationException(CtField field) { + private static void throwFieldValidationException(Field field) { throw new PackUnpackUtilException("it must be a public field: " + field.getName()); } private void addMessagePackMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); - sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( - Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.METHOD_NAME_MSGPACK).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.TYPE_NAME_PACKER).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.KEYWORD_THROWS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_IOEXCEPTION).append( - Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( - Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append( - Constants.METHOD_NAME_PACKARRAY).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length) - .append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - for (CtField field : fields) { - insertCodeOfMessagePack(field, sb); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_PACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(fields.length); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessagePack(sb, field); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messagePack method: " + sb.toString()); + //System.out.println("messagePack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessagePack(CtField field, StringBuilder sb) - throws NotFoundException { - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK) - .append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - field.getName()).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); + private void insertCodeOfMessagePack(StringBuilder sb, Field field) { + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } private void addMessageUnpackMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException, NotFoundException { StringBuilder sb = new StringBuilder(); - sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( - Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.METHOD_NAME_MSGUNPACK).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.TYPE_NAME_UNPACKER).append( - Constants.CHAR_NAME_SPACE).append( - Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.KEYWORD_THROWS).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( - Constants.CHAR_NAME_COMMA).append( - Constants.CHAR_NAME_SPACE).append( - Constants.TYPE_NAME_IOEXCEPTION).append( - Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( - Constants.CHAR_NAME_SPACE); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT).append( - Constants.METHOD_NAME_UNPACKARRAY).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS).append( - Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( - Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - for (CtField field : fields) { - insertCodeOfMessageUnpack(field, sb); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGUNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_UNPACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessageUnpack(sb, field, field.getType()); } sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messageUnpack method: " + sb.toString()); + //System.out.println("messageUnpack method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } - private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb) - throws NotFoundException { - CtClass type = field.getType(); - insertRightVariable(sb, field, type); - insertValueOfMethodAndLeftParenthesis(sb, type); - sb.append(Constants.VARIABLE_NAME_PK).append( - Constants.CHAR_NAME_DOT); - insertUnpackMethod(sb, type); - sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); - insertUnpackMethodArgumenet(sb, field, type); - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - insertValueOfMethodAndRightParenthesis(sb, type); - sb.append(Constants.CHAR_NAME_SEMICOLON).append( - Constants.CHAR_NAME_SPACE); - - } - - private void insertRightVariable(StringBuilder sb, CtField field, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE) - .append(Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - || type - .equals(pool - .get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger - || type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String - || type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[] - || type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List - || type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map - ) { - sb.append(field.getName()) - .append(Constants.CHAR_NAME_SPACE).append( - Constants.CHAR_NAME_EQUAL).append( - Constants.CHAR_NAME_SPACE); - } else { // MessageUnpackable - return; - } - } - - } - - private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - ) { - sb.append(type.getName()).append(Constants.CHAR_NAME_DOT) - .append(Constants.METHOD_NAME_VALUEOF).append( - Constants.CHAR_NAME_LEFT_PARENTHESIS); - } else { - return; - } + private void insertCodeOfMessageUnpack(StringBuilder sb, Field field, + Class type) throws NotFoundException { + if (type.isPrimitive()) { + // primitive type + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (type.equals(Boolean.class) || // Boolean + type.equals(Byte.class) || // Byte + type.equals(Double.class) || // Double + type.equals(Float.class) || // Float + type.equals(Integer.class) || // Integer + type.equals(Long.class) || // Long + type.equals(Short.class)) { // Short + // reference type (wrapper type) + insertCodeOfMessageUnpackForWrapperTypes(sb, field, type); + } else if (type.equals(BigInteger.class) || // BigInteger + type.equals(String.class) || // String + type.equals(byte[].class)) { // byte[] + // reference type (other type) + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (List.class.isAssignableFrom(type)) { + // List + insertCodeOfMessageUnpackForListType(sb, field, type); + } else if (Map.class.isAssignableFrom(type)) { + // Map + insertCodeOfMessageUnpackForMapType(sb, field, type); + } else if (MessageUnpackable.class.isAssignableFrom(type) + || (getCache(type.getName()) != null)) { + // MessageUnpackable + insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type); + } else { + throw new NotFoundException("unknown type: " + type.getName()); } } - private void insertUnpackMethod(StringBuilder sb, CtClass type) + private void insertCodeOfMessageUnpackForPrimitiveTypes( + StringBuilder sb, Field field, Class type) throws NotFoundException { - if (type.equals(CtClass.booleanType)) { // boolean + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert unpack method + if (type.equals(boolean.class)) { // boolean sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); - } else if (type.equals(CtClass.byteType)) { // byte + } else if (type.equals(byte.class)) { // byte sb.append(Constants.METHOD_NAME_UNPACKBYTE); - } else if (type.equals(CtClass.doubleType)) { // double + } else if (type.equals(double.class)) { // double sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); - } else if (type.equals(CtClass.floatType)) { // float + } else if (type.equals(float.class)) { // float sb.append(Constants.METHOD_NAME_UNPACKFLOAT); - } else if (type.equals(CtClass.intType)) { // int + } else if (type.equals(int.class)) { // int sb.append(Constants.METHOD_NAME_UNPACKINT); - } else if (type.equals(CtClass.longType)) { // long + } else if (type.equals(long.class)) { // long sb.append(Constants.METHOD_NAME_UNPACKLONG); - } else if (type.equals(CtClass.shortType)) { // short + } else if (type.equals(short.class)) { // short sb.append(Constants.METHOD_NAME_UNPACKSHORT); } else { // reference type - Class c = null; - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean - sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); - } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte - sb.append(Constants.METHOD_NAME_UNPACKBYTE); - } else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double - sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); - } else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float - sb.append(Constants.METHOD_NAME_UNPACKFLOAT); - } else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer - sb.append(Constants.METHOD_NAME_UNPACKINT); - } else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long - sb.append(Constants.METHOD_NAME_UNPACKLONG); - } else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short - sb.append(Constants.METHOD_NAME_UNPACKSHORT); - } else if (type - .equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger + if (type.equals(BigInteger.class)) { // BigInteger sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER); - } else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String + } else if (type.equals(String.class)) { // String sb.append(Constants.METHOD_NAME_UNPACKSTRING); - } else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[] + } else if (type.equals(byte[].class)) { // byte[] sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY); - } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List - sb.append(Constants.METHOD_NAME_UNPACKARRAY); - } else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map - sb.append(Constants.METHOD_NAME_UNPACKMAP); - } else if (type.subtypeOf(pool - .get(Constants.TYPE_NAME_MSGUNPACKABLE)) - || ((c = getCache(type.getName())) != null)) { // MessageUnpackable - sb.append(Constants.METHOD_NAME_UNPACK); } else { throw new NotFoundException("unknown type: " + type.getName()); } } - } - - private void insertUnpackMethodArgumenet(StringBuilder sb, - CtField field, CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - Class c = null; - if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE)) - || ((c = getCache(type.getName())) != null)) { - sb.append("(org.msgpack.MessageUnpackable)"); - sb.append(field.getName()); - } else { - return; - } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } } - private void insertValueOfMethodAndRightParenthesis(StringBuilder sb, - CtClass type) throws NotFoundException { - if (type.isPrimitive()) { // primitive type - return; - } else { // reference type - if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean - || type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte - || type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double - || type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float - || type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer - || type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long - || type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short - ) { - sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); - } else { - return; - } + private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb, + Field field, Class type) throws NotFoundException { + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); } + sb.append(type.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_VALUEOF); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert the name of a unpack method + if (type.equals(Boolean.class)) { // Boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(Byte.class)) { // Byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(Double.class)) { // Double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(Float.class)) { // Float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(Integer.class)) { // Integer + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(Long.class)) { // Long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(Short.class)) { // Short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { + throw new NotFoundException("unknown type: " + type.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + + private void insertCodeOfMessageUnpackForListType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType = (Class) generic.getActualTypeArguments()[0]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(ArrayList.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ADD); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMapType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType0 = (Class) generic.getActualTypeArguments()[0]; + Class genericType1 = (Class) generic.getActualTypeArguments()[1]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKMAP); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(HashMap.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block map. + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PUT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType0); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessageUnpack(sb, null, genericType1); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMsgUnpackableType( + StringBuilder sb, Field field, Class type) { + // insert a right variable // ignore + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageUnpackable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); } private void addMessageConvertMethod(CtClass enhCtClass, - CtClass origCtClass, CtField[] fields) + CtClass origCtClass, Field[] fields) throws CannotCompileException { StringBuilder sb = new StringBuilder(); sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( @@ -597,7 +702,7 @@ public class PackUnpackUtil { Constants.CHAR_NAME_SPACE); // TODO sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); - System.out.println("messageConvert method: " + sb.toString()); + //System.out.println("messageConvert method: " + sb.toString()); CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); enhCtClass.addMethod(newCtMethod); } @@ -684,6 +789,6 @@ public class PackUnpackUtil { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); - System.out.println(src.equals(dst)); + //System.out.println(src.equals(dst)); } } diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java index 1cde65f2..dbea7cb2 100644 --- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -3,6 +3,9 @@ package org.msgpack.util.annotation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -16,9 +19,9 @@ import org.msgpack.Unpacker; public class TestMessagePackUnpackable extends TestCase { @Test - public void testGeneralPrimitiveTypeFields() throws Exception { - GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil - .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + public void testPrimitiveTypeFields() throws Exception { + PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); src.f0 = (byte) 0; src.f1 = 1; src.f2 = 2; @@ -28,8 +31,8 @@ public class TestMessagePackUnpackable extends TestCase { src.f6 = false; ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(src); - GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil - .newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class); + PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker pac = new Unpacker(in); pac.unpack((MessageUnpackable) dst); @@ -43,7 +46,7 @@ public class TestMessagePackUnpackable extends TestCase { } @MessagePackUnpackable - public static class GeneralPrimitiveTypeFieldsClass { + public static class PrimitiveTypeFieldsClass { public byte f0; public short f1; public int f2; @@ -52,7 +55,7 @@ public class TestMessagePackUnpackable extends TestCase { public double f5; public boolean f6; - public GeneralPrimitiveTypeFieldsClass() { + public PrimitiveTypeFieldsClass() { } } @@ -107,15 +110,93 @@ public class TestMessagePackUnpackable extends TestCase { } } - public void testListAndMap() throws Exception { + public void testListTypes() throws Exception { + SampleListTypes src = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + src.f0 = new ArrayList(); + src.f1 = new ArrayList(); + src.f1.add(1); + src.f1.add(2); + src.f1.add(3); + src.f2 = new ArrayList(); + src.f2.add("e1"); + src.f2.add("e2"); + src.f2.add("e3"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleListTypes dst = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + for (int i = 0; i < src.f1.size(); ++i) { + assertEquals(src.f1.get(i), dst.f1.get(i)); + } + assertEquals(src.f2.size(), dst.f2.size()); + for (int i = 0; i < src.f2.size(); ++i) { + assertEquals(src.f2.get(i), dst.f2.get(i)); + } } @MessagePackUnpackable - public static class ListAndMapClass { - public List f0; - public Map f1; + public static class SampleListTypes { + public List f0; + public List f1; + public List f2; - public ListAndMapClass() { + public SampleListTypes() { + } + } + + public void testMapTypes() throws Exception { + SampleMapTypes src = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + src.f0 = new HashMap(); + src.f1 = new HashMap(); + src.f1.put(1, 1); + src.f1.put(2, 2); + src.f1.put(3, 3); + src.f2 = new HashMap(); + src.f2.put("k1", 1); + src.f2.put("k2", 2); + src.f2.put("k3", 3); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + Iterator srcf1 = src.f1.keySet().iterator(); + Iterator dstf1 = dst.f1.keySet().iterator(); + while (srcf1.hasNext()) { + Integer s1 = srcf1.next(); + Integer d1 = dstf1.next(); + assertEquals(s1, d1); + assertEquals(src.f1.get(s1), dst.f1.get(d1)); + } + assertEquals(src.f2.size(), dst.f2.size()); + Iterator srcf2 = src.f2.keySet().iterator(); + Iterator dstf2 = dst.f2.keySet().iterator(); + while (srcf2.hasNext()) { + String s2 = srcf2.next(); + String d2 = dstf2.next(); + assertEquals(s2, d2); + assertEquals(src.f2.get(s2), dst.f2.get(d2)); + } + } + + @MessagePackUnpackable + public static class SampleMapTypes { + public Map f0; + public Map f1; + public Map f2; + + public SampleMapTypes() { } } From 130d2064d583065c185e7d6d868c6c51b3731fa2 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Fri, 17 Sep 2010 15:28:24 +0900 Subject: [PATCH 09/50] perl: updated benchmark result! gfx++ # performance tuning --- perl/lib/Data/MessagePack.pm | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b1e0174d..7456c3a0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -113,23 +113,32 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) - + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- + -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- =head1 AUTHORS From 2c9d90d463429902226eca593fccebec440a57ef Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:08:15 +0900 Subject: [PATCH 10/50] perl: regen README --- perl/README | 28 ++++++++++++++++++---------- perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/perl/README b/perl/README index e46323da..6272a3b2 100644 --- a/perl/README +++ b/perl/README @@ -56,21 +56,29 @@ SPEED -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- AUTHORS Tokuhiro Matsuno diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 7456c3a0..609ea3cd 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -126,7 +126,7 @@ This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my storable 38399/s -- -73% -89% json 141940/s 270% -- -60% mp 355501/s 826% 150% -- - + -- deserialize JSON::XS: 2.3 Data::MessagePack: 0.24 From e8d8099563ced184991b188bfe7ed4189e24f334 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:08:34 +0900 Subject: [PATCH 11/50] Fix a macro redefinition --- perl/xs-src/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 6ebb48c2..e89b22c5 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -142,7 +142,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -#define template_callback_uint64 template_callback_IV +#define template_callback_int64 template_callback_IV #endif /* IVSIZE */ From a0c18e4380a6c96688101e508acab43982dddfe1 Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:16:33 +0900 Subject: [PATCH 12/50] Docs --- perl/lib/Data/MessagePack.pm | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 609ea3cd..52021090 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -45,7 +45,7 @@ Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); =head1 DESCRIPTION @@ -61,11 +61,11 @@ It enables to exchange structured objects between many languages like JSON. But =over 4 -=item PORTABILITY +=item PORTABLE -Messagepack is language independent binary serialize format. +The MessagePack format does not depend on language nor byte order. -=item SMALL SIZE +=item SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 @@ -76,6 +76,7 @@ The MessagePack format saves memory than JSON and Storable format. =item STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for networking such as RPC. +See L for details. =back @@ -105,13 +106,13 @@ unpack the $msgpackstr to a MessagePack format string. =item $Data::MessagePack::PreferInteger -Pack the string as int when the value looks like int(EXPERIMENTAL). +Packs a string as an integer, when it looks like an integer. =back =head1 SPEED -This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). +This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). -- serialize @@ -156,13 +157,15 @@ FURUHASHI Sadayuki hanekomu +gfx + =head1 LICENSE This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. - =head1 SEE ALSO -L is official web site for MessagePack format. +L is the official web site for the MessagePack format. +=cut From 7c8f8703a19d77e23e07b777054d888eff49774a Mon Sep 17 00:00:00 2001 From: gfx Date: Fri, 17 Sep 2010 18:26:16 +0900 Subject: [PATCH 13/50] Add TODOs --- perl/lib/Data/MessagePack.pm | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 52021090..8c5cfac6 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -141,6 +141,25 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +=head1 TODO + +=over + +=item Error handling + +MessagePack cannot deal with complex scalars such as object references, +filehandles, and code references. We should report the errors more kindly. + +=item Streaming deserializer + +The current implementation of the streaming deserializer does not have internal +buffers while some other bindings (such as Ruby binding) does. This limitation +will astonish those who try to unpack byte streams with an arbitrary buffer size +(e.g. C<< while(read($socket, $buffer, $arbitrary_buffer_size)) { ... } >>). +We should implement the internal buffer for the unpacker. + +=back + =head1 AUTHORS Tokuhiro Matsuno From 845af014dce3c54c4294abefa8eb33b6bb39dd61 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:15:51 +0900 Subject: [PATCH 14/50] perl: gfx is a author. --- perl/lib/Data/MessagePack.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 8c5cfac6..953bdf85 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -166,6 +166,8 @@ Tokuhiro Matsuno Makamaka Hannyaharamitu +gfx + =head1 THANKS TO Jun Kuriyama @@ -176,8 +178,6 @@ FURUHASHI Sadayuki hanekomu -gfx - =head1 LICENSE This library is free software; you can redistribute it and/or modify From 446266776eb1a36fd562307eb6de4c64e6cc36d1 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:16:17 +0900 Subject: [PATCH 15/50] perl: regenerate README file --- perl/README | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/perl/README b/perl/README index 6272a3b2..31052789 100644 --- a/perl/README +++ b/perl/README @@ -2,7 +2,7 @@ NAME Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); DESCRIPTION @@ -14,10 +14,10 @@ ABOUT MESSAGEPACK FORMAT But unlike JSON, it is very fast and small. ADVANTAGES - PORTABILITY - Messagepack is language independent binary serialize format. + PORTABLE + The MessagePack format does not depend on language nor byte order. - SMALL SIZE + SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 @@ -26,7 +26,7 @@ ABOUT MESSAGEPACK FORMAT STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for - networking such as RPC. + networking such as RPC. See Data::MessagePack::Unpacker for details. If you want to get more information about the MessagePack format, please visit to . @@ -47,12 +47,11 @@ METHODS Configuration Variables $Data::MessagePack::PreferInteger - Pack the string as int when the value looks like int(EXPERIMENTAL). + Packs a string as an integer, when it looks like an integer. SPEED - This is the result of benchmark/serialize.pl and - benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu - SMP). + This is a result of benchmark/serialize.pl and benchmark/deserialize.pl + on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). -- serialize JSON::XS: 2.3 @@ -80,11 +79,27 @@ SPEED json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +TODO + Error handling + MessagePack cannot deal with complex scalars such as object + references, filehandles, and code references. We should report the + errors more kindly. + + Streaming deserializer + The current implementation of the streaming deserializer does not + have internal buffers while some other bindings (such as Ruby + binding) does. This limitation will astonish those who try to unpack + byte streams with an arbitrary buffer size (e.g. + "while(read($socket, $buffer, $arbitrary_buffer_size)) { ... }"). We + should implement the internal buffer for the unpacker. + AUTHORS Tokuhiro Matsuno Makamaka Hannyaharamitu + gfx + THANKS TO Jun Kuriyama @@ -99,5 +114,6 @@ LICENSE under the same terms as Perl itself. SEE ALSO - is official web site for MessagePack format. + is the official web site for the MessagePack + format. From 953aa95c648fef85f9c11c5ed251f3ddab988a83 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 06:16:26 +0900 Subject: [PATCH 16/50] perl: added failing test case for streaming unpacker with PP. --- perl/t/11_stream_unpack3.t | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 perl/t/11_stream_unpack3.t diff --git a/perl/t/11_stream_unpack3.t b/perl/t/11_stream_unpack3.t new file mode 100644 index 00000000..0eb8bff7 --- /dev/null +++ b/perl/t/11_stream_unpack3.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use Test::More; +use Data::MessagePack; + +my @data = ( [ 1, 2, 3 ], [ 4, 5, 6 ] ); + +# serialize +my $buffer = ''; +for my $d (@data) { + $buffer .= Data::MessagePack->pack($d); +} + +# deserialize +my $cb = sub { + my ($data) = @_; + + my $d = shift @data; + is_deeply $data, $d; +}; +my $unpacker = Data::MessagePack::Unpacker->new(); +my $nread = 0; +while (1) { + $nread = $unpacker->execute( $buffer, $nread ); + if ( $unpacker->is_finished ) { + my $ret = $unpacker->data; + $cb->( $ret ); + $unpacker->reset; + + $buffer = substr( $buffer, $nread ); + $nread = 0; + next if length($buffer) != 0; + } + last; +} +is scalar(@data), 0; + +done_testing; + From 2c9966a0a304dc4bdb5fc003100ec37b2ec3d70a Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 18 Sep 2010 09:44:32 +0900 Subject: [PATCH 17/50] perl: fixed stream deserializer in pp. --- perl/lib/Data/MessagePack/PP.pm | 48 ++++++++++++++++++--------------- perl/t/03_stream_unpack.t | 4 +-- perl/t/09_stddata.t | 1 + perl/t/10_splitted_bytes.t | 6 +++-- perl/t/12_stream_unpack3.t | 23 ++++++++++++++++ 5 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 perl/t/12_stream_unpack3.t diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 6a06c3c6..0dd64272 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -370,7 +370,7 @@ package use strict; sub new { - bless { stack => [] }, shift; + bless { pos => 0 }, shift; } @@ -384,25 +384,30 @@ sub execute_limit { sub execute { my ( $self, $data, $offset, $limit ) = @_; - my $value = substr( $data, $offset || 0, $limit ? $limit : length $data ); + $offset ||= 0; + my $value = substr( $data, $offset, $limit ? $limit : length $data ); my $len = length $value; + $self->{data} .= $value; + local $self->{stack} = []; + $p = 0; - while ( $len > $p ) { - _count( $self, $value ) or last; + LOOP: while ( length($self->{data}) > $p ) { + _count( $self, $self->{data} ) or last; - if ( @{ $self->{stack} } > 0 ) { - pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0; + while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) { + pop @{ $self->{stack} }; + } + + if (@{$self->{stack}} == 0) { + $self->{is_finished}++; + last LOOP; } } + $self->{pos} = $p; - if ( $len == $p ) { - $self->{ data } .= substr( $value, 0, $p ); - $self->{ remain } = undef; - } - - return $p; + return $p + $offset; } @@ -424,7 +429,9 @@ sub _count { $num = $byte & ~0x90; } - push @{ $self->{stack} }, $num + 1; + if (defined($num) && $num > 0) { + push @{ $self->{stack} }, $num + 1; + } return 1; } @@ -443,7 +450,9 @@ sub _count { $num = $byte & ~0x80; } - push @{ $self->{stack} }, $num * 2 + 1; # a pair + if ($num > 0) { + push @{ $self->{stack} }, $num * 2 + 1; # a pair + } return 1; } @@ -511,22 +520,19 @@ sub _count { sub data { - my $data = Data::MessagePack->unpack( $_[0]->{ data } ); - $_[0]->reset; - return $data; + return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) ); } sub is_finished { my ( $self ) = @_; - ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1; + return $self->{is_finished}; } - sub reset :method { - $_[0]->{ stack } = []; $_[0]->{ data } = undef; - $_[0]->{ remain } = undef; + $_[0]->{ pos } = 0; + $_[0]->{ is_finished } = 0; } 1; diff --git a/perl/t/03_stream_unpack.t b/perl/t/03_stream_unpack.t index a4ab4eba..646fc249 100644 --- a/perl/t/03_stream_unpack.t +++ b/perl/t/03_stream_unpack.t @@ -37,7 +37,7 @@ for (my $i=0; $iexecute("\xc0", 0); # nil } - ok $up->is_finished; - is_deeply $up->data, [undef, undef, undef, undef, undef]; + ok $up->is_finished, 'finished'; + is_deeply $up->data, [undef, undef, undef, undef, undef], 'array, is_deeply'; } diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index a618787d..f98d696b 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -32,6 +32,7 @@ for my $mpac($mpac1, $mpac2) { my $i = 0; while($offset < length($mpac)) { $offset = $mps->execute($mpac, $offset); + ok $mps->is_finished, "data[$i] : is_finished"; is_deeply $mps->data, $data[$i], "data[$i]"; $mps->reset; $i++; diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t index 232d8707..15598f4e 100644 --- a/perl/t/10_splitted_bytes.t +++ b/perl/t/10_splitted_bytes.t @@ -27,12 +27,14 @@ foreach my $size(1 .. 16) { open my $stream, '<:bytes :scalar', \$packed; binmode $stream; my $buff; + my $done = 0; while( read($stream, $buff, $size) ) { #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - $up->execute($buff); + $done = $up->execute($buff); } - ok $up->is_finished, 'is_finished'; + is $done, length($packed); + ok $up->is_finished, "is_finished: $size"; my $data = $up->data; is_deeply $data, $input; } diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack3.t new file mode 100644 index 00000000..118acc30 --- /dev/null +++ b/perl/t/12_stream_unpack3.t @@ -0,0 +1,23 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my @input = ( + +[[]], + [[],[]], + [{"a" => 97},{"a" => 97}], + [{"a" => 97},{"a" => 97},{"a" => 97}], +); + +plan tests => @input * 2; + +for my $input (@input) { + my $packed = Data::MessagePack->pack($input); + my $up = Data::MessagePack::Unpacker->new(); + $up->execute($packed, 0); + ok $up->is_finished, 'finished'; + is_deeply($up->data, $input); +} + From 4767e450351dfbc807e5346ceaa4252bd99ae866 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 13:18:25 +0900 Subject: [PATCH 18/50] perl: fix a test name --- perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} (100%) diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack4.t similarity index 100% rename from perl/t/12_stream_unpack3.t rename to perl/t/12_stream_unpack4.t From 1f07721ec41147e02fa49aea19a3f6aa7b1eb723 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 13:33:27 +0900 Subject: [PATCH 19/50] perl: Scalar::Util is no longer used --- perl/lib/Data/MessagePack/PP.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 0dd64272..abb6e9a0 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -12,7 +12,6 @@ use Carp (); package Data::MessagePack; -use Scalar::Util qw( blessed ); use strict; use B (); From c707392a5a9307504595f6fb9f11930a6a514531 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:30:08 +0900 Subject: [PATCH 20/50] perl: fix int64_t unpacking in both XS and PP --- perl/lib/Data/MessagePack/PP.pm | 67 ++++++++++++++++++++------------- perl/t/data.pl | 30 ++++++++++++--- perl/xs-src/unpack.c | 32 +++++++++++----- 3 files changed, 89 insertions(+), 40 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index abb6e9a0..c3ce230c 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -16,12 +16,44 @@ use strict; use B (); BEGIN { + my $unpack_int64_slow; + my $unpack_uint64_slow; + + if(!eval { pack 'Q', 1 }) { # don't have quad types + $unpack_int64_slow = sub { + require Math::BigInt; + my $high = Math::BigInt->new( unpack_int32( $_[0], $_[1]) ); + my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + + return +($high << 32 | $low)->bstr; + }; + $unpack_uint64_slow = sub { + require Math::BigInt; + my $high = Math::BigInt->new( unpack_uint32( $_[0], $_[1]) ); + my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + return +($high << 32 | $low)->bstr; + }; + } + + *unpack_uint16 = sub { return unpack 'n', substr( $_[0], $_[1], 2 ) }; + *unpack_uint32 = sub { return unpack 'N', substr( $_[0], $_[1], 4 ) }; + # for pack and unpack compatibility if ( $] < 5.010 ) { # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? $v - 0x100000000 : 0; + }; + # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? if($bo_is_le) { *pack_uint64 = sub { @@ -46,20 +78,11 @@ BEGIN { return unpack( 'd', pack( 'N2', @v[1,0] ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { + *unpack_int64 = $unpack_int64_slow ||_sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); }; - *unpack_uint64 = sub { + *unpack_uint64 = $unpack_uint64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'Q', pack( 'N2', @v[1,0] ) ); }; @@ -71,17 +94,8 @@ BEGIN { *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + *unpack_int64 = $unpack_int64_slow || sub { pack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; } } else { @@ -93,8 +107,9 @@ BEGIN { *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; *unpack_int16 = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); }; *unpack_int32 = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); }; - *unpack_int64 = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; - *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; + + *unpack_int64 = $unpack_int64_slow || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } } @@ -283,11 +298,11 @@ sub _unpack { } elsif ( $byte == 0xcd ) { # uint16 $p += 2; - return CORE::unpack 'n', substr( $value, $p - 2, 2 ); + return unpack_uint16( $value, $p - 2 ); } elsif ( $byte == 0xce ) { # unit32 $p += 4; - return CORE::unpack 'N', substr( $value, $p - 4, 4 ); + return unpack_uint32( $value, $p - 4 ); } elsif ( $byte == 0xcf ) { # unit64 $p += 8; diff --git a/perl/t/data.pl b/perl/t/data.pl index 2f58d38c..8ffd25ae 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -5,14 +5,34 @@ no warnings; # i need this, i need this. '92 90 91 91 c0', [[], [[undef]]], '93 c0 c2 c3', [undef, false, true], 'ce 80 00 00 00', 2147483648, - '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], - '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], - '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], - '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', + [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', + ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, - '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', + [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], 'ce 00 ff ff ff' => ''.0xFFFFFF, 'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF, 'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF, + + 'd2 80 00 00 01' => '-2147483647', # int32_t + 'ce 80 00 00 01' => '2147483649', # uint32_t + + 'd2 ff ff ff ff' => '-1', # int32_t + 'ce ff ff ff ff' => '4294967295', # uint32_t + + 'd3 00 00 00 00 80 00 00 01' => '2147483649', # int64_t + 'cf 00 00 00 00 80 00 00 01' => '2147483649', # uint64_t + + 'd3 ff 00 ff ff ff ff ff ff' => '-71776119061217281', # int64_t + 'cf ff 00 ff ff ff ff ff ff' => '18374967954648334335', # uint64_t + + 'd3 ff ff ff ff ff ff ff ff' => '-1', # int64_t + 'cf ff ff ff ff ff ff ff ff' => '18446744073709551615', # uint64_t ) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index e89b22c5..fefb52e8 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -102,13 +102,6 @@ STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const return 0; } -STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) -{ - dTHX; - *o = newSVnv((NV)d); - return 0; -} - STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o) { dTHX; @@ -116,10 +109,31 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - *o = newSVnv((NV)d); + if((uint64_t)(NV)d == d) { + *o = newSVnv((NV)d); + } + else { + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); + *o = newSVpvn(tbuf, len); + } + return 0; +} + +static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +{ + dTHX; + if((uint64_t)(NV)d == (uint64_t)d) { + *o = newSVnv((NV)d); + } + else { + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); + *o = newSVpvn(tbuf, len); + } return 0; } From a86c1624a70b8f8a5012065019d9f1ba4b44595b Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:38:35 +0900 Subject: [PATCH 21/50] perl: More kind error messages in PP --- perl/lib/Data/MessagePack/PP.pm | 19 +++++++++++-------- perl/t/50_leaktrace.t | 6 +++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index c3ce230c..31c08339 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -113,6 +113,9 @@ BEGIN { } } +sub _unexpected { + Carp::confess("Unexpected " . sprintf(shift, @_) . " found"); +} # # PACK @@ -141,7 +144,7 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x90 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; if ( --$max_depth <= 0 ) { Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); @@ -155,7 +158,7 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x80 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; if ( --$max_depth <= 0 ) { Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); @@ -211,7 +214,7 @@ sub _pack { $num < 32 ? CORE::pack( 'C', 0xa0 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num ) - : die "" # don't arrivie here + : _unexpected_number($num) ; return $header . $value; @@ -221,7 +224,7 @@ sub _pack { return pack_double( $value ); } else { - die "???"; + _unexpected("data type %s", $b_obj); } } @@ -365,7 +368,7 @@ sub _unpack { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } } @@ -484,7 +487,7 @@ sub _count { : $byte == 0xcd ? 2 : $byte == 0xce ? 4 : $byte == 0xcf ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -493,7 +496,7 @@ sub _count { : $byte == 0xd1 ? 2 : $byte == 0xd2 ? 4 : $byte == 0xd3 ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -524,7 +527,7 @@ sub _count { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } return 0; diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t index 29485270..440ac901 100644 --- a/perl/t/50_leaktrace.t +++ b/perl/t/50_leaktrace.t @@ -2,8 +2,12 @@ use strict; use Test::Requires { 'Test::LeakTrace' => 0.13 }; use Test::More; - use Data::MessagePack; +BEGIN { + if($INC{'Data/MessagePack/PP.pm'}) { + plan skip_all => 'disabled in PP'; + } +} my $simple_data = "xyz"; my $complex_data = { From bab622de25042d11dc8c149da26e178c22398c56 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:46:10 +0900 Subject: [PATCH 22/50] perl: fix max depth checks in PP --- perl/lib/Data/MessagePack/PP.pm | 16 ++++++++-------- perl/t/12_stream_unpack4.t | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 31c08339..8a904a97 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -124,11 +124,11 @@ sub _unexpected { { no warnings 'recursion'; - my $max_depth; + our $_max_depth; sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; - $max_depth = defined $_[2] ? $_[2] : 512; # init + $_max_depth = defined $_[2] ? $_[2] : 512; # init return _pack( $_[1] ); } @@ -136,6 +136,12 @@ sub pack :method { sub _pack { my ( $value ) = @_; + local $_max_depth = $_max_depth - 1; + + if ( $_max_depth < 0 ) { + Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); + } + return CORE::pack( 'C', 0xc0 ) if ( not defined $value ); if ( ref($value) eq 'ARRAY' ) { @@ -146,9 +152,6 @@ sub _pack { : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } @$value ); } @@ -160,9 +163,6 @@ sub _pack { : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } %$value ); } diff --git a/perl/t/12_stream_unpack4.t b/perl/t/12_stream_unpack4.t index 118acc30..de76e81d 100644 --- a/perl/t/12_stream_unpack4.t +++ b/perl/t/12_stream_unpack4.t @@ -9,6 +9,7 @@ my @input = ( [[],[]], [{"a" => 97},{"a" => 97}], [{"a" => 97},{"a" => 97},{"a" => 97}], + [ map { +{ "foo $_" => "bar $_" } } 'aa' .. 'zz' ], ); plan tests => @input * 2; From e6f6aba2071c0a55b67dc935fa876af6b23e6c11 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:50:52 +0900 Subject: [PATCH 23/50] perl: add portability stuff --- perl/xs-src/unpack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index fefb52e8..a4929c04 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -1,5 +1,6 @@ #define NEED_newRV_noinc #define NEED_sv_2pv_flags +#define NEED_my_snprintf #include "xshelper.h" #define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION From 4902bed4098a4c98fc02555584150b1822c90b6b Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:51:17 +0900 Subject: [PATCH 24/50] perl: more kind testing messages --- perl/t/00_compile.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t index f91b29e7..d465f8dc 100644 --- a/perl/t/00_compile.t +++ b/perl/t/00_compile.t @@ -3,4 +3,5 @@ use warnings; use Test::More tests => 1; use_ok 'Data::MessagePack'; -diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' ); +diag ( "Testing Data::MessagePack/$Data::MessagePack::VERSION (", + $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS', ")" ); From 8935ecfdb86ad7a44c4bbf34acdd99cf80457cf0 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:56:33 +0900 Subject: [PATCH 25/50] perl: requires the latest version of Math::BigInt for PP --- perl/Makefile.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index fafc3876..586a0523 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -37,6 +37,7 @@ NOT_SUPPORT_C99 } else { print "configure PP version\n\n"; + requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken } clean_files qw{ From 1865898cd42c2745850e3fd502f32c4584968263 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 14:58:32 +0900 Subject: [PATCH 26/50] perl: Fix Makefile.PL --- perl/Makefile.PL | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 586a0523..b5d2701d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,3 +1,5 @@ +# Usage: Makefile.PL --pp # disable XS +# Makefile.PL -g # add -g to the compiler and disable optimization flags use inc::Module::Install; use Module::Install::XSUtil 0.32; use Config; @@ -21,8 +23,9 @@ if ( $] >= 5.008005 and want_xs() ) { if ( $has_c99 ) { use_xshelper(); cc_src_paths('xs-src'); - if ($ENV{DEBUG}) { - cc_append_to_ccflags '-g'; + + if($Module::Install::AUTHOR) { + postamble qq{test :: test_pp\n\n}; } } else { @@ -67,10 +70,6 @@ test_requires('Test::Requires'); test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' ); -if($Module::Install::AUTHOR) { - postamble qq{test :: test_pp\n\n}; -} - repository('http://github.com/msgpack/msgpack'); auto_include; WriteAll; From 63f6c86b46464cd5ff0c48c691bbfe2cee0a484d Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:05:22 +0900 Subject: [PATCH 27/50] perl: add a note about 64 bit integers --- perl/README | 6 ++++++ perl/lib/Data/MessagePack.pm | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/perl/README b/perl/README index 31052789..34769602 100644 --- a/perl/README +++ b/perl/README @@ -79,6 +79,12 @@ SPEED json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +CAVEAT + Unpacking 64 bit integers + This module can unpack 64 bit integers even if your perl does not + support them (i.e. where "perl -V:ivsize" is 4), but you cannot + calculate these values unless you use "Math::BigInt". + TODO Error handling MessagePack cannot deal with complex scalars such as object diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 953bdf85..29981781 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -141,6 +141,14 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC json 179443/s 56% -- -16% mp 212910/s 85% 19% -- +=head1 CAVEAT + +=head2 Unpacking 64 bit integers + +This module can unpack 64 bit integers even if your perl does not support them +(i.e. where C<< perl -V:ivsize >> is 4), but you cannot calculate these values +unless you use C. + =head1 TODO =over From 49379140c7c431c3a27f11dc7565da118dd38f16 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:14:29 +0900 Subject: [PATCH 28/50] perl: PERL_ONLY=1 disables XS --- perl/lib/Data/MessagePack.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 29981781..0f389b13 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -24,7 +24,7 @@ sub false () { if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; - if ( $backend !~ /\b pp \b/xms ) { + if ( $backend !~ /\b pp \b/xms or $ENV{PERL_ONLY} ) { eval { require XSLoader; XSLoader::load(__PACKAGE__, $VERSION); From cb85dcfcb8bdaf706ab515bce4bc1991c990716c Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:49:25 +0900 Subject: [PATCH 29/50] perl: tweaks for benchmarks --- perl/benchmark/data.pl | 6 ++++++ perl/benchmark/deserialize.pl | 8 ++------ perl/benchmark/serialize.pl | 7 +------ 3 files changed, 9 insertions(+), 12 deletions(-) create mode 100755 perl/benchmark/data.pl diff --git a/perl/benchmark/data.pl b/perl/benchmark/data.pl new file mode 100755 index 00000000..6908d1cc --- /dev/null +++ b/perl/benchmark/data.pl @@ -0,0 +1,6 @@ ++{ + "method" => "handleMessage", + "params" => [ "user1", "we were just talking", "foo\nbar\nbaz\nqux" ], + "id" => undef, + "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2), 1 .. 100 ], +}; diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 634a79ed..b1d7fdf2 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -7,12 +7,8 @@ use Storable; #$Data::MessagePack::PreferInteger = 1; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; +my $a = do 'benchmark/data.pl'; + my $j = JSON::XS::encode_json($a); my $m = Data::MessagePack->pack($a); my $s = Storable::freeze($a); diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index e0509ffa..33746842 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -5,12 +5,7 @@ use JSON::XS; use Storable; use Benchmark ':all'; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; +my $a = do 'benchmark/data.pl'; print "-- serialize\n"; print "JSON::XS: $JSON::XS::VERSION\n"; From b40284955781a152221e6fe6e0d2568ef4fdc27a Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 15:54:22 +0900 Subject: [PATCH 30/50] perl: docs --- perl/lib/Data/MessagePack.pm | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 0f389b13..4e4064b0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -45,6 +45,8 @@ Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS + use Data::MessagePack; + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); @@ -55,7 +57,8 @@ This module converts Perl data structures to MessagePack and vice versa. =head1 ABOUT MESSAGEPACK FORMAT MessagePack is a binary-based efficient object serialization format. -It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. +It enables to exchange structured objects between many languages like JSON. +But unlike JSON, it is very fast and small. =head2 ADVANTAGES @@ -113,7 +116,7 @@ Packs a string as an integer, when it looks like an integer. =head1 SPEED This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). - +(You should benchmark them with B data if the speed matters, of course.) -- serialize JSON::XS: 2.3 @@ -195,4 +198,8 @@ it under the same terms as Perl itself. L is the official web site for the MessagePack format. +L + +L + =cut From c2bf2a817410b013aecc30d6a5abc0cad0423dff Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:16:51 +0900 Subject: [PATCH 31/50] perl: make pp benchmarks available --- perl/benchmark/deserialize.pl | 10 +++++----- perl/benchmark/serialize.pl | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index b1d7fdf2..faa2582f 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -1,25 +1,25 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; -use Benchmark ':all'; +use JSON; use Storable; +use Benchmark ':all'; #$Data::MessagePack::PreferInteger = 1; my $a = do 'benchmark/data.pl'; -my $j = JSON::XS::encode_json($a); +my $j = JSON::encode_json($a); my $m = Data::MessagePack->pack($a); my $s = Storable::freeze($a); print "-- deserialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::decode_json($j) }, + json => sub { JSON::decode_json($j) }, mp => sub { Data::MessagePack->unpack($m) }, storable => sub { Storable::thaw($s) }, } diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index 33746842..4982ff61 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -1,19 +1,19 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; +use JSON; use Storable; use Benchmark ':all'; my $a = do 'benchmark/data.pl'; print "-- serialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::encode_json($a) }, + json => sub { JSON::encode_json($a) }, storable => sub { Storable::freeze($a) }, mp => sub { Data::MessagePack->pack($a) }, } From 29707bd2ead5fa41dd0908fd7039124be1f6c5d8 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:19:16 +0900 Subject: [PATCH 32/50] perl: fix bootstrap --- perl/lib/Data/MessagePack.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 4e4064b0..154c5b17 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -23,8 +23,8 @@ sub false () { } if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate - my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; - if ( $backend !~ /\b pp \b/xms or $ENV{PERL_ONLY} ) { + my $backend = $ENV{PERL_DATA_MESSAGEPACK} || ($ENV{PERL_ONLY} ? 'pp' : ''); + if ( $backend !~ /\b pp \b/xms ) { eval { require XSLoader; XSLoader::load(__PACKAGE__, $VERSION); From 8d182f1d79808812bd095125eee1a43c7f7904f5 Mon Sep 17 00:00:00 2001 From: gfx Date: Sat, 18 Sep 2010 16:30:07 +0900 Subject: [PATCH 33/50] perl: Changelogging --- perl/Changes | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perl/Changes b/perl/Changes index ce525818..d338cf87 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,7 +1,8 @@ 0.24 - - Fixed a possible SEGV on streaming unpacking (gfx) - - Improve performance, esp. in unpacking (gfx) + - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) + - Fixed unpacking issues for 64 bit integers on 32 bit perls (gfx) + - Improved performance, esp. in unpacking (gfx) 0.23 From a1c01c6722a50db0aedcc73f2bb6cdd6f16c8f3a Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:15:31 +0900 Subject: [PATCH 34/50] perl: regen README --- perl/README | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/perl/README b/perl/README index 34769602..224ff08c 100644 --- a/perl/README +++ b/perl/README @@ -2,6 +2,8 @@ NAME Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS + use Data::MessagePack; + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); @@ -51,7 +53,8 @@ Configuration Variables SPEED This is a result of benchmark/serialize.pl and benchmark/deserialize.pl - on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). + on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). (You should + benchmark them with your data if the speed matters, of course.) -- serialize JSON::XS: 2.3 @@ -123,3 +126,7 @@ SEE ALSO is the official web site for the MessagePack format. + Data::MessagePack::Unpacker + + AnyEvent::MPRPC + From d6a825981d14079bf4fa74a0605ecec8e873f94b Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:16:08 +0900 Subject: [PATCH 35/50] perl: fix unpacking int64_t in PP (based on makamaka's patch) --- perl/lib/Data/MessagePack/PP.pm | 15 ++++++++++++--- perl/t/data.pl | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 8a904a97..44940dec 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -22,10 +22,19 @@ BEGIN { if(!eval { pack 'Q', 1 }) { # don't have quad types $unpack_int64_slow = sub { require Math::BigInt; - my $high = Math::BigInt->new( unpack_int32( $_[0], $_[1]) ); - my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + my $high = unpack_uint32( $_[0], $_[1] ); + my $low = unpack_uint32( $_[0], $_[1] + 4); - return +($high << 32 | $low)->bstr; + if($high < 0xF0000000) { # positive + $high = Math::BigInt->new( $high ); + $low = Math::BigInt->new( $low ); + return +($high << 32 | $low)->bstr; + } + else { # negative + $high = Math::BigInt->new( ~$high ); + $low = Math::BigInt->new( ~$low ); + return +( -($high << 32 | $low + 1) )->bstr; + } }; $unpack_uint64_slow = sub { require Math::BigInt; diff --git a/perl/t/data.pl b/perl/t/data.pl index 8ffd25ae..95eac419 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -35,4 +35,9 @@ no warnings; # i need this, i need this. 'd3 ff ff ff ff ff ff ff ff' => '-1', # int64_t 'cf ff ff ff ff ff ff ff ff' => '18446744073709551615', # uint64_t + + # int64_t + 'd3 00 00 00 10 00 00 00 00' => '68719476736', + 'd3 00 00 00 10 00 00 00 01' => '68719476737', + 'd3 10 00 00 00 00 00 00 00' => '1152921504606846976', ) From 5cd37e550533a4d12b3390ff161af83db9b21246 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:20:03 +0900 Subject: [PATCH 36/50] perl: always unpacking 64 bit ints as a string on 32 bit perls --- perl/xs-src/unpack.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index a4929c04..20b345fa 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -113,28 +113,18 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - if((uint64_t)(NV)d == d) { - *o = newSVnv((NV)d); - } - else { - char tbuf[64]; - STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); - *o = newSVpvn(tbuf, len); - } + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); + *o = newSVpvn(tbuf, len); return 0; } static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) { dTHX; - if((uint64_t)(NV)d == (uint64_t)d) { - *o = newSVnv((NV)d); - } - else { - char tbuf[64]; - STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); - *o = newSVpvn(tbuf, len); - } + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); + *o = newSVpvn(tbuf, len); return 0; } From afefbe4e564686ac3b9fac9e1d9782f0114c3383 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:21:25 +0900 Subject: [PATCH 37/50] perl: update .gitignore and MANIFEST.SKIP --- perl/.gitignore | 1 + perl/MANIFEST.SKIP | 1 + 2 files changed, 2 insertions(+) diff --git a/perl/.gitignore b/perl/.gitignore index 3e0e73e5..16568472 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -1,4 +1,5 @@ META.yml +MYMETA.yml Makefile Makefile.old MessagePack.bs diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 71a24e5c..1d2192f5 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -2,6 +2,7 @@ \bCVS\b ^MANIFEST\. ^Makefile$ +^MYMETA\.yml$ ~$ ^# \.old$ From 7f42ed86f29ad9eeab64c6816644bfb83c6eea11 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:47:32 +0900 Subject: [PATCH 38/50] More tests --- perl/t/02_unpack.t | 5 +++-- perl/t/data.pl | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t index 4da69e91..657387a9 100644 --- a/perl/t/02_unpack.t +++ b/perl/t/02_unpack.t @@ -5,13 +5,14 @@ no warnings 'uninitialized'; # i need this. i need this. sub unpackit { my $v = $_[0]; - $v =~ s/ //g; + $v =~ s/ +//g; $v = pack 'H*', $v; return Data::MessagePack->unpack($v); } sub pis ($$) { - is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0] + or diag( explain(unpackit($_[0])) ); } my @dat = do 't/data.pl'; diff --git a/perl/t/data.pl b/perl/t/data.pl index 95eac419..9bf07b7d 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -4,6 +4,11 @@ no warnings; # i need this, i need this. '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], '92 90 91 91 c0', [[], [[undef]]], '93 c0 c2 c3', [undef, false, true], + + '82 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # fix map + 'de 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 16 + 'df 00 00 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 32 + 'ce 80 00 00 00', 2147483648, '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], @@ -39,5 +44,8 @@ no warnings; # i need this, i need this. # int64_t 'd3 00 00 00 10 00 00 00 00' => '68719476736', 'd3 00 00 00 10 00 00 00 01' => '68719476737', + 'd3 00 00 10 00 00 00 00 00' => '17592186044416', + 'd3 00 10 00 00 00 00 00 00' => '4503599627370496', 'd3 10 00 00 00 00 00 00 00' => '1152921504606846976', + 'd3 11 00 00 00 00 00 00 00' => '1224979098644774912', ) From 6379d0fe0f1bc2d3b76480edbe9bd99947aae54e Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 15:53:54 +0900 Subject: [PATCH 39/50] perl: more tests for nil/true/false entities --- perl/t/07_break.t | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/perl/t/07_break.t b/perl/t/07_break.t index dd27ad25..cf3f1079 100644 --- a/perl/t/07_break.t +++ b/perl/t/07_break.t @@ -1,12 +1,23 @@ use Test::More; use Data::MessagePack; use t::Util; -no warnings 'uninitialized'; # i need this. i need this. -plan tests => 1; +plan tests => 4; -my $d = Data::MessagePack->unpack(Data::MessagePack->pack([{x => undef}])); -$d->[0]->{x} = 1; -ok delete $d->[0]->{x}; -$d->[0] = 4; +my $d = Data::MessagePack->unpack(Data::MessagePack->pack({ + nil => undef, + true => true, + false => false, + foo => [undef, true, false], +})); +$d->{nil} = 42; +is $d->{nil}, 42; + +$d->{true} = 43; +is $d->{true}, 43; + +$d->{false} = 44; +is $d->{false}, 44; + +is_deeply $d->{foo}, [undef, true, false]; From 53899cc492fc8ae4bc4095506d25e4d0eff48309 Mon Sep 17 00:00:00 2001 From: gfx Date: Sun, 19 Sep 2010 22:12:28 +0900 Subject: [PATCH 40/50] perl: tweaks for Makefile.PL --- perl/Makefile.PL | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index b5d2701d..219400f2 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -16,12 +16,12 @@ recursive_author_tests('xt'); if ( $] >= 5.008005 and want_xs() ) { - can_cc or die "This module requires a C compiler. Please retry with --pp"; - my $has_c99 = c99_available(); # msgpack C library requires C99. if ( $has_c99 ) { + requires_c99(); use_xshelper(); + cc_warnings; cc_src_paths('xs-src'); if($Module::Install::AUTHOR) { @@ -40,7 +40,7 @@ NOT_SUPPORT_C99 } else { print "configure PP version\n\n"; - requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken + requires 'Math::BigInt' => 1.89; # old versions of BigInt were broken } clean_files qw{ From f59178bc3369f43c3c63f02823bd01c1d4f8314a Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 09:30:26 +0900 Subject: [PATCH 41/50] Checking in changes prior to tagging of version 0.24. Changelog diff is: --- perl/lib/Data/MessagePack.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 154c5b17..5990bdbd 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.23'; +our $VERSION = '0.24'; our $PreferInteger = 0; sub true () { From 978bb5059f544b44cb277e0408103ca4a0c27d47 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 09:54:25 +0900 Subject: [PATCH 42/50] Checking in changes prior to tagging of version 0.25. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index d338cf8..3d455cb 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.25 + + (NO FEATURE CHANGES) + - oops. I failed releng. 0.24 - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index d338cf87..3d455cb3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.25 + + (NO FEATURE CHANGES) + - oops. I failed releng. 0.24 - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 5990bdbd..2cbbe797 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.24'; +our $VERSION = '0.25'; our $PreferInteger = 0; sub true () { From d973192b5e44ade6be1b290bc5020c749b43e585 Mon Sep 17 00:00:00 2001 From: makamaka Date: Mon, 20 Sep 2010 11:49:52 +0900 Subject: [PATCH 43/50] perl: modified a serious code typo in PP --- perl/lib/Data/MessagePack/PP.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 44940dec..23942225 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -87,7 +87,7 @@ BEGIN { return unpack( 'd', pack( 'N2', @v[1,0] ) ); }; - *unpack_int64 = $unpack_int64_slow ||_sub { + *unpack_int64 = $unpack_int64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); }; From 664eefdddb6633badeecbbd4dc654eb113b3814d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 20 Sep 2010 23:40:12 +0900 Subject: [PATCH 44/50] Checking in changes prior to tagging of version 0.26. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 3d455cb..b717a82 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.26 + + - fixed a serious code typo in PP(makamaka) + 0.25 (NO FEATURE CHANGES) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 3d455cb3..b717a828 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.26 + + - fixed a serious code typo in PP(makamaka) + 0.25 (NO FEATURE CHANGES) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 2cbbe797..f6e68eb3 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.25'; +our $VERSION = '0.26'; our $PreferInteger = 0; sub true () { From ead8edc7cd3e797e5457c8ba63a2374fb8a87067 Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 22 Sep 2010 14:12:19 +0900 Subject: [PATCH 45/50] modified be unpack_(u)int64 in PP --- perl/lib/Data/MessagePack/PP.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 23942225..679dc00e 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -103,8 +103,8 @@ BEGIN { *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int64 = $unpack_int64_slow || sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + *unpack_int64 = $unpack_int64_slow || sub { unpack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { unpack 'Q', substr( $_[0], $_[1], 8 ); }; } } else { From 6d9a629b155c7a6f814a6973c6c24da308b295ce Mon Sep 17 00:00:00 2001 From: makamaka Date: Wed, 22 Sep 2010 14:19:09 +0900 Subject: [PATCH 46/50] perl: modified trivial codes in PP::Unpacker --- perl/lib/Data/MessagePack/PP.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 679dc00e..8234b493 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -419,7 +419,7 @@ sub execute { $p = 0; - LOOP: while ( length($self->{data}) > $p ) { + while ( length($self->{data}) > $p ) { _count( $self, $self->{data} ) or last; while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) { @@ -428,7 +428,7 @@ sub execute { if (@{$self->{stack}} == 0) { $self->{is_finished}++; - last LOOP; + last; } } $self->{pos} = $p; @@ -455,7 +455,7 @@ sub _count { $num = $byte & ~0x90; } - if (defined($num) && $num > 0) { + if ( $num ) { push @{ $self->{stack} }, $num + 1; } @@ -476,7 +476,7 @@ sub _count { $num = $byte & ~0x80; } - if ($num > 0) { + if ( $num ) { push @{ $self->{stack} }, $num * 2 + 1; # a pair } From 80f7c54e4d14e4762714c6b26e1dcd994f2f0727 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:25:08 +0900 Subject: [PATCH 47/50] perl: make scopes --- perl/xs-src/unpack.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 20b345fa..95b5910f 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -31,6 +31,7 @@ typedef struct { #define msgpack_unpack_user unpack_user void init_Data__MessagePack_unpack(pTHX_ bool const cloning) { + // booleans are load on demand (lazy load). if(!cloning) { MY_CXT_INIT; MY_CXT.msgpack_true = NULL; @@ -52,11 +53,17 @@ static SV* load_bool(pTHX_ const char* const name) { CV* const cv = get_cv(name, GV_ADD); dSP; + ENTER; + SAVETMPS; PUSHMARK(SP); call_sv((SV*)cv, G_SCALAR); SPAGAIN; SV* const sv = newSVsv(POPs); PUTBACK; + FREETMPS; + LEAVE; + assert(sv); + assert(sv_isobject(sv)); return sv; } From 68b6fa46e61490c3f5fdca71b3e1dc90ebb4a02d Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:28:14 +0900 Subject: [PATCH 48/50] perl: add tests for boolean values --- perl/t/13_booleans.t | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 perl/t/13_booleans.t diff --git a/perl/t/13_booleans.t b/perl/t/13_booleans.t new file mode 100755 index 00000000..1ecbe646 --- /dev/null +++ b/perl/t/13_booleans.t @@ -0,0 +1,12 @@ +#!perl -w +use strict; +use Test::More tests => 6; +use Data::MessagePack; + +ok defined(Data::MessagePack::true()), 'true (1)'; +ok defined(Data::MessagePack::true()), 'true (2)'; +ok Data::MessagePack::true(), 'true is true'; + +ok defined(Data::MessagePack::false()), 'false (1)'; +ok defined(Data::MessagePack::false()), 'false (2)'; +ok !Data::MessagePack::false(), 'false is false'; From 0a8a6ed168dfd9338c2e97fd6b87b88ef39644ef Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 15:59:21 +0900 Subject: [PATCH 49/50] perl: cleanup PP --- perl/lib/Data/MessagePack/PP.pm | 54 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 8234b493..5dccc0bb 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -1,20 +1,17 @@ package Data::MessagePack::PP; use 5.008001; use strict; +use warnings; +no warnings 'recursion'; + use Carp (); +use B (); # See also # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack - -package - Data::MessagePack; - -use strict; -use B (); - BEGIN { my $unpack_int64_slow; my $unpack_uint64_slow; @@ -120,6 +117,18 @@ BEGIN { *unpack_int64 = $unpack_int64_slow || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } + + # fixin package symbols + no warnings 'once'; + sub pack :method; + sub unpack :method; + *Data::MessagePack::pack = \&pack; + *Data::MessagePack::unpack = \&unpack; + + @Data::MessagePack::Unpacker::ISA = qw(Data::MessagePack::PP::Unpacker); + + *true = \&Data::MessagePack::true; + *false = \&Data::MessagePack::false; } sub _unexpected { @@ -130,10 +139,7 @@ sub _unexpected { # PACK # -{ - no warnings 'recursion'; - - our $_max_depth; +our $_max_depth; sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; @@ -238,20 +244,19 @@ sub _pack { } -} # PACK - - # # UNPACK # -{ - - my $p; # position variables for speed. +my $p; # position variables for speed. sub unpack :method { $p = 0; # init - _unpack( $_[1] ); + my $data = _unpack( $_[1] ); + if($p < length($_[1])) { + Carp::croak("Data::MessagePack->unpack: extra bytes"); + } + return $data; } @@ -383,17 +388,12 @@ sub _unpack { } -} # UNPACK - - # # Data::MessagePack::Unpacker # package - Data::MessagePack::Unpacker; - -use strict; + Data::MessagePack::PP::Unpacker; sub new { bless { pos => 0 }, shift; @@ -404,10 +404,6 @@ sub execute_limit { execute( @_ ); } - -{ - my $p; - sub execute { my ( $self, $data, $offset, $limit ) = @_; $offset ||= 0; @@ -542,8 +538,6 @@ sub _count { return 0; } -} # execute - sub data { return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) ); From 3d905a7a4fe437433dd0746d9864928ec8f464b9 Mon Sep 17 00:00:00 2001 From: gfx Date: Wed, 22 Sep 2010 16:14:55 +0900 Subject: [PATCH 50/50] perl: add tests for 'extra bytes' exceptions --- perl/t/14_invalid_data.t | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 perl/t/14_invalid_data.t diff --git a/perl/t/14_invalid_data.t b/perl/t/14_invalid_data.t new file mode 100755 index 00000000..f5344857 --- /dev/null +++ b/perl/t/14_invalid_data.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my $nil = Data::MessagePack->pack(undef); + +my @data = do 't/data.pl'; +while(my($dump, $data) = splice @data, 0, 2) { + my $s = Data::MessagePack->pack($data); + eval { + Data::MessagePack->unpack($s . $nil); + }; + like $@, qr/extra bytes/, "dump $dump"; +} + +done_testing;