diff --git a/java/pom.xml b/java/pom.xml index 44806d51..70da6a6f 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,6 +29,12 @@ 4.8.1 test + + javassist + javassist + 3.12.1.GA + compile + @@ -101,6 +107,13 @@ MessagePack Maven2 Repository http://msgpack.org/maven2 + + repository.jboss.org + https://repository.jboss.org/nexus/content/groups/public/ + + false + + diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java new file mode 100644 index 00000000..a565292d --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackOptional { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java new file mode 100644 index 00000000..03e50cf3 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.CLASS) +public @interface MessagePackRequired { + int value() default -1; +} \ No newline at end of file diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java new file mode 100644 index 00000000..77f6e6de --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java @@ -0,0 +1,11 @@ +package org.msgpack.util.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface MessagePackUnpackable { +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java new file mode 100644 index 00000000..52384448 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java @@ -0,0 +1,794 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; +import javassist.NotFoundException; + +import org.msgpack.MessageConvertable; +import org.msgpack.MessagePackable; +import org.msgpack.MessageTypeException; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class PackUnpackUtil { + static class Constants { + static final String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced"; + + static final String KEYWORD_MODIFIER_PUBLIC = "public"; + + static final String KEYWORD_FOR = "for"; + + static final String KEYWORD_THROWS = "throws"; + + static final String KEYWORD_NEW = "new"; + + static final String TYPE_NAME_VOID = void.class.getName(); + + static final String TYPE_NAME_OBJECT = Object.class.getName(); + + static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName(); + + static final String TYPE_NAME_PACKER = Packer.class.getName(); + + static final String TYPE_NAME_UNPACKER = Unpacker.class.getName(); + + static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class + .getName(); + + static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class + .getName(); + + static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class + .getName(); + + static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class + .getName(); + + static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class + .getName(); + + static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class + .getName(); + + static final String CHAR_NAME_SPACE = " "; + + static final String CHAR_NAME_COMMA = ","; + + static final String CHAR_NAME_EQUAL = "="; + + static final String CHAR_NAME_PLUS = "+"; + + static final String CHAR_NAME_LESSTHAN = "<"; + + static final String CHAR_NAME_RIGHT_PARENTHESIS = ")"; + + static final String CHAR_NAME_LEFT_PARENTHESIS = "("; + + static final String CHAR_NAME_RIGHT_CURLY_BRACHET = "}"; + + static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{"; + + static final String CHAR_NAME_DOT = "."; + + static final String CHAR_NAME_SEMICOLON = ";"; + + static final String VARIABLE_NAME_PK = "pk"; + + static final String VARIABLE_NAME_OBJ = "obj"; + + static final String VARIABLE_NAME_SIZE = "len"; + + static final String VARIABLE_NAME_I = "i"; + + static final String METHOD_NAME_VALUEOF = "valueOf"; + + static final String METHOD_NAME_ADD = "add"; + + static final String METHOD_NAME_PUT = "put"; + + static final String METHOD_NAME_MSGPACK = "messagePack"; + + static final String METHOD_NAME_MSGUNPACK = "messageUnpack"; + + static final String METHOD_NAME_MSGCONVERT = "messageConvert"; + + static final String METHOD_NAME_PACK = "pack"; + + static final String METHOD_NAME_PACKARRAY = "packArray"; + + static final String METHOD_NAME_UNPACK = "unpack"; + + static final String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean"; + + static final String METHOD_NAME_UNPACKBYTE = "unpackByte"; + + static final String METHOD_NAME_UNPACKDOUBLE = "unpackDouble"; + + static final String METHOD_NAME_UNPACKFLOAT = "unpackFloat"; + + static final String METHOD_NAME_UNPACKINT = "unpackInt"; + + static final String METHOD_NAME_UNPACKLONG = "unpackLong"; + + static final String METHOD_NAME_UNPACKSHORT = "unpackShort"; + + static final String METHOD_NAME_UNPACKSTRING = "unpackString"; + + static final String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger"; + + static final String METHOD_NAME_UNPACKOBJECT = "unpackObject"; + + static final String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray"; + + static final String METHOD_NAME_UNPACKARRAY = "unpackArray"; + + static final String METHOD_NAME_UNPACKMAP = "unpackMap"; + } + + public static class Enhancer { + + private ConcurrentHashMap> classCache; + + private ClassPool pool; + + protected Enhancer() { + classCache = new ConcurrentHashMap>(); + pool = ClassPool.getDefault(); + } + + protected Class getCache(String origName) { + return classCache.get(origName); + } + + protected void setCache(String origName, Class enhClass) { + classCache.putIfAbsent(origName, enhClass); + } + + protected Class generate(Class origClass) + throws NotFoundException, CannotCompileException { + String origName = origClass.getName(); + String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER; + CtClass origCtClass = pool.get(origName); + checkClassValidation(origClass); + checkDefaultConstructorValidation(origClass); + CtClass enhCtClass = pool.makeClass(enhName); + setSuperclass(enhCtClass, origCtClass); + setInterfaces(enhCtClass); + addConstructor(enhCtClass); + Field[] fields = getDeclaredFields(origClass); + addMessagePackMethod(enhCtClass, origCtClass, fields); + addMessageUnpackMethod(enhCtClass, origCtClass, fields); + addMessageConvertMethod(enhCtClass, origCtClass, fields); + return createClass(enhCtClass); + } + + private void checkClassValidation(Class origClass) { + // not public, abstract, final + int mod = origClass.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isAbstract(mod) || Modifier.isFinal(mod)) { + throwClassValidationException(origClass); + } + // interface, enum + if (origClass.isInterface() || origClass.isEnum()) { + throwClassValidationException(origClass); + } + // annotation + checkPackUnpackAnnotation(origClass); + } + + private static void throwClassValidationException(Class origClass) { + throw new PackUnpackUtilException( + "it must be a public class and have @" + + MessagePackUnpackable.class.getName() + ": " + + origClass.getName()); + } + + private void checkPackUnpackAnnotation(Class origClass) { + Annotation anno = origClass + .getAnnotation(MessagePackUnpackable.class); + if (anno == null) { + throwClassValidationException(origClass); + } + } + + private void checkDefaultConstructorValidation(Class origClass) { + Constructor cons = null; + try { + cons = origClass.getDeclaredConstructor(new Class[0]); + } catch (Exception e1) { + throwConstructoValidationException(origClass); + } + + int mod = cons.getModifiers(); + if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) { + throwConstructoValidationException(origClass); + } + } + + private static void throwConstructoValidationException( + Class origClass) { + throw new PackUnpackUtilException( + "it must have a public zero-argument constructor: " + + origClass.getName()); + } + + private void setSuperclass(CtClass enhCtClass, CtClass origCtClass) + throws CannotCompileException { + enhCtClass.setSuperclass(origCtClass); + } + + private void setInterfaces(CtClass enhCtClass) throws NotFoundException { + CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE); + enhCtClass.addInterface(pacCtClass); + CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE); + enhCtClass.addInterface(unpacCtClass); + CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE); + enhCtClass.addInterface(convCtClass); + } + + private void addConstructor(CtClass enhCtClass) + throws CannotCompileException { + CtConstructor newCtCons = CtNewConstructor + .defaultConstructor(enhCtClass); + enhCtClass.addConstructor(newCtCons); + } + + private Field[] getDeclaredFields(Class origClass) { + ArrayList allFields = new ArrayList(); + Class nextClass = origClass; + while (nextClass != Object.class) { + Field[] fields = nextClass.getDeclaredFields(); + for (Field field : fields) { + try { + checkFieldValidation(field, allFields); + allFields.add(field); + } catch (PackUnpackUtilException e) { // ignore + } + } + nextClass = nextClass.getSuperclass(); + } + return allFields.toArray(new Field[0]); + } + + private void checkFieldValidation(Field field, List fields) { + // check modifiers (public or protected) + int mod = field.getModifiers(); + if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) + || Modifier.isStatic(mod) || Modifier.isFinal(mod) + || Modifier.isTransient(mod) || field.isSynthetic()) { + throwFieldValidationException(field); + } + // check same name + for (Field f : fields) { + if (f.getName().equals(field.getName())) { + throwFieldValidationException(field); + } + } + } + + private static void throwFieldValidationException(Field field) { + throw new PackUnpackUtilException("it must be a public field: " + + field.getName()); + } + + private void addMessagePackMethod(CtClass enhCtClass, + CtClass origCtClass, Field[] fields) + throws CannotCompileException, NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_PACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(fields.length); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessagePack(sb, field); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + //System.out.println("messagePack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessagePack(StringBuilder sb, Field field) { + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void addMessageUnpackMethod(CtClass enhCtClass, + CtClass origCtClass, Field[] fields) + throws CannotCompileException, NotFoundException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_VOID); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.METHOD_NAME_MSGUNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.TYPE_NAME_UNPACKER); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_THROWS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.TYPE_NAME_IOEXCEPTION); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + for (Field field : fields) { + insertCodeOfMessageUnpack(sb, field, field.getType()); + } + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + //System.out.println("messageUnpack method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private void insertCodeOfMessageUnpack(StringBuilder sb, Field field, + Class type) throws NotFoundException { + if (type.isPrimitive()) { + // primitive type + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (type.equals(Boolean.class) || // Boolean + type.equals(Byte.class) || // Byte + type.equals(Double.class) || // Double + type.equals(Float.class) || // Float + type.equals(Integer.class) || // Integer + type.equals(Long.class) || // Long + type.equals(Short.class)) { // Short + // reference type (wrapper type) + insertCodeOfMessageUnpackForWrapperTypes(sb, field, type); + } else if (type.equals(BigInteger.class) || // BigInteger + type.equals(String.class) || // String + type.equals(byte[].class)) { // byte[] + // reference type (other type) + insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type); + } else if (List.class.isAssignableFrom(type)) { + // List + insertCodeOfMessageUnpackForListType(sb, field, type); + } else if (Map.class.isAssignableFrom(type)) { + // Map + insertCodeOfMessageUnpackForMapType(sb, field, type); + } else if (MessageUnpackable.class.isAssignableFrom(type) + || (getCache(type.getName()) != null)) { + // MessageUnpackable + insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type); + } else { + throw new NotFoundException("unknown type: " + type.getName()); + } + } + + private void insertCodeOfMessageUnpackForPrimitiveTypes( + StringBuilder sb, Field field, Class type) + throws NotFoundException { + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert unpack method + if (type.equals(boolean.class)) { // boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(byte.class)) { // byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(double.class)) { // double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(float.class)) { // float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(int.class)) { // int + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(long.class)) { // long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(short.class)) { // short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { // reference type + if (type.equals(BigInteger.class)) { // BigInteger + sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER); + } else if (type.equals(String.class)) { // String + sb.append(Constants.METHOD_NAME_UNPACKSTRING); + } else if (type.equals(byte[].class)) { // byte[] + sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY); + } else { + throw new NotFoundException("unknown type: " + + type.getName()); + } + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + + private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb, + Field field, Class type) throws NotFoundException { + // insert a right variable + if (field != null) { + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + } + sb.append(type.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_VALUEOF); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + // insert the name of a unpack method + if (type.equals(Boolean.class)) { // Boolean + sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN); + } else if (type.equals(Byte.class)) { // Byte + sb.append(Constants.METHOD_NAME_UNPACKBYTE); + } else if (type.equals(Double.class)) { // Double + sb.append(Constants.METHOD_NAME_UNPACKDOUBLE); + } else if (type.equals(Float.class)) { // Float + sb.append(Constants.METHOD_NAME_UNPACKFLOAT); + } else if (type.equals(Integer.class)) { // Integer + sb.append(Constants.METHOD_NAME_UNPACKINT); + } else if (type.equals(Long.class)) { // Long + sb.append(Constants.METHOD_NAME_UNPACKLONG); + } else if (type.equals(Short.class)) { // Short + sb.append(Constants.METHOD_NAME_UNPACKSHORT); + } else { + throw new NotFoundException("unknown type: " + type.getName()); + } + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + if (field != null) { + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + } + + private void insertCodeOfMessageUnpackForListType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType = (Class) generic.getActualTypeArguments()[0]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKARRAY); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(ArrayList.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_ADD); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMapType(StringBuilder sb, + Field field, Class type) throws NotFoundException { + ParameterizedType generic = (ParameterizedType) field + .getGenericType(); + Class genericType0 = (Class) generic.getActualTypeArguments()[0]; + Class genericType1 = (Class) generic.getActualTypeArguments()[1]; + + // len + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACKMAP); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // field initializer + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.KEYWORD_NEW); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(HashMap.class.getName()); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + // for loop + sb.append(Constants.KEYWORD_FOR); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(int.class.getName()); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_EQUAL); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(0); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LESSTHAN); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.VARIABLE_NAME_SIZE); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.CHAR_NAME_PLUS); + sb.append(Constants.VARIABLE_NAME_I); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SPACE); + sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + + // block map. + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_PUT); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + insertCodeOfMessageUnpack(sb, null, genericType0); + sb.append(Constants.CHAR_NAME_COMMA); + sb.append(Constants.CHAR_NAME_SPACE); + insertCodeOfMessageUnpack(sb, null, genericType1); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void insertCodeOfMessageUnpackForMsgUnpackableType( + StringBuilder sb, Field field, Class type) { + // insert a right variable // ignore + sb.append(Constants.VARIABLE_NAME_PK); + sb.append(Constants.CHAR_NAME_DOT); + sb.append(Constants.METHOD_NAME_UNPACK); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS); + sb.append(MessageUnpackable.class.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(field.getName()); + sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS); + sb.append(Constants.CHAR_NAME_SEMICOLON); + sb.append(Constants.CHAR_NAME_SPACE); + } + + private void addMessageConvertMethod(CtClass enhCtClass, + CtClass origCtClass, Field[] fields) + throws CannotCompileException { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append( + Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID) + .append(Constants.CHAR_NAME_SPACE).append( + Constants.METHOD_NAME_MSGCONVERT).append( + Constants.CHAR_NAME_LEFT_PARENTHESIS).append( + Constants.TYPE_NAME_OBJECT).append( + Constants.CHAR_NAME_SPACE).append( + Constants.VARIABLE_NAME_OBJ).append( + Constants.CHAR_NAME_RIGHT_PARENTHESIS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.KEYWORD_THROWS).append( + Constants.CHAR_NAME_SPACE).append( + Constants.TYPE_NAME_MSGTYPEEXCEPTION).append( + Constants.CHAR_NAME_SPACE).append( + Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append( + Constants.CHAR_NAME_SPACE); + // TODO + sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET); + //System.out.println("messageConvert method: " + sb.toString()); + CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass); + enhCtClass.addMethod(newCtMethod); + } + + private Class createClass(CtClass enhCtClass) + throws CannotCompileException { + return enhCtClass.toClass(null, null); + } + } + + private static Enhancer enhancer; + + public static Class getEnhancedClass(Class origClass) { + if (enhancer == null) { + enhancer = new Enhancer(); + } + + String origName = origClass.getName(); + Class enhClass = enhancer.getCache(origName); + if (enhClass == null) { + // generate a class object related to the original class + try { + enhClass = enhancer.generate(origClass); + } catch (NotFoundException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (CannotCompileException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + // set the generated class to the cache + enhancer.setCache(origName, enhClass); + } + return enhClass; + } + + public static Object newEnhancedInstance(Class origClass) { + try { + Class enhClass = getEnhancedClass(origClass); + // create a new object of the generated class + return enhClass.newInstance(); + } catch (InstantiationException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new PackUnpackUtilException(e.getMessage(), e); + } + } + + @MessagePackUnpackable + public static class Image { + public String uri = ""; + + public String title = ""; + + public int width = 0; + + public int height = 0; + + public int size = 0; + + public boolean equals(Image obj) { + return uri.equals(obj.uri) && title.equals(obj.title) + && width == obj.width && height == obj.height + && size == obj.size; + } + + public boolean equals(Object obj) { + if (obj.getClass() != Image.class) { + return false; + } + return equals((Image) obj); + } + } + + public static void main(final String[] args) throws Exception { + PackUnpackUtil.getEnhancedClass(Image.class); + Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + src.title = "msgpack"; + src.uri = "http://msgpack.org/"; + src.width = 2560; + src.height = 1600; + src.size = 4096000; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + Image dst = (Image) PackUnpackUtil.newEnhancedInstance(Image.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + //System.out.println(src.equals(dst)); + } +} diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java new file mode 100644 index 00000000..df3af058 --- /dev/null +++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java @@ -0,0 +1,12 @@ +package org.msgpack.util.annotation; + +public class PackUnpackUtilException extends RuntimeException { + + public PackUnpackUtilException(String reason) { + super(reason); + } + + public PackUnpackUtilException(String reason, Throwable t) { + super(reason, t); + } +} diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java new file mode 100644 index 00000000..dbea7cb2 --- /dev/null +++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java @@ -0,0 +1,478 @@ +package org.msgpack.util.annotation; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.msgpack.MessageUnpackable; +import org.msgpack.Packer; +import org.msgpack.Unpacker; + +public class TestMessagePackUnpackable extends TestCase { + + @Test + public void testPrimitiveTypeFields() throws Exception { + PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); + src.f0 = (byte) 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f6 = false; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(PrimitiveTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + } + + @MessagePackUnpackable + public static class PrimitiveTypeFieldsClass { + public byte f0; + public short f1; + public int f2; + public long f3; + public float f4; + public double f5; + public boolean f6; + + public PrimitiveTypeFieldsClass() { + } + } + + @Test + public void testGeneralReferenceTypeFieldsClass() throws Exception { + GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + src.f0 = 0; + src.f1 = 1; + src.f2 = 2; + src.f3 = (long) 3; + src.f4 = (float) 4; + src.f5 = (double) 5; + src.f6 = false; + src.f7 = new BigInteger("7"); + src.f8 = "8"; + src.f9 = new byte[] { 0x01, 0x02 }; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil + .newEnhancedInstance(GeneralReferenceTypeFieldsClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0, dst.f0); + assertEquals(src.f1, dst.f1); + assertEquals(src.f2, dst.f2); + assertEquals(src.f3, dst.f3); + assertEquals(src.f4, dst.f4); + assertEquals(src.f5, dst.f5); + assertEquals(src.f6, dst.f6); + assertEquals(src.f7, dst.f7); + assertEquals(src.f8, dst.f8); + assertEquals(src.f9[0], dst.f9[0]); + assertEquals(src.f9[1], dst.f9[1]); + } + + @MessagePackUnpackable + public static class GeneralReferenceTypeFieldsClass { + public Byte f0; + public Short f1; + public Integer f2; + public Long f3; + public Float f4; + public Double f5; + public Boolean f6; + public BigInteger f7; + public String f8; + public byte[] f9; + + public GeneralReferenceTypeFieldsClass() { + } + } + + public void testListTypes() throws Exception { + SampleListTypes src = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + src.f0 = new ArrayList(); + src.f1 = new ArrayList(); + src.f1.add(1); + src.f1.add(2); + src.f1.add(3); + src.f2 = new ArrayList(); + src.f2.add("e1"); + src.f2.add("e2"); + src.f2.add("e3"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleListTypes dst = (SampleListTypes) PackUnpackUtil + .newEnhancedInstance(SampleListTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + for (int i = 0; i < src.f1.size(); ++i) { + assertEquals(src.f1.get(i), dst.f1.get(i)); + } + assertEquals(src.f2.size(), dst.f2.size()); + for (int i = 0; i < src.f2.size(); ++i) { + assertEquals(src.f2.get(i), dst.f2.get(i)); + } + } + + @MessagePackUnpackable + public static class SampleListTypes { + public List f0; + public List f1; + public List f2; + + public SampleListTypes() { + } + } + + public void testMapTypes() throws Exception { + SampleMapTypes src = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + src.f0 = new HashMap(); + src.f1 = new HashMap(); + src.f1.put(1, 1); + src.f1.put(2, 2); + src.f1.put(3, 3); + src.f2 = new HashMap(); + src.f2.put("k1", 1); + src.f2.put("k2", 2); + src.f2.put("k3", 3); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil + .newEnhancedInstance(SampleMapTypes.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertEquals(src.f0.size(), dst.f0.size()); + assertEquals(src.f1.size(), dst.f1.size()); + Iterator srcf1 = src.f1.keySet().iterator(); + Iterator dstf1 = dst.f1.keySet().iterator(); + while (srcf1.hasNext()) { + Integer s1 = srcf1.next(); + Integer d1 = dstf1.next(); + assertEquals(s1, d1); + assertEquals(src.f1.get(s1), dst.f1.get(d1)); + } + assertEquals(src.f2.size(), dst.f2.size()); + Iterator srcf2 = src.f2.keySet().iterator(); + Iterator dstf2 = dst.f2.keySet().iterator(); + while (srcf2.hasNext()) { + String s2 = srcf2.next(); + String d2 = dstf2.next(); + assertEquals(s2, d2); + assertEquals(src.f2.get(s2), dst.f2.get(d2)); + } + } + + @MessagePackUnpackable + public static class SampleMapTypes { + public Map f0; + public Map f1; + public Map f2; + + public SampleMapTypes() { + } + } + + @Test + public void testDefaultConstructorModifiers() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil + .newEnhancedInstance(PrivateDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil + .newEnhancedInstance(ProtectedDefaultConstructorClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil + .newEnhancedInstance(PackageDefaultConstructorClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public static class NoDefaultConstructorClass { + public NoDefaultConstructorClass(int i) { + } + } + + @MessagePackUnpackable + public static class PrivateDefaultConstructorClass { + private PrivateDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class ProtectedDefaultConstructorClass { + protected ProtectedDefaultConstructorClass() { + } + } + + @MessagePackUnpackable + public static class PackageDefaultConstructorClass { + PackageDefaultConstructorClass() { + } + } + + @Test + public void testClassModifiers() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class); + assertTrue(true); + } catch (PackUnpackUtilException e) { + fail(); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + private static class PrivateModifierClass { + } + + @MessagePackUnpackable + protected static class ProtectedModifierClass { + protected ProtectedModifierClass() { + } + } + + @MessagePackUnpackable + static class PackageModifierClass { + } + + @Test + public void testFinalClassAndAbstractClass() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public final static class FinalModifierClass { + } + + @MessagePackUnpackable + public abstract static class AbstractModifierClass { + } + + @Test + public void testInterfaceAndEnumType() throws Exception { + try { + PackUnpackUtil.newEnhancedInstance(SampleInterface.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + try { + PackUnpackUtil.newEnhancedInstance(SampleEnum.class); + fail(); + } catch (PackUnpackUtilException e) { + assertTrue(true); + } + assertTrue(true); + } + + @MessagePackUnpackable + public interface SampleInterface { + } + + @MessagePackUnpackable + public enum SampleEnum { + } + + @Test + public void testFieldModifiers() throws Exception { + FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil + .newEnhancedInstance(FieldModifiersClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + } + + @MessagePackUnpackable + public static class FieldModifiersClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + + public FieldModifiersClass() { + } + } + + @Test + public void testNestedAnnotatedFieldClass() throws Exception { + NestedClass src2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass src = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + src.f0 = 0; + src2.f2 = 2; + src.f1 = src2; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + NestedClass dst2 = (NestedClass) PackUnpackUtil + .newEnhancedInstance(NestedClass.class); + BaseClass dst = (BaseClass) PackUnpackUtil + .newEnhancedInstance(BaseClass.class); + dst.f1 = dst2; + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src2.f2 == dst.f1.f2); + } + + @MessagePackUnpackable + public static class BaseClass { + public int f0; + public NestedClass f1; + + public BaseClass() { + } + } + + @MessagePackUnpackable + public static class NestedClass { + public int f2; + + public NestedClass() { + } + } + + @Test + public void testExtendedClass() throws Exception { + SampleSubClass src = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + src.f0 = 0; + src.f2 = 2; + src.f3 = 3; + src.f4 = 4; + src.f5 = 5; + src.f8 = 8; + src.f9 = 9; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(src); + SampleSubClass dst = (SampleSubClass) PackUnpackUtil + .newEnhancedInstance(SampleSubClass.class); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker pac = new Unpacker(in); + pac.unpack((MessageUnpackable) dst); + assertTrue(src.f0 == dst.f0); + assertTrue(src.f1 == dst.f1); + assertTrue(src.f2 != dst.f2); + assertTrue(src.f3 == dst.f3); + assertTrue(src.f4 != dst.f4); + assertTrue(src.f5 == dst.f5); + assertTrue(src.f6 == dst.f6); + assertTrue(src.f8 == dst.f8); + assertTrue(src.f9 != dst.f9); + } + + @MessagePackUnpackable + public static class SampleSubClass extends SampleSuperClass { + public int f0; + public final int f1 = 1; + private int f2; + protected int f3; + int f4; + + public SampleSubClass() { + } + } + + public static class SampleSuperClass { + public int f5; + public final int f6 = 2; + private int f7; + protected int f8; + int f9; + + public SampleSuperClass() { + } + } +} diff --git a/perl/.gitignore b/perl/.gitignore index 3e0e73e5..16568472 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -1,4 +1,5 @@ META.yml +MYMETA.yml Makefile Makefile.old MessagePack.bs diff --git a/perl/Changes b/perl/Changes index ce525818..b717a828 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,7 +1,16 @@ +0.26 + + - fixed a serious code typo in PP(makamaka) + +0.25 + + (NO FEATURE CHANGES) + - oops. I failed releng. 0.24 - - Fixed a possible SEGV on streaming unpacking (gfx) - - Improve performance, esp. in unpacking (gfx) + - Fixed a lot of streaming unpacking issues (tokuhirom, gfx) + - Fixed unpacking issues for 64 bit integers on 32 bit perls (gfx) + - Improved performance, esp. in unpacking (gfx) 0.23 diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 71a24e5c..1d2192f5 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -2,6 +2,7 @@ \bCVS\b ^MANIFEST\. ^Makefile$ +^MYMETA\.yml$ ~$ ^# \.old$ diff --git a/perl/Makefile.PL b/perl/Makefile.PL index fafc3876..219400f2 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,3 +1,5 @@ +# Usage: Makefile.PL --pp # disable XS +# Makefile.PL -g # add -g to the compiler and disable optimization flags use inc::Module::Install; use Module::Install::XSUtil 0.32; use Config; @@ -14,15 +16,16 @@ recursive_author_tests('xt'); if ( $] >= 5.008005 and want_xs() ) { - can_cc or die "This module requires a C compiler. Please retry with --pp"; - my $has_c99 = c99_available(); # msgpack C library requires C99. if ( $has_c99 ) { + requires_c99(); use_xshelper(); + cc_warnings; cc_src_paths('xs-src'); - if ($ENV{DEBUG}) { - cc_append_to_ccflags '-g'; + + if($Module::Install::AUTHOR) { + postamble qq{test :: test_pp\n\n}; } } else { @@ -37,6 +40,7 @@ NOT_SUPPORT_C99 } else { print "configure PP version\n\n"; + requires 'Math::BigInt' => 1.89; # old versions of BigInt were broken } clean_files qw{ @@ -66,10 +70,6 @@ test_requires('Test::Requires'); test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' ); -if($Module::Install::AUTHOR) { - postamble qq{test :: test_pp\n\n}; -} - repository('http://github.com/msgpack/msgpack'); auto_include; WriteAll; diff --git a/perl/README b/perl/README index e46323da..224ff08c 100644 --- a/perl/README +++ b/perl/README @@ -2,7 +2,9 @@ NAME Data::MessagePack - MessagePack serialising/deserialising SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + use Data::MessagePack; + + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); DESCRIPTION @@ -14,10 +16,10 @@ ABOUT MESSAGEPACK FORMAT But unlike JSON, it is very fast and small. ADVANTAGES - PORTABILITY - Messagepack is language independent binary serialize format. + PORTABLE + The MessagePack format does not depend on language nor byte order. - SMALL SIZE + SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7 @@ -26,7 +28,7 @@ ABOUT MESSAGEPACK FORMAT STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for - networking such as RPC. + networking such as RPC. See Data::MessagePack::Unpacker for details. If you want to get more information about the MessagePack format, please visit to . @@ -47,36 +49,66 @@ METHODS Configuration Variables $Data::MessagePack::PreferInteger - Pack the string as int when the value looks like int(EXPERIMENTAL). + Packs a string as an integer, when it looks like an integer. SPEED - This is the result of benchmark/serialize.pl and - benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu - SMP). + This is a result of benchmark/serialize.pl and benchmark/deserialize.pl + on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). (You should + benchmark them with your data if the speed matters, of course.) -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- + +CAVEAT + Unpacking 64 bit integers + This module can unpack 64 bit integers even if your perl does not + support them (i.e. where "perl -V:ivsize" is 4), but you cannot + calculate these values unless you use "Math::BigInt". + +TODO + Error handling + MessagePack cannot deal with complex scalars such as object + references, filehandles, and code references. We should report the + errors more kindly. + + Streaming deserializer + The current implementation of the streaming deserializer does not + have internal buffers while some other bindings (such as Ruby + binding) does. This limitation will astonish those who try to unpack + byte streams with an arbitrary buffer size (e.g. + "while(read($socket, $buffer, $arbitrary_buffer_size)) { ... }"). We + should implement the internal buffer for the unpacker. AUTHORS Tokuhiro Matsuno Makamaka Hannyaharamitu + gfx + THANKS TO Jun Kuriyama @@ -91,5 +123,10 @@ LICENSE under the same terms as Perl itself. SEE ALSO - is official web site for MessagePack format. + is the official web site for the MessagePack + format. + + Data::MessagePack::Unpacker + + AnyEvent::MPRPC diff --git a/perl/benchmark/data.pl b/perl/benchmark/data.pl new file mode 100755 index 00000000..6908d1cc --- /dev/null +++ b/perl/benchmark/data.pl @@ -0,0 +1,6 @@ ++{ + "method" => "handleMessage", + "params" => [ "user1", "we were just talking", "foo\nbar\nbaz\nqux" ], + "id" => undef, + "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2), 1 .. 100 ], +}; diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl index 634a79ed..faa2582f 100644 --- a/perl/benchmark/deserialize.pl +++ b/perl/benchmark/deserialize.pl @@ -1,29 +1,25 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; -use Benchmark ':all'; +use JSON; use Storable; +use Benchmark ':all'; #$Data::MessagePack::PreferInteger = 1; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; -my $j = JSON::XS::encode_json($a); +my $a = do 'benchmark/data.pl'; + +my $j = JSON::encode_json($a); my $m = Data::MessagePack->pack($a); my $s = Storable::freeze($a); print "-- deserialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::decode_json($j) }, + json => sub { JSON::decode_json($j) }, mp => sub { Data::MessagePack->unpack($m) }, storable => sub { Storable::thaw($s) }, } diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl index e0509ffa..4982ff61 100644 --- a/perl/benchmark/serialize.pl +++ b/perl/benchmark/serialize.pl @@ -1,24 +1,19 @@ use strict; use warnings; use Data::MessagePack; -use JSON::XS; +use JSON; use Storable; use Benchmark ':all'; -my $a = { - "method" => "handleMessage", - "params" => [ "user1", "we were just talking" ], - "id" => undef, - "array" => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ], -}; +my $a = do 'benchmark/data.pl'; print "-- serialize\n"; -print "JSON::XS: $JSON::XS::VERSION\n"; +print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n"; print "Data::MessagePack: $Data::MessagePack::VERSION\n"; print "Storable: $Storable::VERSION\n"; cmpthese timethese( -1 => { - json => sub { JSON::XS::encode_json($a) }, + json => sub { JSON::encode_json($a) }, storable => sub { Storable::freeze($a) }, mp => sub { Data::MessagePack->pack($a) }, } diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b1e0174d..f6e68eb3 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -3,7 +3,7 @@ use strict; use warnings; use 5.008001; -our $VERSION = '0.23'; +our $VERSION = '0.26'; our $PreferInteger = 0; sub true () { @@ -23,7 +23,7 @@ sub false () { } if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate - my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || ''; + my $backend = $ENV{PERL_DATA_MESSAGEPACK} || ($ENV{PERL_ONLY} ? 'pp' : ''); if ( $backend !~ /\b pp \b/xms ) { eval { require XSLoader; @@ -45,7 +45,9 @@ Data::MessagePack - MessagePack serialising/deserialising =head1 SYNOPSIS - my $packed = Data::MessagePack->pack($dat); + use Data::MessagePack; + + my $packed = Data::MessagePack->pack($dat); my $unpacked = Data::MessagePack->unpack($dat); =head1 DESCRIPTION @@ -55,17 +57,18 @@ This module converts Perl data structures to MessagePack and vice versa. =head1 ABOUT MESSAGEPACK FORMAT MessagePack is a binary-based efficient object serialization format. -It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. +It enables to exchange structured objects between many languages like JSON. +But unlike JSON, it is very fast and small. =head2 ADVANTAGES =over 4 -=item PORTABILITY +=item PORTABLE -Messagepack is language independent binary serialize format. +The MessagePack format does not depend on language nor byte order. -=item SMALL SIZE +=item SMALL IN SIZE say length(JSON::XS::encode_json({a=>1, b=>2})); # => 13 say length(Storable::nfreeze({a=>1, b=>2})); # => 21 @@ -76,6 +79,7 @@ The MessagePack format saves memory than JSON and Storable format. =item STREAMING DESERIALIZER MessagePack supports streaming deserializer. It is useful for networking such as RPC. +See L for details. =back @@ -105,31 +109,67 @@ unpack the $msgpackstr to a MessagePack format string. =item $Data::MessagePack::PreferInteger -Pack the string as int when the value looks like int(EXPERIMENTAL). +Packs a string as an integer, when it looks like an integer. =back =head1 SPEED -This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). +This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). +(You should benchmark them with B data if the speed matters, of course.) -- serialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 5 wallclock secs ( 3.95 usr + 0.00 sys = 3.95 CPU) @ 253164.56/s (n=1000000) - mp: 3 wallclock secs ( 2.69 usr + 0.00 sys = 2.69 CPU) @ 371747.21/s (n=1000000) - storable: 26 wallclock secs (27.21 usr + 0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 1 wallclock secs ( 1.00 usr + 0.01 sys = 1.01 CPU) @ 141939.60/s (n=143359) + mp: 1 wallclock secs ( 1.06 usr + 0.00 sys = 1.06 CPU) @ 355500.94/s (n=376831) + storable: 1 wallclock secs ( 1.12 usr + 0.00 sys = 1.12 CPU) @ 38399.11/s (n=43007) + Rate storable json mp + storable 38399/s -- -73% -89% + json 141940/s 270% -- -60% + mp 355501/s 826% 150% -- -- deserialize JSON::XS: 2.3 - Data::MessagePack: 0.20 + Data::MessagePack: 0.24 Storable: 2.21 - Benchmark: timing 1000000 iterations of json, mp, storable... - json: 4 wallclock secs ( 4.45 usr + 0.00 sys = 4.45 CPU) @ 224719.10/s (n=1000000) - mp: 6 wallclock secs ( 5.45 usr + 0.00 sys = 5.45 CPU) @ 183486.24/s (n=1000000) - storable: 7 wallclock secs ( 7.77 usr + 0.00 sys = 7.77 CPU) @ 128700.13/s (n=1000000) + Benchmark: running json, mp, storable for at least 1 CPU seconds... + json: 0 wallclock secs ( 1.05 usr + 0.00 sys = 1.05 CPU) @ 179442.86/s (n=188415) + mp: 0 wallclock secs ( 1.01 usr + 0.00 sys = 1.01 CPU) @ 212909.90/s (n=215039) + storable: 2 wallclock secs ( 1.14 usr + 0.00 sys = 1.14 CPU) @ 114974.56/s (n=131071) + Rate storable json mp + storable 114975/s -- -36% -46% + json 179443/s 56% -- -16% + mp 212910/s 85% 19% -- + +=head1 CAVEAT + +=head2 Unpacking 64 bit integers + +This module can unpack 64 bit integers even if your perl does not support them +(i.e. where C<< perl -V:ivsize >> is 4), but you cannot calculate these values +unless you use C. + +=head1 TODO + +=over + +=item Error handling + +MessagePack cannot deal with complex scalars such as object references, +filehandles, and code references. We should report the errors more kindly. + +=item Streaming deserializer + +The current implementation of the streaming deserializer does not have internal +buffers while some other bindings (such as Ruby binding) does. This limitation +will astonish those who try to unpack byte streams with an arbitrary buffer size +(e.g. C<< while(read($socket, $buffer, $arbitrary_buffer_size)) { ... } >>). +We should implement the internal buffer for the unpacker. + +=back =head1 AUTHORS @@ -137,6 +177,8 @@ Tokuhiro Matsuno Makamaka Hannyaharamitu +gfx + =head1 THANKS TO Jun Kuriyama @@ -152,8 +194,12 @@ hanekomu This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. - =head1 SEE ALSO -L is official web site for MessagePack format. +L is the official web site for the MessagePack format. +L + +L + +=cut diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm index 6a06c3c6..5dccc0bb 100644 --- a/perl/lib/Data/MessagePack/PP.pm +++ b/perl/lib/Data/MessagePack/PP.pm @@ -1,28 +1,65 @@ package Data::MessagePack::PP; use 5.008001; use strict; +use warnings; +no warnings 'recursion'; + use Carp (); +use B (); # See also # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack - -package - Data::MessagePack; - -use Scalar::Util qw( blessed ); -use strict; -use B (); - BEGIN { + my $unpack_int64_slow; + my $unpack_uint64_slow; + + if(!eval { pack 'Q', 1 }) { # don't have quad types + $unpack_int64_slow = sub { + require Math::BigInt; + my $high = unpack_uint32( $_[0], $_[1] ); + my $low = unpack_uint32( $_[0], $_[1] + 4); + + if($high < 0xF0000000) { # positive + $high = Math::BigInt->new( $high ); + $low = Math::BigInt->new( $low ); + return +($high << 32 | $low)->bstr; + } + else { # negative + $high = Math::BigInt->new( ~$high ); + $low = Math::BigInt->new( ~$low ); + return +( -($high << 32 | $low + 1) )->bstr; + } + }; + $unpack_uint64_slow = sub { + require Math::BigInt; + my $high = Math::BigInt->new( unpack_uint32( $_[0], $_[1]) ); + my $low = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) ); + return +($high << 32 | $low)->bstr; + }; + } + + *unpack_uint16 = sub { return unpack 'n', substr( $_[0], $_[1], 2 ) }; + *unpack_uint32 = sub { return unpack 'N', substr( $_[0], $_[1], 4 ) }; + # for pack and unpack compatibility if ( $] < 5.010 ) { # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ ); # which better? my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE + *unpack_int16 = sub { + my $v = unpack 'n', substr( $_[0], $_[1], 2 ); + return $v ? $v - 0x10000 : 0; + }; + *unpack_int32 = sub { + no warnings; # avoid for warning about Hexadecimal number + my $v = unpack 'N', substr( $_[0], $_[1], 4 ); + return $v ? $v - 0x100000000 : 0; + }; + # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'? if($bo_is_le) { *pack_uint64 = sub { @@ -47,20 +84,11 @@ BEGIN { return unpack( 'd', pack( 'N2', @v[1,0] ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { + *unpack_int64 = $unpack_int64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'q', pack( 'N2', @v[1,0] ) ); }; - *unpack_uint64 = sub { + *unpack_uint64 = $unpack_uint64_slow || sub { my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) ); return unpack( 'Q', pack( 'N2', @v[1,0] ) ); }; @@ -72,17 +100,8 @@ BEGIN { *unpack_float = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); }; *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); }; - *unpack_int16 = sub { - my $v = unpack 'n', substr( $_[0], $_[1], 2 ); - return $v ? $v - 0x10000 : 0; - }; - *unpack_int32 = sub { - no warnings; # avoid for warning about Hexadecimal number - my $v = unpack 'N', substr( $_[0], $_[1], 4 ); - return $v ? $v - 0x100000000 : 0; - }; - *unpack_int64 = sub { pack 'q', substr( $_[0], $_[1], 8 ); }; - *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); }; + *unpack_int64 = $unpack_int64_slow || sub { unpack 'q', substr( $_[0], $_[1], 8 ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { unpack 'Q', substr( $_[0], $_[1], 8 ); }; } } else { @@ -94,24 +113,37 @@ BEGIN { *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); }; *unpack_int16 = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); }; *unpack_int32 = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); }; - *unpack_int64 = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; - *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; + + *unpack_int64 = $unpack_int64_slow || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); }; + *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); }; } + + # fixin package symbols + no warnings 'once'; + sub pack :method; + sub unpack :method; + *Data::MessagePack::pack = \&pack; + *Data::MessagePack::unpack = \&unpack; + + @Data::MessagePack::Unpacker::ISA = qw(Data::MessagePack::PP::Unpacker); + + *true = \&Data::MessagePack::true; + *false = \&Data::MessagePack::false; } +sub _unexpected { + Carp::confess("Unexpected " . sprintf(shift, @_) . " found"); +} # # PACK # -{ - no warnings 'recursion'; - - my $max_depth; +our $_max_depth; sub pack :method { Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2; - $max_depth = defined $_[2] ? $_[2] : 512; # init + $_max_depth = defined $_[2] ? $_[2] : 512; # init return _pack( $_[1] ); } @@ -119,6 +151,12 @@ sub pack :method { sub _pack { my ( $value ) = @_; + local $_max_depth = $_max_depth - 1; + + if ( $_max_depth < 0 ) { + Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); + } + return CORE::pack( 'C', 0xc0 ) if ( not defined $value ); if ( ref($value) eq 'ARRAY' ) { @@ -127,11 +165,8 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x90 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } @$value ); } @@ -141,11 +176,8 @@ sub _pack { $num < 16 ? CORE::pack( 'C', 0x80 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf, $num ) - : die "" # don't arrivie here + : _unexpected("number %d", $num) ; - if ( --$max_depth <= 0 ) { - Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)"); - } return join( '', $header, map { _pack( $_ ) } %$value ); } @@ -197,7 +229,7 @@ sub _pack { $num < 32 ? CORE::pack( 'C', 0xa0 + $num ) : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num ) : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num ) - : die "" # don't arrivie here + : _unexpected_number($num) ; return $header . $value; @@ -207,25 +239,24 @@ sub _pack { return pack_double( $value ); } else { - die "???"; + _unexpected("data type %s", $b_obj); } } -} # PACK - - # # UNPACK # -{ - - my $p; # position variables for speed. +my $p; # position variables for speed. sub unpack :method { $p = 0; # init - _unpack( $_[1] ); + my $data = _unpack( $_[1] ); + if($p < length($_[1])) { + Carp::croak("Data::MessagePack->unpack: extra bytes"); + } + return $data; } @@ -284,11 +315,11 @@ sub _unpack { } elsif ( $byte == 0xcd ) { # uint16 $p += 2; - return CORE::unpack 'n', substr( $value, $p - 2, 2 ); + return unpack_uint16( $value, $p - 2 ); } elsif ( $byte == 0xce ) { # unit32 $p += 4; - return CORE::unpack 'N', substr( $value, $p - 4, 4 ); + return unpack_uint32( $value, $p - 4 ); } elsif ( $byte == 0xcf ) { # unit64 $p += 8; @@ -351,26 +382,21 @@ sub _unpack { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } } -} # UNPACK - - # # Data::MessagePack::Unpacker # package - Data::MessagePack::Unpacker; - -use strict; + Data::MessagePack::PP::Unpacker; sub new { - bless { stack => [] }, shift; + bless { pos => 0 }, shift; } @@ -378,31 +404,32 @@ sub execute_limit { execute( @_ ); } - -{ - my $p; - sub execute { my ( $self, $data, $offset, $limit ) = @_; - my $value = substr( $data, $offset || 0, $limit ? $limit : length $data ); + $offset ||= 0; + my $value = substr( $data, $offset, $limit ? $limit : length $data ); my $len = length $value; + $self->{data} .= $value; + local $self->{stack} = []; + $p = 0; - while ( $len > $p ) { - _count( $self, $value ) or last; + while ( length($self->{data}) > $p ) { + _count( $self, $self->{data} ) or last; - if ( @{ $self->{stack} } > 0 ) { - pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0; + while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) { + pop @{ $self->{stack} }; + } + + if (@{$self->{stack}} == 0) { + $self->{is_finished}++; + last; } } + $self->{pos} = $p; - if ( $len == $p ) { - $self->{ data } .= substr( $value, 0, $p ); - $self->{ remain } = undef; - } - - return $p; + return $p + $offset; } @@ -424,7 +451,9 @@ sub _count { $num = $byte & ~0x90; } - push @{ $self->{stack} }, $num + 1; + if ( $num ) { + push @{ $self->{stack} }, $num + 1; + } return 1; } @@ -443,7 +472,9 @@ sub _count { $num = $byte & ~0x80; } - push @{ $self->{stack} }, $num * 2 + 1; # a pair + if ( $num ) { + push @{ $self->{stack} }, $num * 2 + 1; # a pair + } return 1; } @@ -461,7 +492,7 @@ sub _count { : $byte == 0xcd ? 2 : $byte == 0xce ? 4 : $byte == 0xcf ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -470,7 +501,7 @@ sub _count { : $byte == 0xd1 ? 2 : $byte == 0xd2 ? 4 : $byte == 0xd3 ? 8 - : die; + : _unexpected("byte 0x%02x", $byte); return 1; } @@ -501,32 +532,27 @@ sub _count { } else { - die "???"; + _unexpected("byte 0x%02x", $byte); } return 0; } -} # execute - sub data { - my $data = Data::MessagePack->unpack( $_[0]->{ data } ); - $_[0]->reset; - return $data; + return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) ); } sub is_finished { my ( $self ) = @_; - ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1; + return $self->{is_finished}; } - sub reset :method { - $_[0]->{ stack } = []; $_[0]->{ data } = undef; - $_[0]->{ remain } = undef; + $_[0]->{ pos } = 0; + $_[0]->{ is_finished } = 0; } 1; diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t index f91b29e7..d465f8dc 100644 --- a/perl/t/00_compile.t +++ b/perl/t/00_compile.t @@ -3,4 +3,5 @@ use warnings; use Test::More tests => 1; use_ok 'Data::MessagePack'; -diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' ); +diag ( "Testing Data::MessagePack/$Data::MessagePack::VERSION (", + $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS', ")" ); diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t index 4da69e91..657387a9 100644 --- a/perl/t/02_unpack.t +++ b/perl/t/02_unpack.t @@ -5,13 +5,14 @@ no warnings 'uninitialized'; # i need this. i need this. sub unpackit { my $v = $_[0]; - $v =~ s/ //g; + $v =~ s/ +//g; $v = pack 'H*', $v; return Data::MessagePack->unpack($v); } sub pis ($$) { - is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0] + or diag( explain(unpackit($_[0])) ); } my @dat = do 't/data.pl'; diff --git a/perl/t/03_stream_unpack.t b/perl/t/03_stream_unpack.t index a4ab4eba..646fc249 100644 --- a/perl/t/03_stream_unpack.t +++ b/perl/t/03_stream_unpack.t @@ -37,7 +37,7 @@ for (my $i=0; $iexecute("\xc0", 0); # nil } - ok $up->is_finished; - is_deeply $up->data, [undef, undef, undef, undef, undef]; + ok $up->is_finished, 'finished'; + is_deeply $up->data, [undef, undef, undef, undef, undef], 'array, is_deeply'; } diff --git a/perl/t/07_break.t b/perl/t/07_break.t index dd27ad25..cf3f1079 100644 --- a/perl/t/07_break.t +++ b/perl/t/07_break.t @@ -1,12 +1,23 @@ use Test::More; use Data::MessagePack; use t::Util; -no warnings 'uninitialized'; # i need this. i need this. -plan tests => 1; +plan tests => 4; -my $d = Data::MessagePack->unpack(Data::MessagePack->pack([{x => undef}])); -$d->[0]->{x} = 1; -ok delete $d->[0]->{x}; -$d->[0] = 4; +my $d = Data::MessagePack->unpack(Data::MessagePack->pack({ + nil => undef, + true => true, + false => false, + foo => [undef, true, false], +})); +$d->{nil} = 42; +is $d->{nil}, 42; + +$d->{true} = 43; +is $d->{true}, 43; + +$d->{false} = 44; +is $d->{false}, 44; + +is_deeply $d->{foo}, [undef, true, false]; diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t index a618787d..f98d696b 100644 --- a/perl/t/09_stddata.t +++ b/perl/t/09_stddata.t @@ -32,6 +32,7 @@ for my $mpac($mpac1, $mpac2) { my $i = 0; while($offset < length($mpac)) { $offset = $mps->execute($mpac, $offset); + ok $mps->is_finished, "data[$i] : is_finished"; is_deeply $mps->data, $data[$i], "data[$i]"; $mps->reset; $i++; diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t index 232d8707..15598f4e 100644 --- a/perl/t/10_splitted_bytes.t +++ b/perl/t/10_splitted_bytes.t @@ -27,12 +27,14 @@ foreach my $size(1 .. 16) { open my $stream, '<:bytes :scalar', \$packed; binmode $stream; my $buff; + my $done = 0; while( read($stream, $buff, $size) ) { #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff; - $up->execute($buff); + $done = $up->execute($buff); } - ok $up->is_finished, 'is_finished'; + is $done, length($packed); + ok $up->is_finished, "is_finished: $size"; my $data = $up->data; is_deeply $data, $input; } diff --git a/perl/t/11_stream_unpack3.t b/perl/t/11_stream_unpack3.t new file mode 100644 index 00000000..0eb8bff7 --- /dev/null +++ b/perl/t/11_stream_unpack3.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use Test::More; +use Data::MessagePack; + +my @data = ( [ 1, 2, 3 ], [ 4, 5, 6 ] ); + +# serialize +my $buffer = ''; +for my $d (@data) { + $buffer .= Data::MessagePack->pack($d); +} + +# deserialize +my $cb = sub { + my ($data) = @_; + + my $d = shift @data; + is_deeply $data, $d; +}; +my $unpacker = Data::MessagePack::Unpacker->new(); +my $nread = 0; +while (1) { + $nread = $unpacker->execute( $buffer, $nread ); + if ( $unpacker->is_finished ) { + my $ret = $unpacker->data; + $cb->( $ret ); + $unpacker->reset; + + $buffer = substr( $buffer, $nread ); + $nread = 0; + next if length($buffer) != 0; + } + last; +} +is scalar(@data), 0; + +done_testing; + diff --git a/perl/t/12_stream_unpack4.t b/perl/t/12_stream_unpack4.t new file mode 100644 index 00000000..de76e81d --- /dev/null +++ b/perl/t/12_stream_unpack4.t @@ -0,0 +1,24 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my @input = ( + +[[]], + [[],[]], + [{"a" => 97},{"a" => 97}], + [{"a" => 97},{"a" => 97},{"a" => 97}], + [ map { +{ "foo $_" => "bar $_" } } 'aa' .. 'zz' ], +); + +plan tests => @input * 2; + +for my $input (@input) { + my $packed = Data::MessagePack->pack($input); + my $up = Data::MessagePack::Unpacker->new(); + $up->execute($packed, 0); + ok $up->is_finished, 'finished'; + is_deeply($up->data, $input); +} + diff --git a/perl/t/13_booleans.t b/perl/t/13_booleans.t new file mode 100755 index 00000000..1ecbe646 --- /dev/null +++ b/perl/t/13_booleans.t @@ -0,0 +1,12 @@ +#!perl -w +use strict; +use Test::More tests => 6; +use Data::MessagePack; + +ok defined(Data::MessagePack::true()), 'true (1)'; +ok defined(Data::MessagePack::true()), 'true (2)'; +ok Data::MessagePack::true(), 'true is true'; + +ok defined(Data::MessagePack::false()), 'false (1)'; +ok defined(Data::MessagePack::false()), 'false (2)'; +ok !Data::MessagePack::false(), 'false is false'; diff --git a/perl/t/14_invalid_data.t b/perl/t/14_invalid_data.t new file mode 100755 index 00000000..f5344857 --- /dev/null +++ b/perl/t/14_invalid_data.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; +use t::Util; + +my $nil = Data::MessagePack->pack(undef); + +my @data = do 't/data.pl'; +while(my($dump, $data) = splice @data, 0, 2) { + my $s = Data::MessagePack->pack($data); + eval { + Data::MessagePack->unpack($s . $nil); + }; + like $@, qr/extra bytes/, "dump $dump"; +} + +done_testing; diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t index 29485270..440ac901 100644 --- a/perl/t/50_leaktrace.t +++ b/perl/t/50_leaktrace.t @@ -2,8 +2,12 @@ use strict; use Test::Requires { 'Test::LeakTrace' => 0.13 }; use Test::More; - use Data::MessagePack; +BEGIN { + if($INC{'Data/MessagePack/PP.pm'}) { + plan skip_all => 'disabled in PP'; + } +} my $simple_data = "xyz"; my $complex_data = { diff --git a/perl/t/data.pl b/perl/t/data.pl index 2f58d38c..9bf07b7d 100644 --- a/perl/t/data.pl +++ b/perl/t/data.pl @@ -4,15 +4,48 @@ no warnings; # i need this, i need this. '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], '92 90 91 91 c0', [[], [[undef]]], '93 c0 c2 c3', [undef, false, true], + + '82 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # fix map + 'de 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 16 + 'df 00 00 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 32 + 'ce 80 00 00 00', 2147483648, - '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], - '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], - '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], - '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', + [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', + ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, - '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', + [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], 'ce 00 ff ff ff' => ''.0xFFFFFF, 'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF, 'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF, + + 'd2 80 00 00 01' => '-2147483647', # int32_t + 'ce 80 00 00 01' => '2147483649', # uint32_t + + 'd2 ff ff ff ff' => '-1', # int32_t + 'ce ff ff ff ff' => '4294967295', # uint32_t + + 'd3 00 00 00 00 80 00 00 01' => '2147483649', # int64_t + 'cf 00 00 00 00 80 00 00 01' => '2147483649', # uint64_t + + 'd3 ff 00 ff ff ff ff ff ff' => '-71776119061217281', # int64_t + 'cf ff 00 ff ff ff ff ff ff' => '18374967954648334335', # uint64_t + + 'd3 ff ff ff ff ff ff ff ff' => '-1', # int64_t + 'cf ff ff ff ff ff ff ff ff' => '18446744073709551615', # uint64_t + + # int64_t + 'd3 00 00 00 10 00 00 00 00' => '68719476736', + 'd3 00 00 00 10 00 00 00 01' => '68719476737', + 'd3 00 00 10 00 00 00 00 00' => '17592186044416', + 'd3 00 10 00 00 00 00 00 00' => '4503599627370496', + 'd3 10 00 00 00 00 00 00 00' => '1152921504606846976', + 'd3 11 00 00 00 00 00 00 00' => '1224979098644774912', ) diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c index 6ebb48c2..95b5910f 100644 --- a/perl/xs-src/unpack.c +++ b/perl/xs-src/unpack.c @@ -1,5 +1,6 @@ #define NEED_newRV_noinc #define NEED_sv_2pv_flags +#define NEED_my_snprintf #include "xshelper.h" #define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION @@ -30,6 +31,7 @@ typedef struct { #define msgpack_unpack_user unpack_user void init_Data__MessagePack_unpack(pTHX_ bool const cloning) { + // booleans are load on demand (lazy load). if(!cloning) { MY_CXT_INIT; MY_CXT.msgpack_true = NULL; @@ -51,11 +53,17 @@ static SV* load_bool(pTHX_ const char* const name) { CV* const cv = get_cv(name, GV_ADD); dSP; + ENTER; + SAVETMPS; PUSHMARK(SP); call_sv((SV*)cv, G_SCALAR); SPAGAIN; SV* const sv = newSVsv(POPs); PUTBACK; + FREETMPS; + LEAVE; + assert(sv); + assert(sv_isobject(sv)); return sv; } @@ -102,13 +110,6 @@ STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const return 0; } -STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) -{ - dTHX; - *o = newSVnv((NV)d); - return 0; -} - STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o) { dTHX; @@ -116,10 +117,21 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o) { dTHX; - *o = newSVnv((NV)d); + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d); + *o = newSVpvn(tbuf, len); + return 0; +} + +static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o) +{ + dTHX; + char tbuf[64]; + STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d); + *o = newSVpvn(tbuf, len); return 0; } @@ -142,7 +154,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const return 0; } -#define template_callback_uint64 template_callback_IV +#define template_callback_int64 template_callback_IV #endif /* IVSIZE */