From 8f7f23a0e5fcc595f6d6178e9e532b38b5cd1b46 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 18:11:52 +0200 Subject: [PATCH] Rewrite unpack_/1 using pattern matching to get a 30-40% speedup. Simplify pack_* and unpack_{array,map} function clauses to get more readability and a minor speedup. --- erlang/msgpack.erl | 290 +++++++++++++++++++-------------------------- 1 file changed, 123 insertions(+), 167 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 255542b6..e94262d1 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -42,8 +42,10 @@ pack(O) when is_float(O) -> pack_double(O); pack(nil) -> pack_nil(); -pack(Bool) when is_atom(Bool) -> - pack_bool(Bool); +pack(true) -> + pack_true(); +pack(false) -> + pack_false(); pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> @@ -61,13 +63,8 @@ pack(_O) -> % TODO: error case for imcomplete format when short for any type formats. -spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. -unpack(Bin) when not is_binary(Bin)-> - {error, badarg}; -unpack(Bin) when bit_size(Bin) >= 8 -> - << Flag:8/unsigned-integer, Payload/binary >> = Bin, - unpack_(Flag, Payload); -unpack(<<>>)-> % when bit_size(Bin) < 8 -> - {more, 1}. +unpack(Bin) -> + unpack_(Bin). -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -82,56 +79,52 @@ unpack_all(Data)-> % ===== internal APIs ===== % % positive fixnum -pack_uint_(N) when is_integer( N ) , N < 128 -> +pack_uint_(N) when N < 128 -> << 2#0:1, N:7 >>; % uint 8 -pack_uint_( N ) when is_integer( N ) andalso N < 256 -> +pack_uint_(N) when N < 256 -> << 16#CC:8, N:8 >>; - % uint 16 -pack_uint_( N ) when is_integer( N ) andalso N < 65536 -> +pack_uint_(N) when N < 65536 -> << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>; - % uint 32 -pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF-> +pack_uint_(N) when N < 16#FFFFFFFF-> << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>; - % uint 64 -pack_uint_( N ) when is_integer( N )-> +pack_uint_(N) -> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. % negative fixnum -pack_int_( N ) when is_integer( N ) , N >= -32-> +pack_int_(N) when N >= -32-> << 2#111:3, N:5 >>; % int 8 -pack_int_( N ) when is_integer( N ) , N >= -256 -> +pack_int_(N) when N >= -256 -> << 16#D0:8, N:8 >>; % int 16 -pack_int_( N ) when is_integer( N ), N >= -65536 -> +pack_int_(N) when N >= -65536 -> << 16#D1:8, N:16/big-signed-integer-unit:1 >>; % int 32 -pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF -> +pack_int_(N) when N >= -16#FFFFFFFF -> << 16#D2:8, N:32/big-signed-integer-unit:1 >>; % int 64 -pack_int_( N ) when is_integer( N )-> +pack_int_(N) -> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. -% nil -pack_nil()-> << 16#C0:8 >>. -% pack_true / pack_false -pack_bool(true)-> << 16#C3:8 >>; -pack_bool(false)-> << 16#C2:8 >>. +% nil/true/false +pack_nil() -> << 16#C0:8 >>. +pack_true()-> << 16#C3:8 >>. +pack_false()-> << 16#C2:8 >>. % float : erlang's float is always IEEE 754 64bit format. %pack_float(F) when is_float(F)-> % << 16#CA:8, F:32/big-float-unit:1 >>. % pack_double(F). % double -pack_double(F) when is_float(F)-> +pack_double(F) -> << 16#CB:8, F:64/big-float-unit:1 >>. % raw bytes -pack_raw(Bin) when is_binary(Bin)-> +pack_raw(Bin) -> case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; @@ -142,24 +135,22 @@ pack_raw(Bin) when is_binary(Bin)-> end. % list / tuple -pack_array(L) when is_list(L)-> +pack_array(L) -> case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>; + << 16#DC:8, Len:16/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len -> - << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >> + << 16#DD:8, Len:32/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >> end. pack_array_([], Acc) -> Acc; pack_array_([Head|Tail], Acc) -> pack_array_(Tail, <>). % FIXME! this should be tail-recursive and without lists:reverse/1 -unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; -unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; -unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; -unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> +unpack_array_(Remain, 0, RetList) -> {lists:reverse(RetList), Remain}; +unpack_array_(Bin, RestLen, RetList) -> case unpack(Bin) of {more, Len} -> {more, Len+RestLen-1}; {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) @@ -181,10 +172,10 @@ pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). % FIXME: write test for unpack_map/1 --spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> - {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin}; -unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> +-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}]) -> + {more, non_neg_integer()} | {any(), binary()}. +unpack_map_(Bin, 0, Acc) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(Bin, Len, Acc) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> @@ -195,142 +186,107 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> end end. -% {more, --spec unpack_(Flag::integer(), Payload::binary())-> - {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. -unpack_(Flag, Payload)-> - PayloadLen = byte_size(Payload), - case Flag of - 16#C0 -> - {nil, Payload}; - 16#C2 -> - {false, Payload}; - 16#C3 -> - {true, Payload}; - 16#CA when PayloadLen >= 4 -> % 32bit float - << Return:32/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CA -> - {more, 4-PayloadLen}; % at least more +-spec unpack_(Payload::binary()) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. +unpack_(<<16#C0, Rest/binary>>) -> + {nil, Rest}; +unpack_(<<16#C2, Rest/binary>>) -> + {false, Rest}; +unpack_(<<16#C3, Rest/binary>>) -> + {true, Rest}; - 16#CB when PayloadLen >= 8 -> % 64bit float - << Return:64/float-unit:1, Rest/binary >> = Payload, - {Return, Rest}; - 16#CB -> - {more, 8-PayloadLen}; +unpack_(<<16#CA, Return:32/float-unit:1, Rest/binary>>) -> % 32bit float + {Return, Rest}; +unpack_(<<16#CA, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#CB, Return:64/float-unit:1, Rest/binary>>) -> % 64bit float + {Return, Rest}; +unpack_(<<16#CB, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CC when PayloadLen >= 1 -> % uint 8 - << Int:8/unsigned-integer, Rest/binary >> = Payload, - {Int, Rest}; - 16#CC -> - {more, 1}; +unpack_(<<16#CC, Int:8/unsigned-integer, Rest/binary>>) -> % uint 8 + {Int, Rest}; +unpack_(<<16#CC>>) -> + {more, 1}; +unpack_(<<16#CD, Int:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 16 + {Int, Rest}; +unpack_(<<16#CD, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#CE, Int:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 32 + {Int, Rest}; +unpack_(<<16#CE, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#CF, Int:64/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 64 + {Int, Rest}; +unpack_(<<16#CF, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CD when PayloadLen >= 2 -> % uint 16 - << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CD -> - {more, 2-PayloadLen}; +unpack_(<<16#D0, Int:8/signed-integer, Rest/binary>>) -> % int 8 + {Int, Rest}; +unpack_(<<16#D0>>) -> + {more, 1}; +unpack_(<<16#D1, Int:16/big-signed-integer-unit:1, Rest/binary>>) -> % int 16 + {Int, Rest}; +unpack_(<<16#D1, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#D2, Int:32/big-signed-integer-unit:1, Rest/binary>>) -> % int 32 + {Int, Rest}; +unpack_(<<16#D2, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; +unpack_(<<16#D3, Int:64/big-signed-integer-unit:1, Rest/binary>>) -> % int 64 + {Int, Rest}; +unpack_(<<16#D3, Rest/binary>>) -> + {more, 8-byte_size(Rest)}; - 16#CE when PayloadLen >= 4 -> - << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CE -> - {more, 4-PayloadLen}; % at least more +unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 16 + {Val, Rest}; +unpack_(<<16#DA, Rest/binary>>) -> + {more, 16-byte_size(Rest)}; +unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 32 + {Val, Rest}; +unpack_(<<16#DB, Rest/binary>>) -> + {more, 32-byte_size(Rest)}; - 16#CF when PayloadLen >= 8 -> - << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#CF -> - {more, 8-PayloadLen}; +unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 16 + unpack_array_(Rest, Len, []); +unpack_(<<16#DC, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 32 + unpack_array_(Rest, Len, []); +unpack_(<<16#DD, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; - 16#D0 when PayloadLen >= 1 -> % int 8 - << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D0 -> - {more, 1}; +unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 16 + unpack_map_(Rest, Len, []); +unpack_(<<16#DE, Rest/binary>>) -> + {more, 2-byte_size(Rest)}; +unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 32 + unpack_map_(Rest, Len, []); +unpack_(<<16#DF, Rest/binary>>) -> + {more, 4-byte_size(Rest)}; - 16#D1 when PayloadLen >= 2 -> % int 16 - << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D1 -> - {more, 2-PayloadLen}; +unpack_(<<0:1, Value:7, Rest/binary>>) -> % positive fixnum + {Value, Rest}; +unpack_(<<2#111:3, Value:5, Rest/binary>>) -> % negative fixnum + {Value - 2#100000, Rest}; +unpack_(<<2#101:3, Len:5, Value:Len/binary, Rest/binary>>) -> % fixraw + {Value, Rest}; +unpack_(<<2#101:3, Len:5, Rest/binary>>) -> + {more, Len-byte_size(Rest)}; +unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> % fixarray + unpack_array_(Rest, Len, []); +unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> % fixmap + unpack_map_(Rest, Len, []); - 16#D2 when PayloadLen >= 4 -> % int 32 - << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D2 -> - {more, 4-PayloadLen}; +%unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9-> +% {error, {badarg, <>}}. +%unpack_(Other) when is_binary(Bin) -> +% {more, 1}. +unpack_(<<>>) -> + {more, 1}. +unpack_(Other) -> + {error, {badarg, Other}}. - 16#D3 when PayloadLen >= 8 -> % int 64 - << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload, - {Int, Rest}; - 16#D3 -> - {more, 8-PayloadLen}; - - 16#DA when PayloadLen >= 2 -> % raw 16 - << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DA -> - {more, 16-PayloadLen}; - - 16#DB when PayloadLen >= 4 -> % raw 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - << Return:Len/binary, Remain/binary >> = Rest, - {Return, Remain}; - 16#DB -> - {more, 4-PayloadLen}; - - 16#DC when PayloadLen >= 2 -> % array 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DC -> - {more, 2-PayloadLen}; - - 16#DD when PayloadLen >= 4 -> % array 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_array_(Rest, Len, []); - 16#DD -> - {more, 4-PayloadLen}; - - 16#DE when PayloadLen >= 2 -> % map 16 - << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - 16#DE -> - {more, 2-PayloadLen}; - - 16#DF when PayloadLen >= 4 -> % map 32 - << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); - - % positive fixnum - Code when Code >= 2#00000000, Code < 2#10000000-> - {Code, Payload}; - - % negative fixnum - Code when Code >= 2#11100000 -> - {(Code - 16#100), Payload}; - - Code when Code >= 2#10100000 , Code < 2#11000000 -> -% 101XXXXX for FixRaw - Len = Code rem 2#10100000, - << Return:Len/binary, Remain/binary >> = Payload, - {Return, Remain}; - - Code when Code >= 2#10010000 , Code < 2#10100000 -> -% 1001XXXX for FixArray - Len = Code rem 2#10010000, - unpack_array_(Payload, Len, []); - - Code when Code >= 2#10000000 , Code < 2#10010000 -> -% 1000XXXX for FixMap - Len = Code rem 2#10000000, - unpack_map_(Payload, Len, []); - - _Other -> - {error, no_code_matches} - end. % ===== test codes ===== % -include_lib("eunit/include/eunit.hrl"). @@ -419,7 +375,7 @@ unknown_test()-> test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), - {S, <<>>} = msgpack:unpack( Pack ), + ?assertEqual({S, <<>>}, msgpack:unpack(Pack)), 1+test_(Rest). other_test()->