generic-library/msgpack
1
0
Fork 0
mirror of https://github.com/msgpack/msgpack-c.git synced 2025-03-19 04:52:59 +01:00
Code Issues Projects Releases Wiki Activity

Fix the stream unpacker

Browse Source
This commit is contained in:
gfx 2010-09-16 20:24:01 +09:00
parent 4cb6d6995f
commit afbddbfcda
5 changed files with 148 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
perl/t/06_stream_unpack2.t
View File

@ -1,9 +1,9 @@
use strict; use strict;
use warnings; use warnings;
use Data::MessagePack; use Data::MessagePack;
use Test::More tests => 6; use Test::More tests => 9;
my $input = [42, "foo", { x => [ (undef)x16 ] }, 3.14 ]; my $input = [ 42, "foo", { x => [ (1) x 16 ] }, undef, 1 ];
my $packed = Data::MessagePack->pack($input); my $packed = Data::MessagePack->pack($input);
is_deeply(Data::MessagePack->unpack($packed), $input); is_deeply(Data::MessagePack->unpack($packed), $input);
@ -26,24 +26,19 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
{ {
my $up = Data::MessagePack::Unpacker->new(); my $up = Data::MessagePack::Unpacker->new();
my $offset = 0; my $size = 8;
my $size = 5;
note "length: ", length($packed); note "packed size: ", length($packed);
while(not $up->is_finished) { open my $stream, '<:bytes :scalar', \$packed;
note "offset: ", $offset; my $buff;
my $bytes = substr($packed, $offset, $size); while( read($stream, $buff, $size) ) {
note join " ", map { unpack 'H2', $_ } split //, $bytes; note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
my $x = $up->execute($bytes, 0);
if($x <= 0) { $up->execute($buff);
diag "Something's wrong: $x";
last;
}
else {
$offset += $x;
}
} }
ok $up->is_finished; ok $up->is_finished, 'is_finished';
is_deeply $up->data, $input; my $data = $up->data;
note explain($data);
is_deeply $data, $input;
} }

1
perl/xs-src/MessagePack.c
View File

@ -14,6 +14,7 @@ void boot_Data__MessagePack_pack(void);
XS(boot_Data__MessagePack) { XS(boot_Data__MessagePack) {
dXSARGS; dXSARGS;
PERL_UNUSED_VAR(items);
boot_Data__MessagePack_pack(); boot_Data__MessagePack_pack();

2
perl/xs-src/pack.c
View File

@ -145,7 +145,7 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
} }
static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth); STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) { STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
dTHX; dTHX;

254
perl/xs-src/unpack.c
View File

@ -26,14 +26,9 @@ typedef struct {
/* utility functions */ /* utility functions */
STATIC_INLINE SV * STATIC_INLINE SV *
get_bool (const char *name) { get_bool (const char* const name) {
dTHX; dTHX;
SV * sv = sv_mortalcopy(get_sv( name, 1 )); return newSVsv(get_sv( name, GV_ADD ));
SvREADONLY_on(sv);
SvREADONLY_on( SvRV(sv) );
return sv;
} }
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
@ -49,85 +44,76 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL) STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
{ {
dTHX; return NULL;
return &PL_sv_undef;
} }
STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o) #if IVSIZE == 4
STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSVuv(d)); *o = newSVuv(d);
return 0; return 0;
} }
STATIC_INLINE int template_callback_uint16(unpack_user* u PERL_UNUSED_DECL, uint16_t d, SV** o) STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSVuv(d)); *o = newSVnv((NV)d);
return 0; return 0;
} }
STATIC_INLINE int template_callback_uint32(unpack_user* u PERL_UNUSED_DECL, uint32_t d, SV** o) STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSVuv(d)); *o = newSViv(d);
return 0; return 0;
} }
STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t d, SV** o) STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
{ {
dTHX; dTHX;
#if IVSIZE==4 *o = newSVnv((NV)d);
*o = sv_2mortal(newSVnv(d));
#else
*o = sv_2mortal(newSVuv(d));
#endif
return 0; return 0;
} }
STATIC_INLINE int template_callback_int8(unpack_user* u PERL_UNUSED_DECL, int8_t d, SV** o) #else /* IVSIZE == 8 */
STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSViv(d)); *o = newSVuv(d);
return 0; return 0;
} }
STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16_t d, SV** o) #define template_callback_uint64 template_callback_UV
STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSViv(d)); *o = newSViv(d);
return 0; return 0;
} }
STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32_t d, SV** o) #define template_callback_uint64 template_callback_IV
{
dTHX;
*o = sv_2mortal(newSViv(d));
return 0;
}
STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o) #endif /* IVSIZE */
{
dTHX;
#if IVSIZE==4
*o = sv_2mortal(newSVnv(d));
#else
*o = sv_2mortal(newSViv(d));
#endif
return 0;
}
STATIC_INLINE int template_callback_float(unpack_user* u PERL_UNUSED_DECL, float d, SV** o) #define template_callback_uint8 template_callback_UV
{ #define template_callback_uint16 template_callback_UV
dTHX; #define template_callback_uint32 template_callback_UV
*o = sv_2mortal(newSVnv(d));
return 0; #define template_callback_int8 template_callback_IV
} #define template_callback_int16 template_callback_IV
#define template_callback_int32 template_callback_IV
#define template_callback_float template_callback_double
STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o) STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
{ {
dTHX; dTHX;
*o = sv_2mortal(newSVnv(d)); *o = newSVnv(d);
return 0; return 0;
} }
@ -135,7 +121,7 @@ STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, doub
STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o) STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
{ {
dTHX; dTHX;
*o = sv_newmortal(); *o = newSV(0);
return 0; return 0;
} }
@ -148,7 +134,8 @@ STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o
STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o) STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
{ {
dTHX; *o = get_bool("Data::MessagePack::false"); dTHX;
*o = get_bool("Data::MessagePack::false");
return 0; return 0;
} }
@ -156,7 +143,7 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig
{ {
dTHX; dTHX;
AV* const a = newAV(); AV* const a = newAV();
*o = sv_2mortal(newRV_noinc((SV*)a)); *o = newRV_noinc((SV*)a);
av_extend(a, n + 1); av_extend(a, n + 1);
return 0; return 0;
} }
@ -167,7 +154,6 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
AV* const a = (AV*)SvRV(*c); AV* const a = (AV*)SvRV(*c);
assert(SvTYPE(a) == SVt_PVAV); assert(SvTYPE(a) == SVt_PVAV);
(void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o) (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o)
SvREFCNT_inc_simple_void_NN(o);
return 0; return 0;
} }
@ -175,7 +161,7 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne
{ {
dTHX; dTHX;
HV* const h = newHV(); HV* const h = newHV();
*o = sv_2mortal(newRV_noinc((SV*)h)); *o = newRV_noinc((SV*)h);
return 0; return 0;
} }
@ -185,7 +171,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
HV* const h = (HV*)SvRV(*c); HV* const h = (HV*)SvRV(*c);
assert(SvTYPE(h) == SVt_PVHV); assert(SvTYPE(h) == SVt_PVHV);
(void)hv_store_ent(h, k, v, 0); (void)hv_store_ent(h, k, v, 0);
SvREFCNT_inc_simple_void_NN(v); SvREFCNT_dec(k);
return 0; return 0;
} }
@ -193,10 +179,12 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
{ {
dTHX; dTHX;
/* newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */ /* newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */
*o = ((l==0) ? newSVpvs_flags("", SVs_TEMP) : newSVpvn_flags(p, l, SVs_TEMP)); *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l));
return 0; return 0;
} }
#include "msgpack/unpack_template.h"
#define UNPACKER(from, name) \ #define UNPACKER(from, name) \
msgpack_unpack_t *name; \ msgpack_unpack_t *name; \
if(!(SvROK(from) && SvIOK(SvRV(from)))) { \ if(!(SvROK(from) && SvIOK(SvRV(from)))) { \
@ -207,36 +195,6 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
} }
#include "msgpack/unpack_template.h"
STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
msgpack_unpack_t mp;
dTHX;
unpack_user u = {false, false};
int ret;
size_t from = 0;
STRLEN dlen;
const char * dptr = SvPV_const(data, dlen);
SV* obj;
template_init(&mp);
mp.user = u;
ret = template_execute(&mp, dptr, (size_t)dlen, &from);
obj = template_data(&mp);
if(ret < 0) {
Perl_croak(aTHX_ "parse error.");
} else if(ret == 0) {
Perl_croak(aTHX_ "insufficient bytes.");
} else {
if(from < dlen) {
Perl_croak(aTHX_ "extra bytes.");
}
return obj;
}
}
XS(xs_unpack) { XS(xs_unpack) {
dXSARGS; dXSARGS;
SV* const data = ST(1); SV* const data = ST(1);
@ -252,17 +210,40 @@ XS(xs_unpack) {
Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])"); Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])");
} }
ST(0) = _msgpack_unpack(data, limit); STRLEN dlen;
const char* const dptr = SvPV_const(data, dlen);
msgpack_unpack_t mp;
template_init(&mp);
unpack_user const u = {false, false};
mp.user = u;
size_t from = 0;
int const ret = template_execute(&mp, dptr, (size_t)dlen, &from);
SV* const obj = template_data(&mp);
sv_2mortal(obj);
if(ret < 0) {
Perl_croak(aTHX_ "Data::MessagePack->unpack: parse error");
} else if(ret == 0) {
Perl_croak(aTHX_ "Data::MessagePack->unpack: insufficient bytes");
} else {
if(from < dlen) {
Perl_croak(aTHX_ "Data::MessagePack->unpack: extra bytes");
}
}
ST(0) = obj;
XSRETURN(1); XSRETURN(1);
} }
/* ------------------------------ stream -- */ /* ------------------------------ stream -- */
/* http://twitter.com/frsyuki/status/13249304748 */ /* http://twitter.com/frsyuki/status/13249304748 */
STATIC_INLINE void _reset(SV* self) { STATIC_INLINE void _reset(SV* const self) {
dTHX; dTHX;
unpack_user u = {false, false}; unpack_user const u = {false, false};
UNPACKER(self, mp); UNPACKER(self, mp);
template_init(mp); template_init(mp);
@ -275,10 +256,10 @@ XS(xs_unpacker_new) {
Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()");
} }
SV* self = sv_newmortal(); SV* const self = sv_newmortal();
msgpack_unpack_t *mp; msgpack_unpack_t *mp;
Newx(mp, 1, msgpack_unpack_t); Newxz(mp, 1, msgpack_unpack_t);
sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); sv_setref_pv(self, "Data::MessagePack::Unpacker", mp);
_reset(self); _reset(self);
@ -287,65 +268,64 @@ XS(xs_unpacker_new) {
XSRETURN(1); XSRETURN(1);
} }
STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) { STATIC_INLINE SV*
_execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
dTHX; dTHX;
if(offset >= limit) {
Perl_croak(aTHX_ "offset (%"UVuf") is bigger than data buffer size (%"UVuf")",
offset, limit);
}
UNPACKER(self, mp); UNPACKER(self, mp);
size_t from = off; size_t from = offset;
const char* dptr = SvPV_nolen_const(data); const char* const dptr = SvPV_nolen_const(data);
int ret;
if(from >= limit) { int const ret = template_execute(mp, dptr, limit, &from);
Perl_croak(aTHX_ "offset (%lu) is bigger than data buffer size (%lu)", (unsigned long)off, (unsigned long)limit);
}
ret = template_execute(mp, dptr, limit, &from); if(ret < 0) {
Perl_croak(aTHX_ "Data::MessagePack::Unpacker: parse error while executing");
if(ret < 0) { } else {
Perl_croak(aTHX_ "parse error."); mp->user.finished = (ret > 0) ? true : false;
} else { return sv_2mortal(newSVuv(from));
mp->user.finished = (ret > 0) ? true : false; }
return sv_2mortal(newSVuv(from));
}
} }
XS(xs_unpacker_execute) { XS(xs_unpacker_execute) {
dXSARGS; dXSARGS;
if (items != 3) { SV* const self = ST(0);
Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)"); SV* const data = ST(1);
UV offset;
if (items == 2) {
offset = 0;
}
else if (items == 3) {
offset = SvUVx(ST(2));
}
else {
Perl_croak(aTHX_ "Usage: $unpacker->execute(data, offset = 0)");
} }
UNPACKER(ST(0), mp); UNPACKER(self, mp);
{
SV* self = ST(0);
SV* data = ST(1);
IV off = SvIV(ST(2)); /* offset of $data. normaly, 0. */
ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data));
if (!mp->user.incremented) {
SV* tmp_obj = template_data(mp);
SvREFCNT_inc_simple_void_NN(tmp_obj);
mp->user.incremented = true;
}
}
ST(0) = _execute_impl(self, data, offset, sv_len(data));
XSRETURN(1); XSRETURN(1);
} }
XS(xs_unpacker_execute_limit) { XS(xs_unpacker_execute_limit) {
dXSARGS; dXSARGS;
if (items != 4) { if (items != 4) {
Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, offset, limit)");
} }
SV* self = ST(0); SV* const self = ST(0);
SV* data = ST(1); SV* const data = ST(1);
IV off = SvIV(ST(2)); UV const offset = SvUVx(ST(2));
IV limit = SvIV(ST(3)); UV const limit = SvUVx(ST(3));
ST(0) = _execute_impl(self, data, off, (size_t)limit);
ST(0) = _execute_impl(self, data, offset, limit);
XSRETURN(1); XSRETURN(1);
} }
@ -355,9 +335,8 @@ XS(xs_unpacker_is_finished) {
Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); Perl_croak(aTHX_ "Usage: $unpacker->is_finished()");
} }
UNPACKER(ST(0), mp); UNPACKER(ST(0), mp);
ST(0) = boolSV(mp->user.finished); ST(0) = boolSV(mp->user.finished);
XSRETURN(1); XSRETURN(1);
} }
@ -367,9 +346,8 @@ XS(xs_unpacker_data) {
Perl_croak(aTHX_ "Usage: $unpacker->data()"); Perl_croak(aTHX_ "Usage: $unpacker->data()");
} }
UNPACKER(ST(0), mp); UNPACKER(ST(0), mp);
ST(0) = template_data(mp); ST(0) = template_data(mp);
XSRETURN(1); XSRETURN(1);
} }
@ -379,10 +357,10 @@ XS(xs_unpacker_reset) {
Perl_croak(aTHX_ "Usage: $unpacker->reset()"); Perl_croak(aTHX_ "Usage: $unpacker->reset()");
} }
UNPACKER(ST(0), mp); UNPACKER(ST(0), mp);
SV* const data = template_data(mp); SV* const data = template_data(mp);
SvREFCNT_dec(data); sv_2mortal(data);
_reset(ST(0)); _reset(ST(0));
XSRETURN(0); XSRETURN(0);
@ -394,10 +372,10 @@ XS(xs_unpacker_destroy) {
Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()"); Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()");
} }
UNPACKER(ST(0), mp); UNPACKER(ST(0), mp);
SV* const data = template_data(mp); SV* const data = template_data(mp);
SvREFCNT_dec(data); sv_2mortal(data);
Safefree(mp); Safefree(mp);
XSRETURN(0); XSRETURN(0);

22
perl/xt/leaks/leaktrace.t
View File

@ -5,7 +5,8 @@ use Test::More;
use Data::MessagePack; use Data::MessagePack;
my $data = { my $simple_data = "xyz";
my $complex_data = {
a => 'foo', a => 'foo',
b => 42, b => 42,
c => undef, c => undef,
@ -13,8 +14,10 @@ my $data = {
e => 3.14, e => 3.14,
}; };
note 'pack';
no_leaks_ok { no_leaks_ok {
my $s = Data::MessagePack->pack($data); my $s = Data::MessagePack->pack($complex_data);
}; };
no_leaks_ok { no_leaks_ok {
@ -23,16 +26,23 @@ no_leaks_ok {
$@ or die "it must die"; $@ or die "it must die";
}; };
my $s = Data::MessagePack->pack($data); note 'unpack';
my $s = Data::MessagePack->pack($simple_data);
my $c = Data::MessagePack->pack($complex_data);
no_leaks_ok { no_leaks_ok {
my $data = Data::MessagePack->unpack($s); my $data = Data::MessagePack->unpack($s);
}; };
no_leaks_ok { no_leaks_ok {
my $ss = $s; my $data = Data::MessagePack->unpack($c);
chop $ss; };
eval { Data::MessagePack->unpack($ss) };
no_leaks_ok {
my $broken = $s;
chop $broken;
eval { Data::MessagePack->unpack($broken) };
#note $@; #note $@;
$@ or die "it must die"; $@ or die "it must die";
}; };