mirror of
https://github.com/USCiLab/cereal.git
synced 2025-09-21 20:59:31 +02:00
working on json unordered loads, almost settled on a solution
This commit is contained in:
Shane Grant
parent
279d60c008
commit
7bd5dffff9
@ -122,7 +122,9 @@ namespace cereal
|
||||
|
||||
//! Starts a new node in the JSON output
|
||||
/*! The node can optionally be given a name by calling setNextName prior
|
||||
to creating the node */
|
||||
to creating the node
|
||||
|
||||
Nodes only need to be started for types that are themselves objects or arrays */
|
||||
void startNode()
|
||||
{
|
||||
writeName();
|
||||
@ -300,17 +302,24 @@ namespace cereal
|
||||
/*! @param stream The stream to read from */
|
||||
JSONInputArchive(std::istream & stream) :
|
||||
InputArchive<JSONInputArchive>(this),
|
||||
itsNextName( nullptr ),
|
||||
itsReadStream(stream)
|
||||
{
|
||||
itsDocument.ParseStream<0>(itsReadStream);
|
||||
itsIteratorStack.push_back(itsDocument.MemberBegin());
|
||||
itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd());
|
||||
}
|
||||
|
||||
//! Loads some binary data, encoded as a base64 string
|
||||
/*! This will automatically start and finish a node to load the data, and can be called directly by
|
||||
users. */
|
||||
void loadBinaryValue( void * data, size_t size )
|
||||
users.
|
||||
|
||||
Note that this follows the same ordering rules specified in the class description in regards
|
||||
to loading in/out of order */
|
||||
|
||||
void loadBinaryValue( void * data, size_t size, const char * name = nullptr )
|
||||
{
|
||||
itsNextName = name;
|
||||
|
||||
std::string encoded;
|
||||
loadValue( encoded );
|
||||
auto decoded = base64::decode( encoded );
|
||||
@ -319,6 +328,7 @@ namespace cereal
|
||||
throw Exception("Decoded binary data size does not match specified size");
|
||||
|
||||
std::memcpy( data, decoded.data(), decoded.size() );
|
||||
itsNextName = nullptr;
|
||||
};
|
||||
|
||||
private:
|
||||
@ -334,15 +344,15 @@ namespace cereal
|
||||
class Iterator
|
||||
{
|
||||
public:
|
||||
Iterator() : nextName( nullptr ), itsType(Null) {}
|
||||
Iterator() : itsType(Null) {}
|
||||
|
||||
Iterator(MemberIterator it) :
|
||||
nextName( nullptr ),
|
||||
itsMemberIt(it), itsType(Member) {}
|
||||
Iterator(MemberIterator begin, MemberIterator end) :
|
||||
itsMemberIt(begin), itsMemberItEnd(end), itsType(Member)
|
||||
{ }
|
||||
|
||||
Iterator(ValueIterator it) :
|
||||
nextName( nullptr ),
|
||||
itsValueIt(it), itsType(Value) {}
|
||||
Iterator(ValueIterator begin, ValueIterator end) :
|
||||
itsValueIt(begin), itsValueItEnd(end), itsType(Value)
|
||||
{ }
|
||||
|
||||
//! Advance to the next node
|
||||
Iterator & operator++()
|
||||
@ -350,7 +360,7 @@ namespace cereal
|
||||
switch(itsType)
|
||||
{
|
||||
case Value : ++itsValueIt; break;
|
||||
case Member: ++itsMemberIt; break;
|
||||
case Member: std::cerr << "Advancing from " << name() << std::endl; ++itsMemberIt; break;
|
||||
default: throw cereal::Exception("Invalid Iterator Type!");
|
||||
}
|
||||
return *this;
|
||||
@ -370,24 +380,57 @@ namespace cereal
|
||||
//! Get the name of the current node, or nullptr if it has no name
|
||||
const char * name() const
|
||||
{
|
||||
switch(itsType)
|
||||
{
|
||||
case Member:
|
||||
return itsMemberIt->name.GetString();
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
if( itsType == Member && itsMemberIt != itsMemberItEnd )
|
||||
return itsMemberIt->name.GetString();
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
public:
|
||||
const char * nextName; //!< The NVP name for next next child node
|
||||
//! Adjust our position such that we are at the node with the given name
|
||||
/*! @throws Exception if no such named node exists */
|
||||
inline void search( const char * name, GenericValue const & parent )
|
||||
{
|
||||
auto member = parent.FindMember( name );
|
||||
if( member )
|
||||
itsMemberIt = member;
|
||||
else
|
||||
throw Exception("JSON Parsing failed - provided NVP not found");
|
||||
}
|
||||
|
||||
private:
|
||||
MemberIterator itsMemberIt; //!< The member iterator (object)
|
||||
ValueIterator itsValueIt; //!< The value iterator (array)
|
||||
enum Type {Value, Member, Null} itsType; //!< Whether this holds values (array) or members (objects) or nothing
|
||||
MemberIterator itsMemberIt, itsMemberItEnd; //!< The member iterator (object)
|
||||
ValueIterator itsValueIt, itsValueItEnd; //!< The value iterator (array)
|
||||
enum Type {Value, Member, Null} itsType; //!< Whether this holds values (array) or members (objects) or nothing
|
||||
};
|
||||
|
||||
//! Searches for the expectedName node if it doesn't match the actualName
|
||||
/*! @throws Exception if an expectedName is given and not found */
|
||||
inline void search()
|
||||
{
|
||||
// The name an NVP provided with setNextName()
|
||||
if( itsNextName )
|
||||
{
|
||||
std::cerr << "Next name is " << itsNextName << std::endl;
|
||||
std::cerr << itsIteratorStack.size() << std::endl;
|
||||
// The actual name of the current node
|
||||
auto const actualName = itsIteratorStack.back().name();
|
||||
|
||||
if( itsIteratorStack.back().value().IsNull() || ( actualName && std::strcmp( itsNextName, actualName ) != 0 ) )
|
||||
{
|
||||
std::cerr << "Searching for " << itsNextName << std::endl;
|
||||
std::cerr << "Actual name was: " << (actualName?actualName:"null") << std::endl;
|
||||
std::cerr << itsIteratorStack.size() << std::endl;
|
||||
// names don't match, perform a search and adjust our current iterator
|
||||
itsIteratorStack.back().search( itsNextName,
|
||||
/*if*/ (itsIteratorStack.size() > 1 ?
|
||||
/*then*/ (itsIteratorStack.rbegin() + 1)->value() :
|
||||
/*else*/ itsDocument ) );
|
||||
}
|
||||
}
|
||||
|
||||
itsNextName = nullptr;
|
||||
}
|
||||
|
||||
public:
|
||||
//! Starts a new node, going into its proper iterator
|
||||
/*! This places an iterator for the next node to be parsed onto the iterator stack. If the next
|
||||
@ -402,24 +445,12 @@ namespace cereal
|
||||
named after the NVP that is being loaded. If that NVP does not exist, we throw an exception */
|
||||
void startNode()
|
||||
{
|
||||
auto const expectedName = itsIteratorStack.back().nextName; // this is the expected name from the NVP, if provided
|
||||
auto const actualName = itsIteratorStack.back().name(); // this is the name our next node actually has
|
||||
search();
|
||||
|
||||
// If we were given an NVP name, look for it in the current level of the document.
|
||||
// We only need to do this if the NVP name does not match the name of the node we would normally read next
|
||||
if( expectedName && ( std::strcmp( expectedName, actualName) ) != 0 )
|
||||
{
|
||||
if( !actualName || !itsIteratorStack.back().value().HasMember( actualName ) )
|
||||
throw Exception("JSON Parsing failed - provided NVP not found");
|
||||
}
|
||||
if(itsIteratorStack.back().value().IsArray())
|
||||
itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End());
|
||||
else
|
||||
{
|
||||
// proceed as normal
|
||||
if(itsIteratorStack.back().value().IsArray())
|
||||
itsIteratorStack.push_back(itsIteratorStack.back().value().Begin());
|
||||
else
|
||||
itsIteratorStack.push_back(itsIteratorStack.back().value().MemberBegin());
|
||||
}
|
||||
itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), itsIteratorStack.back().value().MemberEnd());
|
||||
}
|
||||
|
||||
//! Finishes the most recently started node
|
||||
@ -427,12 +458,13 @@ namespace cereal
|
||||
{
|
||||
itsIteratorStack.pop_back();
|
||||
++itsIteratorStack.back();
|
||||
std::cerr << "Finishing a node " << itsIteratorStack.size() << std::endl;
|
||||
}
|
||||
|
||||
//! Sets the name for the next node created with startNode
|
||||
void setNextName( const char * name )
|
||||
{
|
||||
itsIteratorStack.back().nextName = name;
|
||||
itsNextName = name;
|
||||
}
|
||||
|
||||
//! Loads a value from the current node - small signed overload
|
||||
@ -440,6 +472,8 @@ namespace cereal
|
||||
typename std::enable_if<std::is_signed<T>::value && sizeof(T) < sizeof(int64_t), void>::type
|
||||
loadValue(T & val)
|
||||
{
|
||||
search();
|
||||
|
||||
val = itsIteratorStack.back().value().GetInt();
|
||||
++itsIteratorStack.back();
|
||||
}
|
||||
@ -450,22 +484,24 @@ namespace cereal
|
||||
!std::is_same<bool, T>::value, void>::type
|
||||
loadValue(T & val)
|
||||
{
|
||||
search();
|
||||
|
||||
val = itsIteratorStack.back().value().GetUint();
|
||||
++itsIteratorStack.back();
|
||||
}
|
||||
|
||||
//! Loads a value from the current node - bool overload
|
||||
void loadValue(bool & val) { val = itsIteratorStack.back().value().GetBool(); ++itsIteratorStack.back(); }
|
||||
void loadValue(bool & val) { search(); val = itsIteratorStack.back().value().GetBool(); ++itsIteratorStack.back(); }
|
||||
//! Loads a value from the current node - int64 overload
|
||||
void loadValue(int64_t & val) { val = itsIteratorStack.back().value().GetInt64(); ++itsIteratorStack.back(); }
|
||||
void loadValue(int64_t & val) { search(); val = itsIteratorStack.back().value().GetInt64(); ++itsIteratorStack.back(); }
|
||||
//! Loads a value from the current node - uint64 overload
|
||||
void loadValue(uint64_t & val) { val = itsIteratorStack.back().value().GetUint64(); ++itsIteratorStack.back(); }
|
||||
void loadValue(uint64_t & val) { search(); val = itsIteratorStack.back().value().GetUint64(); ++itsIteratorStack.back(); }
|
||||
//! Loads a value from the current node - float overload
|
||||
void loadValue(float & val) { val = static_cast<float>(itsIteratorStack.back().value().GetDouble()); ++itsIteratorStack.back(); }
|
||||
void loadValue(float & val) { search(); val = static_cast<float>(itsIteratorStack.back().value().GetDouble()); ++itsIteratorStack.back(); }
|
||||
//! Loads a value from the current node - double overload
|
||||
void loadValue(double & val) { val = itsIteratorStack.back().value().GetDouble(); ++itsIteratorStack.back(); }
|
||||
void loadValue(double & val) { search(); val = itsIteratorStack.back().value().GetDouble(); ++itsIteratorStack.back(); }
|
||||
//! Loads a value from the current node - string overload
|
||||
void loadValue(std::string & val) { val = itsIteratorStack.back().value().GetString(); ++itsIteratorStack.back(); }
|
||||
void loadValue(std::string & val) { search(); val = itsIteratorStack.back().value().GetString(); ++itsIteratorStack.back(); }
|
||||
|
||||
//! Loads a value from the current node - long double and long long overloads
|
||||
/*! These data types will automatically be encoded as base64 strings */
|
||||
@ -493,6 +529,7 @@ namespace cereal
|
||||
//! @}
|
||||
|
||||
private:
|
||||
const char * itsNextName; //!< Next name set by NVP
|
||||
ReadStream itsReadStream; //!< Rapidjson write stream
|
||||
std::vector<Iterator> itsIteratorStack; //!< 'Stack' of rapidJSON iterators
|
||||
rapidjson::Document itsDocument; //!< Rapidjson document
|
||||
|
||||
2
include/cereal/external/rapidjson/document.h
vendored
2
include/cereal/external/rapidjson/document.h
vendored
@ -614,6 +614,7 @@ private:
|
||||
Array a;
|
||||
}; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode
|
||||
|
||||
public:
|
||||
//! Find member by name.
|
||||
Member* FindMember(const Ch* name) {
|
||||
RAPIDJSON_ASSERT(name);
|
||||
@ -629,6 +630,7 @@ private:
|
||||
return 0;
|
||||
}
|
||||
const Member* FindMember(const Ch* name) const { return const_cast<GenericValue&>(*this).FindMember(name); }
|
||||
private:
|
||||
|
||||
// Initialize this value as array with initial data, without calling destructor.
|
||||
void SetArrayRaw(GenericValue* values, SizeType count, Allocator& alloctaor) {
|
||||
|
||||
@ -289,6 +289,38 @@ class Stuff
|
||||
}
|
||||
};
|
||||
|
||||
struct OOJson
|
||||
{
|
||||
OOJson() = default;
|
||||
OOJson( int aa, int bb, bool cc, double dd ) :
|
||||
a( aa ), b( bb ), c{ cc, dd }
|
||||
{
|
||||
d[0] = 0; d[1] = 1; d[2] = 2;
|
||||
}
|
||||
|
||||
int a;
|
||||
int b;
|
||||
std::pair<bool, double> c;
|
||||
float d[3];
|
||||
|
||||
template <class Archive>
|
||||
void serialize( Archive & ar )
|
||||
{
|
||||
ar( CEREAL_NVP(c) );
|
||||
ar( CEREAL_NVP(a) );
|
||||
ar( b );
|
||||
ar( CEREAL_NVP(d) );
|
||||
}
|
||||
};
|
||||
|
||||
template <class T>
|
||||
void ARRAY( T && a )
|
||||
{
|
||||
std::cerr << "---------------" << std::endl;
|
||||
std::cerr << std::is_array<typename std::remove_reference<T>::type>::value << std::endl;
|
||||
std::cerr << std::is_array<T>::value << std::endl;
|
||||
}
|
||||
|
||||
// ######################################################################
|
||||
int main()
|
||||
{
|
||||
@ -303,6 +335,7 @@ int main()
|
||||
//oar( f );
|
||||
//oar( f2 );
|
||||
Stuff s; s.fillData();
|
||||
//ARRAY( s );
|
||||
oar( cereal::make_nvp("best data ever", s) );
|
||||
}
|
||||
|
||||
@ -332,6 +365,66 @@ int main()
|
||||
archive( Bla::x );
|
||||
}
|
||||
|
||||
// test out of order
|
||||
std::stringstream oos;
|
||||
{
|
||||
cereal::JSONOutputArchive ar(oos);
|
||||
cereal::JSONOutputArchive ar2(std::cout);
|
||||
|
||||
ar( cereal::make_nvp( "1", 1 ),
|
||||
cereal::make_nvp( "2", 2 ),
|
||||
3,
|
||||
0, // unused
|
||||
cereal::make_nvp( "4", 4 ),
|
||||
cereal::make_nvp( "5", 5 ) );
|
||||
|
||||
int x = 33;
|
||||
ar.saveBinaryValue( &x, sizeof(int), "bla" );
|
||||
|
||||
ar2( cereal::make_nvp( "1", 1 ),
|
||||
cereal::make_nvp( "2", 2 ),
|
||||
3,
|
||||
0, // unused
|
||||
cereal::make_nvp( "4", 4 ),
|
||||
cereal::make_nvp( "5", 5 ) );
|
||||
ar2.saveBinaryValue( &x, sizeof(int), "bla" );
|
||||
|
||||
OOJson oo( 1, 2, 3, 4 );
|
||||
ar( CEREAL_NVP(oo) );
|
||||
ar2( CEREAL_NVP(oo) );
|
||||
|
||||
//int bla[][3] = {{1,2,3},{4,5,6}};
|
||||
//ar2( cereal::make_nvp("asdf", bla) );
|
||||
//ARRAY(asdf);
|
||||
}
|
||||
|
||||
{
|
||||
cereal::JSONInputArchive ar(oos);
|
||||
int i1, i2, i3, i4, i5, x;
|
||||
|
||||
ar( i1 );
|
||||
ar( cereal::make_nvp( "2", i2 ), i3 );
|
||||
|
||||
ar( cereal::make_nvp( "4", i4 ),
|
||||
i5 );
|
||||
|
||||
ar.loadBinaryValue( &x, sizeof(int) );
|
||||
|
||||
OOJson ii;
|
||||
ar( cereal::make_nvp("oo", ii) );
|
||||
ar( cereal::make_nvp( "2", i2 ) );
|
||||
|
||||
std::cout << i1 << " " << i2 << " " << i3 << " " << i4 << " " << i5 << std::endl;
|
||||
std::cout << x << std::endl;
|
||||
std::cout << ii.a << " " << ii.b << " " << ii.c.first << " " << ii.c.second << " ";
|
||||
for( auto z : ii.d )
|
||||
std::cout << z << " ";
|
||||
std::cout << std::endl;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//{
|
||||
// std::ifstream is("file.json");
|
||||
|
||||
@ -3082,6 +3082,7 @@ void test_unordered_loads()
|
||||
std::ostringstream os;
|
||||
{
|
||||
OArchive oar(os);
|
||||
OArchive oar2(std::cout);
|
||||
|
||||
oar( cereal::make_nvp( name1, o_int1 ),
|
||||
cereal::make_nvp( name2, o_double2 ),
|
||||
@ -3090,6 +3091,14 @@ void test_unordered_loads()
|
||||
cereal::make_nvp( name5, o_int5 ),
|
||||
cereal::make_nvp( name6, o_int6 ),
|
||||
cereal::make_nvp( name7, o_un7 ) );
|
||||
|
||||
oar2( cereal::make_nvp( name1, o_int1 ),
|
||||
cereal::make_nvp( name2, o_double2 ),
|
||||
cereal::make_nvp( name3, o_vecbool3 ),
|
||||
cereal::make_nvp( name4, o_int4 ),
|
||||
cereal::make_nvp( name5, o_int5 ),
|
||||
cereal::make_nvp( name6, o_int6 ),
|
||||
cereal::make_nvp( name7, o_un7 ) );
|
||||
}
|
||||
|
||||
decltype(o_int1) i_int1;
|
||||
@ -3114,7 +3123,7 @@ void test_unordered_loads()
|
||||
}
|
||||
|
||||
BOOST_CHECK_EQUAL(o_int1, i_int1);
|
||||
BOOST_CHECK_EQUAL(o_double2, i_double2);
|
||||
BOOST_CHECK_CLOSE(o_double2 , o_double2, 1e-5);
|
||||
BOOST_CHECK_EQUAL(o_vecbool3.size(), i_vecbool3.size());
|
||||
BOOST_CHECK_EQUAL_COLLECTIONS(i_vecbool3.begin(), i_vecbool3.end(), o_vecbool3.begin(), o_vecbool3.end());
|
||||
BOOST_CHECK_EQUAL(o_int4, i_int4);
|
||||
@ -3130,7 +3139,7 @@ BOOST_AUTO_TEST_CASE( xml_unordered_loads )
|
||||
test_unordered_loads<cereal::XMLInputArchive, cereal::XMLOutputArchive>();
|
||||
}
|
||||
|
||||
//BOOST_AUTO_TEST_CASE( json_unordered_loads )
|
||||
//{
|
||||
// test_unordered_loads<cereal::JSONInputArchive, cereal::JSONOutputArchive>();
|
||||
//}
|
||||
BOOST_AUTO_TEST_CASE( json_unordered_loads )
|
||||
{
|
||||
test_unordered_loads<cereal::JSONInputArchive, cereal::JSONOutputArchive>();
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user