working on json unordered loads, almost settled on a solution

include/cereal/archives/json.hpp

@@ -122,7 +122,9 @@ namespace cereal
 
       //! Starts a new node in the JSON output
       /*! The node can optionally be given a name by calling setNextName prior
-          to creating the node */
+          to creating the node
+
+          Nodes only need to be started for types that are themselves objects or arrays */
       void startNode()
       {
         writeName();
@@ -300,17 +302,24 @@ namespace cereal
       /*! @param stream The stream to read from */
       JSONInputArchive(std::istream & stream) :
         InputArchive<JSONInputArchive>(this),
+        itsNextName( nullptr ),
         itsReadStream(stream)
       {
         itsDocument.ParseStream<0>(itsReadStream);
-        itsIteratorStack.push_back(itsDocument.MemberBegin());
+        itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd());
       }
 
       //! Loads some binary data, encoded as a base64 string
       /*! This will automatically start and finish a node to load the data, and can be called directly by
-          users. */
+          users.
-      void loadBinaryValue( void * data, size_t size )
+
+          Note that this follows the same ordering rules specified in the class description in regards
+          to loading in/out of order */
+
+      void loadBinaryValue( void * data, size_t size, const char * name = nullptr )
       {
+        itsNextName = name;
+
         std::string encoded;
         loadValue( encoded );
         auto decoded = base64::decode( encoded );
@@ -319,6 +328,7 @@ namespace cereal
           throw Exception("Decoded binary data size does not match specified size");
 
         std::memcpy( data, decoded.data(), decoded.size() );
+        itsNextName = nullptr;
       };
 
     private:
@@ -334,15 +344,15 @@ namespace cereal
       class Iterator
       {
         public:
-          Iterator() : nextName( nullptr ), itsType(Null) {}
+          Iterator() : itsType(Null) {}
 
-          Iterator(MemberIterator it) :
+          Iterator(MemberIterator begin, MemberIterator end) :
-            nextName( nullptr ),
+            itsMemberIt(begin), itsMemberItEnd(end), itsType(Member)
-            itsMemberIt(it), itsType(Member) {}
+          { }
 
-          Iterator(ValueIterator it) :
+          Iterator(ValueIterator begin, ValueIterator end) :
-            nextName( nullptr ),
+            itsValueIt(begin), itsValueItEnd(end), itsType(Value)
-            itsValueIt(it), itsType(Value) {}
+          { }
 
           //! Advance to the next node
           Iterator & operator++()
@@ -350,7 +360,7 @@ namespace cereal
             switch(itsType)
             {
               case Value : ++itsValueIt; break;
-              case Member: ++itsMemberIt; break;
+              case Member: std::cerr << "Advancing from " << name() << std::endl; ++itsMemberIt; break;
               default: throw cereal::Exception("Invalid Iterator Type!");
             }
             return *this;
@@ -370,24 +380,57 @@ namespace cereal
           //! Get the name of the current node, or nullptr if it has no name
           const char * name() const
           {
-            switch(itsType)
+            if( itsType == Member && itsMemberIt != itsMemberItEnd )
-            {
+              return itsMemberIt->name.GetString();
-              case Member:
+            else
-                return itsMemberIt->name.GetString();
+              return nullptr;
-              default:
-                return nullptr;
-            }
           }
 
-        public:
+          //! Adjust our position such that we are at the node with the given name
-          const char * nextName;                   //!< The NVP name for next next child node
+          /*! @throws Exception if no such named node exists */
+          inline void search( const char * name, GenericValue const & parent )
+          {
+            auto member = parent.FindMember( name );
+            if( member )
+              itsMemberIt = member;
+            else
+              throw Exception("JSON Parsing failed - provided NVP not found");
+          }
 
         private:
-          MemberIterator itsMemberIt;              //!< The member iterator (object)
+          MemberIterator itsMemberIt, itsMemberItEnd; //!< The member iterator (object)
-          ValueIterator itsValueIt;                //!< The value iterator (array)
+          ValueIterator itsValueIt, itsValueItEnd;    //!< The value iterator (array)
-          enum Type {Value, Member, Null} itsType; //!< Whether this holds values (array) or members (objects) or nothing
+          enum Type {Value, Member, Null} itsType;    //!< Whether this holds values (array) or members (objects) or nothing
       };
 
+      //! Searches for the expectedName node if it doesn't match the actualName
+      /*! @throws Exception if an expectedName is given and not found */
+      inline void search()
+      {
+        // The name an NVP provided with setNextName()
+        if( itsNextName )
+        {
+          std::cerr << "Next name is " << itsNextName << std::endl;
+          std::cerr << itsIteratorStack.size() << std::endl;
+          // The actual name of the current node
+          auto const actualName = itsIteratorStack.back().name();
+
+          if( itsIteratorStack.back().value().IsNull() || ( actualName && std::strcmp( itsNextName, actualName ) != 0 ) )
+          {
+            std::cerr << "Searching for " << itsNextName << std::endl;
+            std::cerr << "Actual name was: " << (actualName?actualName:"null") << std::endl;
+            std::cerr << itsIteratorStack.size() << std::endl;
+            // names don't match, perform a search and adjust our current iterator
+            itsIteratorStack.back().search( itsNextName,
+            /*if*/   (itsIteratorStack.size() > 1 ?
+            /*then*/ (itsIteratorStack.rbegin() + 1)->value() :
+            /*else*/ itsDocument ) );
+          }
+        }
+
+        itsNextName = nullptr;
+      }
+
     public:
       //! Starts a new node, going into its proper iterator
       /*! This places an iterator for the next node to be parsed onto the iterator stack.  If the next
@@ -402,24 +445,12 @@ namespace cereal
           named after the NVP that is being loaded.  If that NVP does not exist, we throw an exception */
       void startNode()
       {
-        auto const expectedName = itsIteratorStack.back().nextName; // this is the expected name from the NVP, if provided
+        search();
-        auto const actualName   = itsIteratorStack.back().name();   // this is the name our next node actually has
 
-        // If we were given an NVP name, look for it in the current level of the document.
+        if(itsIteratorStack.back().value().IsArray())
-        //    We only need to do this if the NVP name does not match the name of the node we would normally read next
+          itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End());
-        if( expectedName && ( std::strcmp( expectedName, actualName) ) != 0 )
-        {
-          if( !actualName || !itsIteratorStack.back().value().HasMember( actualName ) )
-            throw Exception("JSON Parsing failed - provided NVP not found");
-        }
         else
-        {
+          itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), itsIteratorStack.back().value().MemberEnd());
-          // proceed as normal
-          if(itsIteratorStack.back().value().IsArray())
-            itsIteratorStack.push_back(itsIteratorStack.back().value().Begin());
-          else
-            itsIteratorStack.push_back(itsIteratorStack.back().value().MemberBegin());
-        }
       }
 
       //! Finishes the most recently started node
@@ -427,12 +458,13 @@ namespace cereal
       {
         itsIteratorStack.pop_back();
         ++itsIteratorStack.back();
+        std::cerr << "Finishing a node " << itsIteratorStack.size() << std::endl;
       }
 
       //! Sets the name for the next node created with startNode
       void setNextName( const char * name )
       {
-        itsIteratorStack.back().nextName = name;
+        itsNextName = name;
       }
 
       //! Loads a value from the current node - small signed overload
@@ -440,6 +472,8 @@ namespace cereal
       typename std::enable_if<std::is_signed<T>::value && sizeof(T) < sizeof(int64_t), void>::type
       loadValue(T & val)
       {
+        search();
+
         val = itsIteratorStack.back().value().GetInt();
         ++itsIteratorStack.back();
       }
@@ -450,22 +484,24 @@ namespace cereal
                               !std::is_same<bool, T>::value, void>::type
       loadValue(T & val)
       {
+        search();
+
         val = itsIteratorStack.back().value().GetUint();
         ++itsIteratorStack.back();
       }
 
       //! Loads a value from the current node - bool overload
-      void loadValue(bool & val)        { val = itsIteratorStack.back().value().GetBool();   ++itsIteratorStack.back(); }
+      void loadValue(bool & val)        { search(); val = itsIteratorStack.back().value().GetBool();   ++itsIteratorStack.back(); }
       //! Loads a value from the current node - int64 overload
-      void loadValue(int64_t & val)     { val = itsIteratorStack.back().value().GetInt64();  ++itsIteratorStack.back(); }
+      void loadValue(int64_t & val)     { search(); val = itsIteratorStack.back().value().GetInt64();  ++itsIteratorStack.back(); }
       //! Loads a value from the current node - uint64 overload
-      void loadValue(uint64_t & val)    { val = itsIteratorStack.back().value().GetUint64(); ++itsIteratorStack.back(); }
+      void loadValue(uint64_t & val)    { search(); val = itsIteratorStack.back().value().GetUint64(); ++itsIteratorStack.back(); }
       //! Loads a value from the current node - float overload
-      void loadValue(float & val)       { val = static_cast<float>(itsIteratorStack.back().value().GetDouble()); ++itsIteratorStack.back(); }
+      void loadValue(float & val)       { search(); val = static_cast<float>(itsIteratorStack.back().value().GetDouble()); ++itsIteratorStack.back(); }
       //! Loads a value from the current node - double overload
-      void loadValue(double & val)      { val = itsIteratorStack.back().value().GetDouble(); ++itsIteratorStack.back(); }
+      void loadValue(double & val)      { search(); val = itsIteratorStack.back().value().GetDouble(); ++itsIteratorStack.back(); }
       //! Loads a value from the current node - string overload
-      void loadValue(std::string & val) { val = itsIteratorStack.back().value().GetString(); ++itsIteratorStack.back(); }
+      void loadValue(std::string & val) { search(); val = itsIteratorStack.back().value().GetString(); ++itsIteratorStack.back(); }
 
       //! Loads a value from the current node - long double and long long overloads
       /*! These data types will automatically be encoded as base64 strings */
@@ -493,6 +529,7 @@ namespace cereal
       //! @}
 
     private:
+      const char * itsNextName;               //!< Next name set by NVP
       ReadStream itsReadStream;               //!< Rapidjson write stream
       std::vector<Iterator> itsIteratorStack; //!< 'Stack' of rapidJSON iterators
       rapidjson::Document itsDocument;        //!< Rapidjson document

include/cereal/external/rapidjson/document.h

@@ -614,6 +614,7 @@ private:
 		Array a;
 	};	// 12 bytes in 32-bit mode, 16 bytes in 64-bit mode
 
+public:
 	//! Find member by name.
 	Member* FindMember(const Ch* name) {
 		RAPIDJSON_ASSERT(name);
@@ -629,6 +630,7 @@ private:
 		return 0;
 	}
 	const Member* FindMember(const Ch* name) const { return const_cast<GenericValue&>(*this).FindMember(name); }
+private:
 
 	// Initialize this value as array with initial data, without calling destructor.
 	void SetArrayRaw(GenericValue* values, SizeType count, Allocator& alloctaor) {

sandbox_json.cpp

@@ -289,6 +289,38 @@ class Stuff
     }
 };
 
+struct OOJson
+{
+  OOJson() = default;
+  OOJson( int aa, int bb, bool cc, double dd ) :
+    a( aa ), b( bb ), c{ cc, dd }
+  {
+    d[0] = 0; d[1] = 1; d[2] = 2;
+  }
+
+  int a;
+  int b;
+  std::pair<bool, double> c;
+  float d[3];
+
+  template <class Archive>
+  void serialize( Archive & ar )
+  {
+    ar( CEREAL_NVP(c) );
+    ar( CEREAL_NVP(a) );
+    ar( b );
+    ar( CEREAL_NVP(d) );
+  }
+};
+
+template <class T>
+void ARRAY( T && a )
+{
+  std::cerr << "---------------" << std::endl;
+  std::cerr << std::is_array<typename std::remove_reference<T>::type>::value << std::endl;
+  std::cerr << std::is_array<T>::value << std::endl;
+}
+
 // ######################################################################
 int main()
 {
@@ -303,6 +335,7 @@ int main()
     //oar( f );
     //oar( f2 );
     Stuff s; s.fillData();
+    //ARRAY( s );
     oar( cereal::make_nvp("best data ever", s) );
   }
 
@@ -332,6 +365,66 @@ int main()
     archive( Bla::x );
   }
 
+  // test out of order
+  std::stringstream oos;
+  {
+    cereal::JSONOutputArchive ar(oos);
+    cereal::JSONOutputArchive ar2(std::cout);
+
+    ar( cereal::make_nvp( "1", 1 ),
+        cereal::make_nvp( "2", 2 ),
+        3,
+        0, // unused
+        cereal::make_nvp( "4", 4 ),
+        cereal::make_nvp( "5", 5 ) );
+
+    int x = 33;
+    ar.saveBinaryValue( &x, sizeof(int), "bla" );
+
+    ar2( cereal::make_nvp( "1", 1 ),
+         cereal::make_nvp( "2", 2 ),
+         3,
+         0, // unused
+         cereal::make_nvp( "4", 4 ),
+         cereal::make_nvp( "5", 5 ) );
+    ar2.saveBinaryValue( &x, sizeof(int), "bla" );
+
+    OOJson oo( 1, 2, 3, 4 );
+    ar( CEREAL_NVP(oo) );
+    ar2( CEREAL_NVP(oo) );
+
+    //int bla[][3] = {{1,2,3},{4,5,6}};
+    //ar2( cereal::make_nvp("asdf", bla) );
+    //ARRAY(asdf);
+  }
+
+  {
+    cereal::JSONInputArchive ar(oos);
+    int i1, i2, i3, i4, i5, x;
+
+    ar( i1 );
+    ar( cereal::make_nvp( "2", i2 ), i3 );
+
+    ar( cereal::make_nvp( "4", i4 ),
+        i5 );
+
+    ar.loadBinaryValue( &x, sizeof(int) );
+
+    OOJson ii;
+    ar( cereal::make_nvp("oo", ii) );
+    ar( cereal::make_nvp( "2", i2 ) );
+
+    std::cout << i1 << " " << i2 << " " << i3 << " " << i4 << " " << i5 << std::endl;
+    std::cout << x << std::endl;
+    std::cout << ii.a << " " << ii.b << " " << ii.c.first << " " << ii.c.second << " ";
+    for( auto z : ii.d )
+      std::cout << z << " ";
+    std::cout << std::endl;
+
+  }
+
+
+
 
   //{
   //  std::ifstream is("file.json");

unittests.cpp

@@ -3082,6 +3082,7 @@ void test_unordered_loads()
     std::ostringstream os;
     {
       OArchive oar(os);
+      OArchive oar2(std::cout);
 
       oar( cereal::make_nvp( name1, o_int1 ),
            cereal::make_nvp( name2, o_double2 ),
@@ -3090,6 +3091,14 @@ void test_unordered_loads()
            cereal::make_nvp( name5, o_int5 ),
            cereal::make_nvp( name6, o_int6 ),
            cereal::make_nvp( name7, o_un7 ) );
+
+      oar2( cereal::make_nvp( name1, o_int1 ),
+           cereal::make_nvp( name2, o_double2 ),
+           cereal::make_nvp( name3, o_vecbool3 ),
+           cereal::make_nvp( name4, o_int4 ),
+           cereal::make_nvp( name5, o_int5 ),
+           cereal::make_nvp( name6, o_int6 ),
+           cereal::make_nvp( name7, o_un7 ) );
     }
 
     decltype(o_int1) i_int1;
@@ -3114,7 +3123,7 @@ void test_unordered_loads()
     }
 
     BOOST_CHECK_EQUAL(o_int1, i_int1);
-    BOOST_CHECK_EQUAL(o_double2, i_double2);
+    BOOST_CHECK_CLOSE(o_double2 , o_double2, 1e-5);
     BOOST_CHECK_EQUAL(o_vecbool3.size(), i_vecbool3.size());
     BOOST_CHECK_EQUAL_COLLECTIONS(i_vecbool3.begin(),    i_vecbool3.end(),    o_vecbool3.begin(),  o_vecbool3.end());
     BOOST_CHECK_EQUAL(o_int4, i_int4);
@@ -3130,7 +3139,7 @@ BOOST_AUTO_TEST_CASE( xml_unordered_loads )
   test_unordered_loads<cereal::XMLInputArchive, cereal::XMLOutputArchive>();
 }
 
-//BOOST_AUTO_TEST_CASE( json_unordered_loads )
+BOOST_AUTO_TEST_CASE( json_unordered_loads )
-//{
+{
-//  test_unordered_loads<cereal::JSONInputArchive, cereal::JSONOutputArchive>();
+  test_unordered_loads<cereal::JSONInputArchive, cereal::JSONOutputArchive>();
-//}
+}