added API for storing OpenCV data structures to text string and reading them back

modules/core/doc/xml_yaml_persistence.rst

@@ -156,9 +156,9 @@ The constructors.
 
 .. ocv:function:: FileStorage::FileStorage()
 
-.. ocv:function:: FileStorage::FileStorage(const string& filename, int flags, const string& encoding=string())
+.. ocv:function:: FileStorage::FileStorage(const string& source, int flags, const string& encoding=string())
 
-    :param filename: Name of the file to open. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``.
+    :param source: Name of the file to open or the text string to read the data from. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``. If both ``FileStorage::WRITE`` and ``FileStorage::MEMORY`` flags are specified, ``source`` is used just to specify the output file format (e.g. ``mydata.xml``, ``.yml`` etc.).
 
     :param flags: Mode of operation. Possible values are:
 
@@ -168,6 +168,8 @@ The constructors.
 
         * **FileStorage::APPEND** Open the file for appending.
         
+        * **FileStorage::MEMORY** Read data from ``source`` or write data to the internal buffer (which is returned by ``FileStorage::release``)
+
     :param encoding: Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it.
 
 The full constructor opens the file. Alternatively you can use the default constructor and then call :ocv:func:`FileStorage::open`.
@@ -197,9 +199,9 @@ FileStorage::release
 --------------------
 Closes the file and releases all the memory buffers.
 
-.. ocv:function:: void FileStorage::release()
+.. ocv:function:: string FileStorage::release()
 
-Call this method after all I/O operations with the storage are finished.
+Call this method after all I/O operations with the storage are finished. If the storage was opened for writing data and ``FileStorage::WRITE`` was specified
 
 
 FileStorage::getFirstTopLevelNode

modules/core/include/opencv2/core/core.hpp

@@ -3941,7 +3941,12 @@ public:
     {
         READ=0, //! read mode
         WRITE=1, //! write mode
-        APPEND=2 //! append mode
+        APPEND=2, //! append mode
+        MEMORY=4,
+        FORMAT_MASK=(7<<3),
+        FORMAT_AUTO=0,
+        FORMAT_XML=(1<<3),
+        FORMAT_YAML=(2<<3)
     };
     enum
     {
@@ -3953,7 +3958,7 @@ public:
     //! the default constructor
     CV_WRAP FileStorage();
     //! the full constructor that opens file storage for reading or writing
-    CV_WRAP FileStorage(const string& filename, int flags, const string& encoding=string());
+    CV_WRAP FileStorage(const string& source, int flags, const string& encoding=string());
     //! the constructor that takes pointer to the C FileStorage structure
     FileStorage(CvFileStorage* fs);
     //! the destructor. calls release()
@@ -3964,7 +3969,7 @@ public:
     //! returns true if the object is associated with currently opened file.
     CV_WRAP virtual bool isOpened() const;
     //! closes the file and releases all the memory buffers
-    CV_WRAP virtual void release();
+    CV_WRAP virtual string release();
 
     //! returns the first element of the top-level mapping
     CV_WRAP FileNode getFirstTopLevelNode() const;

modules/core/include/opencv2/core/types_c.h

@@ -1740,6 +1740,11 @@ typedef struct CvFileStorage CvFileStorage;
 #define CV_STORAGE_WRITE_TEXT    CV_STORAGE_WRITE
 #define CV_STORAGE_WRITE_BINARY  CV_STORAGE_WRITE
 #define CV_STORAGE_APPEND        2
+#define CV_STORAGE_MEMORY        4
+#define CV_STORAGE_FORMAT_MASK   (7<<3)
+#define CV_STORAGE_FORMAT_AUTO   0
+#define CV_STORAGE_FORMAT_XML    8
+#define CV_STORAGE_FORMAT_YAML  16
 
 /* List of attributes: */
 typedef struct CvAttrList

modules/core/src/persistence.cpp

@@ -41,7 +41,10 @@
 //M*/
 
 #include "precomp.hpp"
+
 #include <ctype.h>
+#include <deque>
+#include <iterator>
 #include <wchar.h>
 #include <zlib.h>
 
@@ -208,7 +211,7 @@ typedef void (*CvStartNextStream)( struct CvFileStorage* fs );
 typedef struct CvFileStorage
 {
     int flags;
-    int is_xml;
+    int fmt;
     int write_mode;
     int is_first;
     CvMemStorage* memstorage;
@@ -240,52 +243,85 @@ typedef struct CvFileStorage
     CvWriteString write_string;
     CvWriteComment write_comment;
     CvStartNextStream start_next_stream;
-    //CvParse parse;
+    
+    const char* strbuf;
+    size_t strbufsize, strbufpos;
+    std::deque<char>* outbuf;
+    
+    bool is_opened;
 }
 CvFileStorage;
 
 static void icvPuts( CvFileStorage* fs, const char* str )
 {
-    CV_Assert( fs->file || fs->gzfile );
+    if( fs->outbuf )
-    if( fs->file )
+        std::copy(str, str + strlen(str), std::back_inserter(*fs->outbuf));
+    else if( fs->file )
         fputs( str, fs->file );
-    else
+    else if( fs->gzfile )
         gzputs( fs->gzfile, str );
+    else
+        CV_Error( CV_StsError, "The storage is not opened" );
 }
 
 static char* icvGets( CvFileStorage* fs, char* str, int maxCount )
 {
-    CV_Assert( fs->file || fs->gzfile );
+    if( fs->strbuf )
+    {
+        size_t i = fs->strbufpos, len = fs->strbufsize, j = 0;
+        const char* instr = fs->strbuf;
+        while( i < len && j < maxCount-1 )
+        {
+            char c = instr[i++];
+            if( c == '\0' )
+                break;
+            str[j++] = c;
+            if( c == '\n' )
+                break;
+        }
+        str[j++] = '\0';
+        fs->strbufpos = i;
+        return j > 1 ? str : 0;
+    }
     if( fs->file )
         return fgets( str, maxCount, fs->file );
+    if( fs->gzfile )
         return gzgets( fs->gzfile, str, maxCount );
+    CV_Error( CV_StsError, "The storage is not opened" );
+    return 0;
 }
 
 static int icvEof( CvFileStorage* fs )
 {
-    CV_Assert( fs->file || fs->gzfile );
+    if( fs->strbuf )
+        return fs->strbufpos >= fs->strbufsize;
     if( fs->file )
         return feof(fs->file);
+    if( fs->gzfile )
         return gzeof(fs->gzfile);
+    return false;
 }
 
-static void icvClose( CvFileStorage* fs )
+static void icvCloseFile( CvFileStorage* fs )
 {
     if( fs->file )
         fclose( fs->file );
-    if( fs->gzfile )
+    else if( fs->gzfile )
         gzclose( fs->gzfile );
     fs->file = 0;
     fs->gzfile = 0;
+    fs->strbuf = 0;
+    fs->strbufpos = 0;
+    fs->is_opened = false;
 }
 
 static void icvRewind( CvFileStorage* fs )
 {
-    CV_Assert( fs->file || fs->gzfile );
     if( fs->file )
         rewind(fs->file);
-    else
+    else if( fs->gzfile )
         gzrewind(fs->gzfile);
+    fs->strbufpos = 0;
 }
 
 #define CV_YML_INDENT  3
@@ -373,7 +409,7 @@ icvFSCreateCollection( CvFileStorage* fs, int tag, CvFileNode* collection )
     {
         if( collection->tag != CV_NODE_NONE )
         {
-            assert( fs->is_xml != 0 );
+            assert( fs->fmt == CV_STORAGE_FORMAT_XML );
             CV_PARSE_ERROR( "Sequence element should not have name (use <_></_>)" );
         }
 
@@ -477,6 +513,40 @@ icvFSFlush( CvFileStorage* fs )
 }
 
 
+static void
+icvClose( CvFileStorage* fs, std::string* out )
+{
+    if( out )
+        out->clear();
+    
+    if( !fs )
+        CV_Error( CV_StsNullPtr, "NULL double pointer to file storage" );
+    
+    if( fs->is_opened )
+    {
+        if( fs->write_mode && (fs->file || fs->gzfile || fs->outbuf) )
+        {
+            if( fs->write_stack )
+            {
+                while( fs->write_stack->total > 0 )
+                    cvEndWriteStruct(fs);
+            }
+            icvFSFlush(fs);
+            if( fs->fmt == CV_STORAGE_FORMAT_XML )
+                icvPuts( fs, "</opencv_storage>\n" );
+        }
+        
+        icvCloseFile(fs);
+    }
+        
+    if( fs->outbuf && out )
+    {
+        out->resize(fs->outbuf->size());
+        std::copy(fs->outbuf->begin(), fs->outbuf->end(), out->begin());
+    }
+}
+
+
 /* closes file storage and deallocates buffers */
 CV_IMPL  void
 cvReleaseFileStorage( CvFileStorage** p_fs )
@@ -489,27 +559,15 @@ cvReleaseFileStorage( CvFileStorage** p_fs )
         CvFileStorage* fs = *p_fs;
         *p_fs = 0;
         
-        if( fs->write_mode && (fs->file || fs->gzfile) )
+        icvClose(fs, 0);
-        {
-            if( fs->write_stack )
-            {
-                while( fs->write_stack->total > 0 )
-                    cvEndWriteStruct(fs);
-            }
-            icvFSFlush(fs);
-            if( fs->is_xml )
-                icvPuts( fs, "</opencv_storage>\n" );
-        }
-
-        //icvFSReleaseCollection( fs->roots ); // delete all the user types recursively
-
-        icvClose(fs);
         
         cvReleaseMemStorage( &fs->strstorage );        
-
         cvFree( &fs->buffer_start );
         cvReleaseMemStorage( &fs->memstorage );
         
+        if( fs->outbuf )
+            delete fs->outbuf;
+        
         memset( fs, 0, sizeof(*fs) );
         cvFree( &fs );
     }
@@ -2601,10 +2659,22 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
     char* xml_buf = 0;
     int default_block_size = 1 << 18;
     bool append = (flags & 3) == CV_STORAGE_APPEND;
+    bool mem = (flags & CV_STORAGE_MEMORY) != 0;
+    bool write_mode = (flags & 3) != 0;
     bool isGZ = false;
+    size_t fnamelen = 0;
 
-    if( !filename )
+    if( !filename || filename[0] == '\0' )
-        CV_Error( CV_StsNullPtr, "NULL filename" );
+    {
+        if( !write_mode )
+            CV_Error( CV_StsNullPtr, mem ? "NULL or empty filename" : "NULL or empty buffer" );
+        mem = true;
+    }
+    else
+        fnamelen = strlen(filename);
+    
+    if( mem && append )
+        CV_Error( CV_StsBadFlag, "CV_STORAGE_APPEND and CV_STORAGE_MEMORY are not currently compatible" );
 
     fs = (CvFileStorage*)cvAlloc( sizeof(*fs) );
     memset( fs, 0, sizeof(*fs));
@@ -2612,10 +2682,11 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
     fs->memstorage = cvCreateMemStorage( default_block_size );
     fs->dststorage = dststorage ? dststorage : fs->memstorage;
 
-    int fnamelen = (int)strlen(filename);
+    fs->flags = CV_FILE_STORAGE;
-    if( !fnamelen )
+    fs->write_mode = write_mode;
-        CV_Error( CV_StsError, "Empty filename" );
     
+    if( !mem )
+    {
         fs->filename = (char*)cvMemStorageAlloc( fs->memstorage, fnamelen+1 );
         strcpy( fs->filename, filename );
 
@@ -2633,9 +2704,6 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
                 dot_pos[3] = '\0', fnamelen--;
         }
 
-    fs->flags = CV_FILE_STORAGE;
-    fs->write_mode = (flags & 3) != 0;
-
         if( !isGZ )
         {
             fs->file = fopen(fs->filename, !fs->write_mode ? "rt" : !append ? "wt" : "a+t" );
@@ -2649,6 +2717,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
             if( !fs->gzfile )
                 goto _exit_;
         }
+    }
     
     fs->roots = 0;
     fs->struct_indent = 0;
@@ -2657,27 +2726,38 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
 
     if( fs->write_mode )
     {
+        int fmt = flags & CV_STORAGE_FORMAT_MASK;
+        
+        if( mem )
+            fs->outbuf = new std::deque<char>;
+        
+        if( fmt == CV_STORAGE_FORMAT_AUTO && filename )
+        {
+            const char* dot_pos = filename + fnamelen - (isGZ ? 7 : 4);
+            fs->fmt = (dot_pos >= filename && (memcmp( dot_pos, ".xml", 4) == 0 ||
+                    memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0)) ?
+                CV_STORAGE_FORMAT_XML : CV_STORAGE_FORMAT_YAML;
+        }
+        else
+            fs->fmt = fmt != CV_STORAGE_FORMAT_AUTO ? fmt : CV_STORAGE_FORMAT_XML;
+        
         // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (&apos; and &quot;)
         // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB))
-        int buf_size = CV_FS_MAX_LEN*(fs->is_xml ? 6 : 4) + 1024;
+        int buf_size = CV_FS_MAX_LEN*(fs->fmt == CV_STORAGE_FORMAT_XML ? 6 : 4) + 1024;
-
-        dot_pos = fs->filename + fnamelen - (isGZ ? 7 : 4);
-        fs->is_xml = dot_pos > fs->filename && (memcmp( dot_pos, ".xml", 4) == 0 ||
-                      memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0);
 
         if( append )
             fseek( fs->file, 0, SEEK_END );
 
-        fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->is_xml ?
+        fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->fmt == CV_STORAGE_FORMAT_XML ?
                 sizeof(CvXMLStackRecord) : sizeof(int), fs->memstorage );
         fs->is_first = 1;
         fs->struct_indent = 0;
         fs->struct_flags = CV_NODE_EMPTY;
         fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 );
         fs->buffer_end = fs->buffer_start + buf_size;
-        if( fs->is_xml )
+        if( fs->fmt == CV_STORAGE_FORMAT_XML )
         {
-            int file_size = fs->file ? (int)ftell( fs->file ) : 0;
+            size_t file_size = fs->file ? ftell( fs->file ) : (size_t)0;
             fs->strstorage = cvCreateChildMemStorage( fs->memstorage );
             if( !append || file_size == 0 )
             {
@@ -2724,7 +2804,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
                 }
                 if( last_occurence < 0 )
                     CV_Error( CV_StsError, "Could not find </opencv_storage> in the end of file.\n" );
-                icvClose( fs );
+                icvCloseFile( fs );
                 fs->file = fopen( fs->filename, "r+t" );
                 fseek( fs->file, last_occurence, SEEK_SET );
                 // replace the last "</opencv_storage>" with " <!-- resumed -->", which has the same length
@@ -2757,18 +2837,30 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
     }
     else
     {
-        int buf_size = 1 << 20;
+        if( mem )
+        {
+            fs->strbuf = filename;
+            fs->strbufsize = fnamelen;
+        }
+        
+        size_t buf_size = 1 << 20;
         const char* yaml_signature = "%YAML:";
         char buf[16];
         icvGets( fs, buf, sizeof(buf)-2 );
-        fs->is_xml = strncmp( buf, yaml_signature, strlen(yaml_signature) ) != 0;
+        fs->fmt = strncmp( buf, yaml_signature, strlen(yaml_signature) ) == 0 ?
+            CV_STORAGE_FORMAT_YAML : CV_STORAGE_FORMAT_XML;
 
         if( !isGZ )
+        {
+            if( !mem )
             {
                 fseek( fs->file, 0, SEEK_END );
                 buf_size = ftell( fs->file );
-            buf_size = MIN( buf_size, (1 << 20) );
+            }
-            buf_size = MAX( buf_size, CV_FS_MAX_LEN*2 + 1024 );
+            else
+                buf_size = fs->strbufsize;
+            buf_size = MIN( buf_size, (size_t)(1 << 20) );
+            buf_size = MAX( buf_size, (size_t)(CV_FS_MAX_LEN*2 + 1024) );
         }
         icvRewind(fs);
 
@@ -2785,7 +2877,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
 
         //mode = cvGetErrMode();
         //cvSetErrMode( CV_ErrModeSilent );
-        if( fs->is_xml )
+        if( fs->fmt == CV_STORAGE_FORMAT_XML )
             icvXMLParse( fs );
         else
             icvYMLParse( fs );
@@ -2795,16 +2887,21 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
         cvFree( &fs->buffer_start );
         fs->buffer = fs->buffer_end = 0;
     }
+    fs->is_opened = true;
+    
 _exit_:
     if( fs )
     {
-        if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile) )
+        if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile && !fs->outbuf && !fs->strbuf) )
         {
             cvReleaseFileStorage( &fs );
         }
         else if( !fs->write_mode )
         {
-            icvClose(fs);
+            icvCloseFile(fs);
+            // we close the file since it's not needed anymore. But icvCloseFile() resets is_opened,
+            // which may be misleading. Since we restore the value of is_opened.
+            fs->is_opened = true;
         }
     }
 
@@ -3057,7 +3154,7 @@ cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt )
                     return;
                 }
 
-                if( fs->is_xml )
+                if( fs->fmt == CV_STORAGE_FORMAT_XML )
                 {
                     int buf_len = (int)strlen(ptr);
                     icvXMLWriteScalar( fs, 0, ptr, buf_len );
@@ -5054,14 +5151,20 @@ bool FileStorage::open(const string& filename, int flags, const string& encoding
 
 bool FileStorage::isOpened() const
 {
-    return !fs.empty();
+    return !fs.empty() && fs.obj->is_opened;
 }
 
-void FileStorage::release()
+string FileStorage::release()
 {
+    string buf;
+    
+    if( fs.obj && fs.obj->outbuf )
+        icvClose(fs.obj, &buf); 
+        
     fs.release();
     structs.clear();
     state = UNDEFINED;
+    return buf;
 }
 
 FileNode FileStorage::root(int streamidx) const

modules/core/test/test_io.cpp

@@ -91,7 +91,7 @@ protected:
             {-1000000, 1000000}, {-10, 10}, {-10, 10}};
         RNG& rng = ts->get_rng();
         RNG rng0;
-        test_case_count = 2;
+        test_case_count = 4;
         int progress = 0;
         MemStorage storage(cvCreateMemStorage(0));
         
@@ -102,9 +102,10 @@ protected:
             
             cvClearMemStorage(storage);
             
+            bool mem = (idx % 4) >= 2;
             string filename = tempfile(idx % 2 ? ".yml" : ".xml");
             
-            FileStorage fs(filename.c_str(), FileStorage::WRITE);
+            FileStorage fs(filename, FileStorage::WRITE + (mem ? FileStorage::MEMORY : 0));
             
             int test_int = (int)cvtest::randInt(rng);
             double test_real = (cvtest::randInt(rng)%2?1:-1)*exp(cvtest::randReal(rng)*18-9);
@@ -179,11 +180,11 @@ protected:
             fs.writeObj("test_graph",graph);
             CvGraph* graph2 = (CvGraph*)cvClone(graph);
             
-            fs.release();
+            string content = fs.release();
             
-            if(!fs.open(filename.c_str(), FileStorage::READ))
+            if(!fs.open(mem ? content : filename, FileStorage::READ + (mem ? FileStorage::MEMORY : 0)))
             {
-                ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", filename.c_str() );
+                ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", !mem ? filename.c_str() : content.c_str());
                 ts->set_failed_test_info( cvtest::TS::FAIL_MISSING_TEST_DATA );
                 return;
             }
@@ -310,7 +311,6 @@ protected:
             int real_width = (int)tm["width"];
             int real_height = (int)tm["height"];
             
-            
             int real_lbp_val = 0;
             FileNodeIterator it;
             it = tm_lbp.begin();
@@ -370,6 +370,7 @@ protected:
             }
             
             fs.release();
+            if( !mem )
                 remove(filename.c_str());
         }
     }