Commit missing files

git-svn-id: https://rapidjson.googlecode.com/svn/trunk@41 c5894555-1306-4e8d-425f-1f6f381ee07c

include/rapidjson/allocators.h

@@ -0,0 +1,219 @@
+#ifndef RAPIDJSON_ALLOCATORS_H_
+#define RAPIDJSON_ALLOCATORS_H_
+
+#include "rapidjson.h"
+
+namespace rapidjson {
+
+///////////////////////////////////////////////////////////////////////////////
+// Allocator
+
+/*! \class rapidjson::Allocator
+	\brief Concept for allocating, resizing and freeing memory block.
+	
+	Note that Malloc() and Realloc() are non-static but Free() is static.
+	
+	So if an allocator need to support Free(), it needs to put its pointer in 
+	the header of memory block.
+
+\code
+concept Allocator {
+	static const bool kNeedFree;	//!< Whether this allocator needs to call Free().
+
+	// Allocate a memory block.
+	// \param size of the memory block in bytes.
+	// \returns pointer to the memory block.
+	void* Malloc(size_t size);
+
+	// Resize a memory block.
+	// \param originalPtr The pointer to current memory block. Null pointer is permitted.
+	// \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.)
+	// \param newSize the new size in bytes.
+	void* Realloc(void* originalPtr, size_t originalSize, size_t newSize);
+
+	// Free a memory block.
+	// \param pointer to the memory block. Null pointer is permitted.
+	static void Free(void *ptr);
+};
+\endcode
+*/
+
+///////////////////////////////////////////////////////////////////////////////
+// CrtAllocator
+
+//! C-runtime library allocator.
+/*! This class is just wrapper for standard C library memory routines.
+	\implements Allocator
+*/
+class CrtAllocator {
+public:
+	static const bool kNeedFree = true;
+	void* Malloc(size_t size) { return malloc(size); }
+	void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { return realloc(originalPtr, newSize); }
+	static void Free(void *ptr) { free(ptr); }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// MemoryPoolAllocator
+
+//! Default memory allocator used by the parser and DOM.
+/*! This allocator allocate memory blocks from pre-allocated memory chunks. 
+
+    It does not free memory blocks. And Realloc() only allocate new memory.
+
+    The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default.
+
+    User may also supply a buffer as the first chunk.
+
+    If the user-buffer is full then additional chunks are allocated by BaseAllocator.
+
+    The user-buffer is not deallocated by this allocator.
+
+    \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator.
+	\implements Allocator
+*/
+template <typename BaseAllocator = CrtAllocator>
+class MemoryPoolAllocator {
+public:
+	static const bool kNeedFree = false;	//!< Tell users that no need to call Free() with this allocator. (concept Allocator)
+
+	//! Constructor with chunkSize.
+	/*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize.
+		\param baseAllocator The allocator for allocating memory chunks.
+	*/
+	MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : 
+		chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+	{
+		if (!baseAllocator_)
+			ownBaseAllocator_ = baseAllocator_ = new BaseAllocator();
+		AddChunk(chunk_capacity_);
+	}
+
+	//! Constructor with user-supplied buffer.
+	/*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size.
+
+		The user buffer will not be deallocated when this allocator is destructed.
+
+		\param buffer User supplied buffer.
+		\param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader).
+		\param chunkSize The size of memory chunk. The default is kDefaultChunkSize.
+		\param baseAllocator The allocator for allocating memory chunks.
+	*/
+	MemoryPoolAllocator(char *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) :
+		chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+	{
+		RAPIDJSON_ASSERT(buffer != 0);
+		RAPIDJSON_ASSERT(size > sizeof(ChunkHeader));
+		chunkHead_ = (ChunkHeader*)buffer;
+		chunkHead_->capacity = size - sizeof(ChunkHeader);
+		chunkHead_->size = 0;
+		chunkHead_->next = 0;
+	}
+
+	//! Destructor.
+	/*! This deallocates all memory chunks, excluding the user-supplied buffer.
+	*/
+	~MemoryPoolAllocator() {
+		Clear();
+		delete ownBaseAllocator_;
+	}
+
+	//! Deallocates all memory chunks, excluding the user-supplied buffer.
+	void Clear() {
+		while(chunkHead_ != 0 && chunkHead_ != (ChunkHeader *)userBuffer_) {
+			ChunkHeader* next = chunkHead_->next;
+			baseAllocator_->Free(chunkHead_);
+			chunkHead_ = next;
+		}
+	}
+
+	//! Computes the total capacity of allocated memory chunks.
+	/*! \return total capacity in bytes.
+	*/
+	size_t Capacity() {
+		size_t capacity = 0;
+		for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+			capacity += c->capacity;
+		return capacity;
+	}
+
+	//! Computes the memory blocks allocated.
+	/*! \return total used bytes.
+	*/
+	size_t Size() {
+		size_t size = 0;
+		for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+			size += c->size;
+		return size;
+	}
+
+	//! Allocates a memory block. (concept Allocator)
+	void* Malloc(size_t size) {
+		if (chunkHead_->size + size > chunkHead_->capacity)
+			AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size);
+
+		char *buffer = (char *)(chunkHead_ + 1) + chunkHead_->size;
+		chunkHead_->size += size;
+		return buffer;
+	}
+
+	//! Resizes a memory block (concept Allocator)
+	void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) {
+		if (originalPtr == 0)
+			return Malloc(newSize);
+
+		// Do not shrink if new size is smaller than original
+		if (originalSize >= newSize)
+			return originalPtr;
+
+		// Simply expand it if it is the last allocation and there is sufficient space
+		if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) {
+			size_t increment = newSize - originalSize;
+			if (chunkHead_->size + increment <= chunkHead_->capacity) {
+				chunkHead_->size += increment;
+				return originalPtr;
+			}
+		}
+
+		// Realloc process: allocate and copy memory, do not free original buffer.
+		void* newBuffer = Malloc(newSize);
+		RAPIDJSON_ASSERT(newBuffer != 0);	// Do not handle out-of-memory explicitly.
+		return memcpy(newBuffer, originalPtr, originalSize);
+	}
+
+	//! Frees a memory block (concept Allocator)
+	static void Free(void *ptr) {} // Do nothing
+
+private:
+	//! Creates a new chunk.
+	/*! \param capacity Capacity of the chunk in bytes.
+	*/
+	void AddChunk(size_t capacity) {
+		ChunkHeader* chunk = (ChunkHeader*)baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity);
+		chunk->capacity = capacity;
+		chunk->size = 0;
+		chunk->next = chunkHead_;
+		chunkHead_ =  chunk;
+	}
+
+	static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity.
+
+	//! Chunk header for perpending to each chunk.
+	/*! Chunks are stored as a singly linked list.
+	*/
+	struct ChunkHeader {
+		size_t capacity;	//!< Capacity of the chunk in bytes (excluding the header itself).
+		size_t size;		//!< Current size of allocated memory in bytes.
+		ChunkHeader *next;	//!< Next chunk in the linked list.
+	};
+
+	ChunkHeader *chunkHead_;	//!< Head of the chunk linked-list. Only the head chunk serves allocation.
+	size_t chunk_capacity_;		//!< The minimum capacity of chunk when they are allocated.
+	char *userBuffer_;			//!< User supplied buffer.
+	BaseAllocator* baseAllocator_;	//!< base allocator for allocating memory chunks.
+	BaseAllocator* ownBaseAllocator_;	//!< base allocator created by this object.
+};
+
+} // namespace rapidjson
+
+#endif // RAPIDJSON_ENCODINGS_H_

test/unittest/encodingstest.cpp

@@ -0,0 +1,122 @@
+#include "unittest.h"
+#include "rapidjson/filereadstream.h"
+#include "rapidjson/filewritestream.h"
+#include "rapidjson/encodedstream.h"
+
+using namespace rapidjson;
+
+class EncodingsTest : public ::testing::Test {
+public:
+	FILE* Open(const char* filename) {
+		char buffer[1024];
+		sprintf(buffer, "encodings/%s", filename);
+		FILE *fp = fopen(buffer, "rb");
+		if (!fp) {
+			sprintf(buffer, "../../bin/encodings/%s", filename);
+			fp = fopen(buffer, "rb");
+		}
+		return fp;
+	}
+
+	virtual void SetUp() {
+		FILE *fp = Open("utf8.json");
+		ASSERT_TRUE(fp != 0);
+
+		fseek(fp, 0, SEEK_END);
+		length_ = (size_t)ftell(fp);
+		fseek(fp, 0, SEEK_SET);
+		json_ = (char*)malloc(length_ + 1);
+		fread(json_, 1, length_, fp);
+		json_[length_] = '\0';
+		fclose(fp);
+	}
+
+	virtual void TearDown() {
+		free(json_);
+	}
+
+protected:
+	const char* filename_;
+	char *json_;
+	size_t length_;
+};
+
+TEST_F(EncodingsTest, EncodedInputStream_UTF8BOM) {
+	char buffer[16];
+	FILE *fp = Open("utf8bom.json");
+	ASSERT_TRUE(fp != 0);
+	FileReadStream fs(fp, buffer, sizeof(buffer));
+	EncodedInputStream<UTF8<>, FileReadStream> eis(fs);
+	StringStream s(json_);
+
+	while (eis.Peek() != '\0') {
+		unsigned expected, actual;
+		UTF8<>::Decode(s, &expected);
+		UTF8<>::Decode(eis, &actual);
+		EXPECT_EQ(expected, actual);
+	}
+	EXPECT_EQ('\0', s.Peek());
+	fclose(fp);
+}
+
+TEST_F(EncodingsTest, EncodedInputStream_UTF16LEBOM) {
+	char buffer[16];
+	FILE *fp = Open("utf16lebom.json");
+	ASSERT_TRUE(fp != 0);
+	FileReadStream fs(fp, buffer, sizeof(buffer));
+	EncodedInputStream<UTF16LE<>, FileReadStream> eis(fs);
+	StringStream s(json_);
+
+	while (eis.Peek() != '\0') {
+		unsigned expected, actual;
+		UTF8<>::Decode(s, &expected);
+		UTF16<>::Decode(eis, &actual);
+		EXPECT_EQ(expected, actual);
+	}
+	EXPECT_EQ('\0', s.Peek());
+	fclose(fp);
+}
+
+TEST_F(EncodingsTest, EncodedInputStream_UTF16BEBOM) {
+	char buffer[16];
+	FILE *fp = Open("utf16bebom.json");
+	ASSERT_TRUE(fp != 0);
+	FileReadStream fs(fp, buffer, sizeof(buffer));
+	EncodedInputStream<UTF16BE<>, FileReadStream> eis(fs);
+	StringStream s(json_);
+
+	while (eis.Peek() != '\0') {
+		unsigned expected, actual;
+		UTF8<>::Decode(s, &expected);
+		UTF16<>::Decode(eis, &actual);
+		EXPECT_EQ(expected, actual);
+	}
+	EXPECT_EQ('\0', s.Peek());
+	fclose(fp);
+}
+
+TEST_F(EncodingsTest, AutoUTFInputStream) {
+#define TEST_FILE(filename) \
+	{ \
+		char buffer[16]; \
+		FILE *fp = Open(filename); \
+		ASSERT_TRUE(fp != 0); \
+		FileReadStream fs(fp, buffer, sizeof(buffer)); \
+		AutoUTFInputStream<wchar_t, FileReadStream> eis(fs); \
+		StringStream s(json_); \
+		while (eis.Peek() != '\0') { \
+			unsigned expected, actual; \
+			UTF8<>::Decode(s, &expected); \
+			AutoUTF<wchar_t>::Decode(eis, &actual); \
+			EXPECT_EQ(expected, actual); \
+		} \
+		EXPECT_EQ('\0', s.Peek()); \
+		fclose(fp); \
+	}
+
+	TEST_FILE("utf8.json");
+	TEST_FILE("utf8bom.json");
+	TEST_FILE("utf16lebom.json");
+	TEST_FILE("utf16bebom.json");
+#undef TEST_FILE
+}