submitted 1.2.0

Foundation/include/Poco/StreamTokenizer.h

@@ -0,0 +1,119 @@
+//
+// StreamTokenizer.h
+//
+// $Id: //poco/1.2/Foundation/include/Poco/StreamTokenizer.h#1 $
+//
+// Library: Foundation
+// Package: Streams
+// Module:  StreamTokenizer
+//
+// Definition of the StreamTokenizer class.
+//
+// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+// 
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+//
+
+
+#ifndef Foundation_StreamTokenizer_INCLUDED
+#define Foundation_StreamTokenizer_INCLUDED
+
+
+#include "Poco/Foundation.h"
+#include "Poco/Token.h"
+#include <istream>
+#include <vector>
+
+
+namespace Poco {
+
+
+class Foundation_API StreamTokenizer
+	/// A stream tokenizer splits an input stream
+	/// into a sequence of tokens of different kinds.
+	/// Various token kinds can be registered with
+	/// the tokenizer.
+{
+public:
+	StreamTokenizer();
+		/// Creates a StreamTokenizer with no attached stream.
+
+	StreamTokenizer(std::istream& istr);
+		/// Creates a StreamTokenizer with no attached stream.
+
+	virtual ~StreamTokenizer();
+		/// Destroys the StreamTokenizer and deletes all
+		/// registered tokens.
+
+	void attachToStream(std::istream& istr);
+		/// Attaches the tokenizer to an input stream.
+
+	void addToken(Token* pToken);
+		/// Adds a token class to the tokenizer. The
+		/// tokenizer takes ownership of the token and
+		/// deletes it when no longer needed. Comment
+		/// and whitespace tokens will be marked as
+		/// ignorable, which means that next() will not
+		/// return them.
+	
+	void addToken(Token* pToken, bool ignore);
+		/// Adds a token class to the tokenizer. The
+		/// tokenizer takes ownership of the token and
+		/// deletes it when no longer needed.
+		/// If ignore is true, the token will be marked
+		/// as ignorable, which means that next() will
+		/// not return it.
+		
+	const Token* next();
+		/// Extracts the next token from the input stream.
+		/// Returns a pointer to an EOFToken if there are
+		/// no more characters to read. 
+		/// Returns a pointer to an InvalidToken if an
+		/// invalid character is encountered.
+		/// If a token is marked as ignorable, it will not
+		/// be returned, and the next token will be
+		/// examined.
+		/// Never returns a NULL pointer.
+		/// You must not delete the token returned by next().
+
+private:
+	struct TokenInfo
+	{
+		Token* pToken;
+		bool   ignore;
+	};
+	
+	typedef std::vector<TokenInfo> TokenVec;
+	
+	TokenVec      _tokens;
+	std::istream* _pIstr;
+	InvalidToken  _invalidToken;
+	EOFToken      _eofToken;
+};
+
+
+} // namespace Poco
+
+
+#endif // Foundation_StreamTokenizer_INCLUDED