fixed GH #1141: Poco::StringTokenizer::TOK_TRIM changes behavior between 1.4 and 1.6

Conflicts:
	Foundation/include/Poco/StringTokenizer.h
	Foundation/src/StringTokenizer.cpp

Foundation/include/Poco/StringTokenizer.h

@@ -68,12 +68,12 @@ public:
 	bool has(const std::string& token) const;
 		/// Returns true if token exists, false otherwise.
 
-	std::size_t find(const std::string& token, std::size_t pos = 0) const;
+	std::string::size_type find(const std::string& token, std::string::size_type pos = 0) const;
-		/// Returns the index of the first occurrence of the token
+		/// Returns the index of the first occurence of the token
 		/// starting at position pos.
 		/// Throws a NotFoundException if the token is not found.
 
-	std::size_t replace(const std::string& oldToken, const std::string& newToken, std::size_t pos = 0);
+	std::size_t replace(const std::string& oldToken, const std::string& newToken, std::string::size_type pos = 0);
 		/// Starting at position pos, replaces all subsequent tokens having value 
 		/// equal to oldToken with newToken.
 		/// Returns the number of modified tokens.
@@ -88,7 +88,7 @@ private:
 	StringTokenizer(const StringTokenizer&);
 	StringTokenizer& operator = (const StringTokenizer&);
 	
-	void trim (std::string& token);
+	void trim(std::string& token);
 
 	TokenVec _tokens;
 };

Foundation/src/StringTokenizer.cpp

@@ -16,9 +16,9 @@
 
 #include "Poco/StringTokenizer.h"
 #include "Poco/Ascii.h"
-
 #include <algorithm>
 
+
 namespace Poco {
 
 
@@ -31,14 +31,14 @@ StringTokenizer::StringTokenizer(const std::string& str, const std::string& sepa
 	bool ignoreEmpty = ((options & TOK_IGNORE_EMPTY) != 0);
 	bool lastToken = false;
 
-	for (;it != itEnd; ++it)
+	for (; it != itEnd; ++it)
 	{
 		if (separators.find(*it) != std::string::npos) 
 		{
 			if (doTrim) trim(token);
-			if (!token.empty() || !ignoreEmpty)_tokens.push_back(token);
+			if (!token.empty() || !ignoreEmpty) _tokens.push_back(token);
 			if (!ignoreEmpty) lastToken = true;
-			token = "";
+			token.clear();
 		}
 		else
 		{
@@ -50,9 +50,12 @@ StringTokenizer::StringTokenizer(const std::string& str, const std::string& sepa
 	if (!token.empty())
 	{
 		if (doTrim) trim(token);
-		if (!token.empty()) _tokens.push_back(token);
+		if (!token.empty() || !ignoreEmpty) _tokens.push_back(token);
+	}
+	else if (lastToken) 
+	{
+		_tokens.push_back(std::string());
 	}
-	else if (lastToken) _tokens.push_back("");
 }
 
 
@@ -61,9 +64,11 @@ StringTokenizer::~StringTokenizer()
 }
 
 
-void StringTokenizer::trim (std::string& token)
+void StringTokenizer::trim(std::string& token)
 {
-	std::size_t front = 0, back = 0, length = token.length();
+	std::string::size_type front = 0;
+	std::string::size_type back = 0;
+	std::string::size_type length = token.length();
 	std::string::const_iterator tIt = token.begin();
 	std::string::const_iterator tEnd = token.end();
 	for (; tIt != tEnd; ++tIt, ++front)
@@ -87,7 +92,7 @@ std::size_t StringTokenizer::count(const std::string& token) const
 {
 	std::size_t result = 0;
 	TokenVec::const_iterator it = std::find(_tokens.begin(), _tokens.end(), token);
-	while(it != _tokens.end())
+	while (it != _tokens.end())
 	{
 		result++;
 		it = std::find(++it, _tokens.end(), token);
@@ -96,27 +101,29 @@ std::size_t StringTokenizer::count(const std::string& token) const
 }
 
 
-std::size_t StringTokenizer::find(const std::string& token, std::size_t pos) const
+std::string::size_type StringTokenizer::find(const std::string& token, std::string::size_type pos) const
 {	
 	TokenVec::const_iterator it = std::find(_tokens.begin() + pos, _tokens.end(), token);
-	if ( it != _tokens.end() )
+	if (it != _tokens.end())
 	{
 		return it - _tokens.begin();
 	}
 	throw NotFoundException(token);
 }
 
+
 bool StringTokenizer::has(const std::string& token) const
 {
 	TokenVec::const_iterator it = std::find(_tokens.begin(), _tokens.end(), token);
 	return it != _tokens.end();
 }
 
-std::size_t StringTokenizer::replace(const std::string& oldToken, const std::string& newToken, std::size_t pos)
+
+std::size_t StringTokenizer::replace(const std::string& oldToken, const std::string& newToken, std::string::size_type pos)
 {
 	std::size_t result = 0;
 	TokenVec::iterator it = std::find(_tokens.begin() + pos, _tokens.end(), oldToken);
-	while(it != _tokens.end())
+	while (it != _tokens.end())
 	{
 		result++;
 		*it = newToken;

Foundation/testsuite/src/StringTokenizerTest.cpp

@@ -326,6 +326,13 @@ void StringTokenizerTest::testStringTokenizer()
 		assert (st.find("2") == 1);
 		assert (st.find("3") == 2);
 	}
+	
+	{
+		Poco::StringTokenizer st(" 2- ","-", Poco::StringTokenizer::TOK_TRIM);
+		assert (st.count() == 2);
+		assert (st[0] == "2");
+		assert (st[1] == "");
+	}
 }