diff --git a/Foundation/include/Poco/TextEncoding.h b/Foundation/include/Poco/TextEncoding.h index 5902e8f2a..39a93b607 100644 --- a/Foundation/include/Poco/TextEncoding.h +++ b/Foundation/include/Poco/TextEncoding.h @@ -30,7 +30,7 @@ class TextEncodingManager; class Foundation_API TextEncoding /// An abstract base class for implementing text encodings - /// like UTF-8 or ISO 8859-1. + /// like UTF-8 or ISO 8859-1. /// /// Subclasses must override the canonicalName(), isA(), /// characterMap() and convert() methods and need to be @@ -42,12 +42,12 @@ class Foundation_API TextEncoding { public: typedef SharedPtr Ptr; - + enum { - MAX_SEQUENCE_LENGTH = 6 /// The maximum character byte sequence length supported. + MAX_SEQUENCE_LENGTH = 4 /// The maximum character byte sequence length supported. }; - + typedef int CharacterMap[256]; /// The map[b] member gives information about byte sequences /// whose first byte is b. @@ -55,7 +55,7 @@ public: /// If map[b] is -1, then the byte sequence is malformed. /// If map[b] is -n, where n >= 2, then b is the first byte of an n-byte /// sequence that encodes a single Unicode scalar value. Byte sequences up - /// to 6 bytes in length are supported. + /// to 4 bytes in length are supported. virtual ~TextEncoding(); /// Destroys the encoding. @@ -70,7 +70,7 @@ public: /// For example, the "ISO-8859-1" encoding is also known as "Latin-1". /// /// Encoding name comparision are be case insensitive. - + virtual const CharacterMap& characterMap() const = 0; /// Returns the CharacterMap for the encoding. /// The CharacterMap should be kept in a static member. As @@ -78,52 +78,54 @@ public: /// implemented in such a way that it just returns a static /// map. If the map is built at runtime, this should be /// done in the constructor. - + virtual int convert(const unsigned char* bytes) const; /// The convert function is used to convert multibyte sequences; - /// bytes will point to a byte sequence of n bytes where + /// bytes will point to a byte sequence of n bytes where /// sequenceLength(bytes, length) == -n, with length >= n. /// /// The convert function must return the Unicode scalar value /// represented by this byte sequence or -1 if the byte sequence is malformed. + /// /// The default implementation returns (int) bytes[0]. virtual int queryConvert(const unsigned char* bytes, int length) const; - /// The queryConvert function is used to convert single byte characters + /// The queryConvert function is used to convert single byte characters /// or multibyte sequences; /// bytes will point to a byte sequence of length bytes. /// /// The queryConvert function must return the Unicode scalar value /// represented by this byte sequence or -1 if the byte sequence is malformed - /// or -n where n is number of bytes requested for the sequence, if lenght is + /// or -n where n is number of bytes requested for the sequence, if length is /// shorter than the sequence. - /// The length of the sequence might not be determined by the first byte, + /// The length of the sequence might not be determined by the first byte, /// in which case the conversion becomes an iterative process: /// First call with length == 1 might return -2, - /// Then a second call with lenght == 2 might return -4 - /// Eventually, the third call with length == 4 should return either a + /// Then a second call with length == 2 might return -4 + /// Eventually, the third call with length == 4 should return either a /// Unicode scalar value, or -1 if the byte sequence is malformed. + /// /// The default implementation returns (int) bytes[0]. virtual int sequenceLength(const unsigned char* bytes, int length) const; /// The sequenceLength function is used to get the lenth of the sequence pointed - /// by bytes. The length paramater should be greater or equal to the length of + /// by bytes. The length paramater should be greater or equal to the length of /// the sequence. /// - /// The sequenceLength function must return the lenght of the sequence - /// represented by this byte sequence or a negative value -n if length is - /// shorter than the sequence, where n is the number of byte requested + /// The sequenceLength function must return the length of the sequence + /// represented by this byte sequence or a negative value -n if length is + /// shorter than the sequence, where n is the number of byte requested /// to determine the length of the sequence. - /// The length of the sequence might not be determined by the first byte, - /// in which case the conversion becomes an iterative process as long as the + /// The length of the sequence might not be determined by the first byte, + /// in which case the conversion becomes an iterative process as long as the /// result is negative: /// First call with length == 1 might return -2, - /// Then a second call with lenght == 2 might return -4 + /// Then a second call with length == 2 might return -4 /// Eventually, the third call with length == 4 should return 4. /// The default implementation returns 1. virtual int convert(int ch, unsigned char* bytes, int length) const; - /// Transform the Unicode character ch into the encoding's + /// Transform the Unicode character ch into the encoding's /// byte sequence. The method returns the number of bytes /// used. The method must not use more than length characters. /// Bytes and length can also be null - in this case only the number @@ -136,7 +138,7 @@ public: /// Returns the TextEncoding object for the given encoding name. /// /// Throws a NotFoundException if the encoding with given name is not available. - + static TextEncoding::Ptr find(const std::string& encodingName); /// Returns a pointer to the TextEncoding object for the given encodingName, /// or NULL if no such TextEncoding object exists. @@ -170,7 +172,7 @@ public: static const std::string GLOBAL; /// Name of the global TextEncoding, which is the empty string. - + protected: static TextEncodingManager& manager(); /// Returns the TextEncodingManager. diff --git a/Foundation/src/UTF8Encoding.cpp b/Foundation/src/UTF8Encoding.cpp index 372bbb8ed..78fc6d4a8 100644 --- a/Foundation/src/UTF8Encoding.cpp +++ b/Foundation/src/UTF8Encoding.cpp @@ -27,24 +27,24 @@ const char* UTF8Encoding::_names[] = }; -const TextEncoding::CharacterMap UTF8Encoding::_charMap = +const TextEncoding::CharacterMap UTF8Encoding::_charMap = { - /* 00 */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - /* 10 */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - /* 20 */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - /* 30 */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - /* 40 */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - /* 50 */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - /* 60 */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - /* 70 */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - /* 80 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - /* 90 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - /* a0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - /* b0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - /* c0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - /* d0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - /* e0 */ -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, - /* f0 */ -4, -4, -4, -4, -4, -4, -4, -4, -5, -5, -5, -5, -6, -6, -1, -1, + /* 00 */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + /* 10 */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + /* 20 */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + /* 30 */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + /* 40 */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + /* 50 */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + /* 60 */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + /* 70 */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + /* 80 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 90 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* a0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* b0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* c0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + /* d0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + /* e0 */ -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + /* f0 */ -4, -4, -4, -4, -4, -4, -4, -4, -1, -1, -1, -1, -1, -1, -1, -1, }; @@ -85,15 +85,13 @@ int UTF8Encoding::convert(const unsigned char* bytes) const { int n = _charMap[*bytes]; int uc; - + switch (n) { - case -6: - case -5: case -1: return -1; - case -4: - case -3: + case -4: + case -3: case -2: if (!isLegal(bytes, -n)) return -1; uc = *bytes & ((0x07 << (n + 4)) | 0x03); @@ -102,8 +100,8 @@ int UTF8Encoding::convert(const unsigned char* bytes) const return n; } - while (n++ < -1) - { + while (n++ < -1) + { uc <<= 6; uc |= (*++bytes & 0x3F); } @@ -165,8 +163,6 @@ int UTF8Encoding::queryConvert(const unsigned char* bytes, int length) const { switch (n) { - case -6: - case -5: case -1: return -1; case -4: @@ -178,8 +174,8 @@ int UTF8Encoding::queryConvert(const unsigned char* bytes, int length) const default: return n; } - while (n++ < -1) - { + while (n++ < -1) + { uc <<= 6; uc |= (*++bytes & 0x3F); } @@ -207,7 +203,7 @@ bool UTF8Encoding::isLegal(const unsigned char *bytes, int length) // Note: The following is loosely based on the isLegalUTF8 function // from ftp://ftp.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c // Excuse the ugliness... - + if (0 == bytes || 0 == length) return false; unsigned char a; @@ -219,7 +215,7 @@ bool UTF8Encoding::isLegal(const unsigned char *bytes, int length) // Everything else falls through when true. case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: + case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; case 2: a = (*--srcptr); diff --git a/Foundation/testsuite/src/UTF8StringTest.cpp b/Foundation/testsuite/src/UTF8StringTest.cpp index 6f072c984..3c155fb28 100644 --- a/Foundation/testsuite/src/UTF8StringTest.cpp +++ b/Foundation/testsuite/src/UTF8StringTest.cpp @@ -31,32 +31,32 @@ void UTF8StringTest::testCompare() { std::string a1("aaaaa"); std::string b1("bbbbb"); - + assert (UTF8::icompare(a1, b1) < 0); std::string a2("aaaaa"); std::string b2("BBBBB"); - + assert (UTF8::icompare(a2, b2) < 0); std::string a3("AAAAA"); std::string b3("bbbbb"); - + assert (UTF8::icompare(a3, b3) < 0); std::string a4("aaaaa"); std::string b4("AAAAA"); - + assert (UTF8::icompare(a4, b4) == 0); - + std::string a5("AAAAA"); std::string b5("bbbbb"); - + assert (UTF8::icompare(a5, b5) < 0); std::string a6("\303\274\303\266\303\244"); // "u"o"a std::string b6("\303\234\303\226\303\204"); // "U"O"A - + assert (UTF8::icompare(a6, b6) == 0); } @@ -72,14 +72,14 @@ void UTF8StringTest::testTransform() assert (s2 == "ABCDE123"); std::string s3("\303\274\303\266\303\244"); // "u"o"a - UTF8::toUpperInPlace(s3); + UTF8::toUpperInPlace(s3); assert (s3 == "\303\234\303\226\303\204"); // "U"O"A UTF8::toLowerInPlace(s3); assert (s3 == "\303\274\303\266\303\244"); // "u"o"a // a mix of invalid sequences std::string str = "\xC2\xE5\xF0\xF8\xE8\xED\xFB+-++"; - assert ("???" == UTF8::toLower(str)); + assert ("???+-++" == UTF8::toLower(str)); }