generic-library/poco
1
0
Fork 0
mirror of https://github.com/pocoproject/poco.git synced 2025-10-17 11:05:03 +02:00
Code Issues Projects Releases Wiki Activity

fixed doc typos; changed maximum sequence length to 4

Browse Source
This commit is contained in:
Günter Obiltschnig
2018-02-13 21:32:17 +01:00
parent bdd0478ead
commit 6ed085f288
3 changed files with 60 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
Foundation/include/Poco/TextEncoding.h
View File

@@ -30,7 +30,7 @@ class TextEncodingManager;
class Foundation_API TextEncoding class Foundation_API TextEncoding
/// An abstract base class for implementing text encodings /// An abstract base class for implementing text encodings
/// like UTF-8 or ISO 8859-1. /// like UTF-8 or ISO 8859-1.
/// ///
/// Subclasses must override the canonicalName(), isA(), /// Subclasses must override the canonicalName(), isA(),
/// characterMap() and convert() methods and need to be /// characterMap() and convert() methods and need to be
@@ -42,12 +42,12 @@ class Foundation_API TextEncoding
{ {
public: public:
typedef SharedPtr<TextEncoding> Ptr; typedef SharedPtr<TextEncoding> Ptr;
enum enum
{ {
MAX_SEQUENCE_LENGTH = 6 /// The maximum character byte sequence length supported. MAX_SEQUENCE_LENGTH = 4 /// The maximum character byte sequence length supported.
}; };
typedef int CharacterMap[256]; typedef int CharacterMap[256];
/// The map[b] member gives information about byte sequences /// The map[b] member gives information about byte sequences
/// whose first byte is b. /// whose first byte is b.
@@ -55,7 +55,7 @@ public:
/// If map[b] is -1, then the byte sequence is malformed. /// If map[b] is -1, then the byte sequence is malformed.
/// If map[b] is -n, where n >= 2, then b is the first byte of an n-byte /// If map[b] is -n, where n >= 2, then b is the first byte of an n-byte
/// sequence that encodes a single Unicode scalar value. Byte sequences up /// sequence that encodes a single Unicode scalar value. Byte sequences up
/// to 6 bytes in length are supported. /// to 4 bytes in length are supported.
virtual ~TextEncoding(); virtual ~TextEncoding();
/// Destroys the encoding. /// Destroys the encoding.
@@ -70,7 +70,7 @@ public:
/// For example, the "ISO-8859-1" encoding is also known as "Latin-1". /// For example, the "ISO-8859-1" encoding is also known as "Latin-1".
/// ///
/// Encoding name comparision are be case insensitive. /// Encoding name comparision are be case insensitive.
virtual const CharacterMap& characterMap() const = 0; virtual const CharacterMap& characterMap() const = 0;
/// Returns the CharacterMap for the encoding. /// Returns the CharacterMap for the encoding.
/// The CharacterMap should be kept in a static member. As /// The CharacterMap should be kept in a static member. As
@@ -78,52 +78,54 @@ public:
/// implemented in such a way that it just returns a static /// implemented in such a way that it just returns a static
/// map. If the map is built at runtime, this should be /// map. If the map is built at runtime, this should be
/// done in the constructor. /// done in the constructor.
virtual int convert(const unsigned char* bytes) const; virtual int convert(const unsigned char* bytes) const;
/// The convert function is used to convert multibyte sequences; /// The convert function is used to convert multibyte sequences;
/// bytes will point to a byte sequence of n bytes where /// bytes will point to a byte sequence of n bytes where
/// sequenceLength(bytes, length) == -n, with length >= n. /// sequenceLength(bytes, length) == -n, with length >= n.
/// ///
/// The convert function must return the Unicode scalar value /// The convert function must return the Unicode scalar value
/// represented by this byte sequence or -1 if the byte sequence is malformed. /// represented by this byte sequence or -1 if the byte sequence is malformed.
///
/// The default implementation returns (int) bytes[0]. /// The default implementation returns (int) bytes[0].
virtual int queryConvert(const unsigned char* bytes, int length) const; virtual int queryConvert(const unsigned char* bytes, int length) const;
/// The queryConvert function is used to convert single byte characters /// The queryConvert function is used to convert single byte characters
/// or multibyte sequences; /// or multibyte sequences;
/// bytes will point to a byte sequence of length bytes. /// bytes will point to a byte sequence of length bytes.
/// ///
/// The queryConvert function must return the Unicode scalar value /// The queryConvert function must return the Unicode scalar value
/// represented by this byte sequence or -1 if the byte sequence is malformed /// represented by this byte sequence or -1 if the byte sequence is malformed
/// or -n where n is number of bytes requested for the sequence, if lenght is /// or -n where n is number of bytes requested for the sequence, if length is
/// shorter than the sequence. /// shorter than the sequence.
/// The length of the sequence might not be determined by the first byte, /// The length of the sequence might not be determined by the first byte,
/// in which case the conversion becomes an iterative process: /// in which case the conversion becomes an iterative process:
/// First call with length == 1 might return -2, /// First call with length == 1 might return -2,
/// Then a second call with lenght == 2 might return -4 /// Then a second call with length == 2 might return -4
/// Eventually, the third call with length == 4 should return either a /// Eventually, the third call with length == 4 should return either a
/// Unicode scalar value, or -1 if the byte sequence is malformed. /// Unicode scalar value, or -1 if the byte sequence is malformed.
///
/// The default implementation returns (int) bytes[0]. /// The default implementation returns (int) bytes[0].
virtual int sequenceLength(const unsigned char* bytes, int length) const; virtual int sequenceLength(const unsigned char* bytes, int length) const;
/// The sequenceLength function is used to get the lenth of the sequence pointed /// The sequenceLength function is used to get the lenth of the sequence pointed
/// by bytes. The length paramater should be greater or equal to the length of /// by bytes. The length paramater should be greater or equal to the length of
/// the sequence. /// the sequence.
/// ///
/// The sequenceLength function must return the lenght of the sequence /// The sequenceLength function must return the length of the sequence
/// represented by this byte sequence or a negative value -n if length is /// represented by this byte sequence or a negative value -n if length is
/// shorter than the sequence, where n is the number of byte requested /// shorter than the sequence, where n is the number of byte requested
/// to determine the length of the sequence. /// to determine the length of the sequence.
/// The length of the sequence might not be determined by the first byte, /// The length of the sequence might not be determined by the first byte,
/// in which case the conversion becomes an iterative process as long as the /// in which case the conversion becomes an iterative process as long as the
/// result is negative: /// result is negative:
/// First call with length == 1 might return -2, /// First call with length == 1 might return -2,
/// Then a second call with lenght == 2 might return -4 /// Then a second call with length == 2 might return -4
/// Eventually, the third call with length == 4 should return 4. /// Eventually, the third call with length == 4 should return 4.
/// The default implementation returns 1. /// The default implementation returns 1.
virtual int convert(int ch, unsigned char* bytes, int length) const; virtual int convert(int ch, unsigned char* bytes, int length) const;
/// Transform the Unicode character ch into the encoding's /// Transform the Unicode character ch into the encoding's
/// byte sequence. The method returns the number of bytes /// byte sequence. The method returns the number of bytes
/// used. The method must not use more than length characters. /// used. The method must not use more than length characters.
/// Bytes and length can also be null - in this case only the number /// Bytes and length can also be null - in this case only the number
@@ -136,7 +138,7 @@ public:
/// Returns the TextEncoding object for the given encoding name. /// Returns the TextEncoding object for the given encoding name.
/// ///
/// Throws a NotFoundException if the encoding with given name is not available. /// Throws a NotFoundException if the encoding with given name is not available.
static TextEncoding::Ptr find(const std::string& encodingName); static TextEncoding::Ptr find(const std::string& encodingName);
/// Returns a pointer to the TextEncoding object for the given encodingName, /// Returns a pointer to the TextEncoding object for the given encodingName,
/// or NULL if no such TextEncoding object exists. /// or NULL if no such TextEncoding object exists.
@@ -170,7 +172,7 @@ public:
static const std::string GLOBAL; static const std::string GLOBAL;
/// Name of the global TextEncoding, which is the empty string. /// Name of the global TextEncoding, which is the empty string.
protected: protected:
static TextEncodingManager& manager(); static TextEncodingManager& manager();
/// Returns the TextEncodingManager. /// Returns the TextEncodingManager.

56
Foundation/src/UTF8Encoding.cpp
View File

@@ -27,24 +27,24 @@ const char* UTF8Encoding::_names[] =
}; };
const TextEncoding::CharacterMap UTF8Encoding::_charMap = const TextEncoding::CharacterMap UTF8Encoding::_charMap =
{ {
/* 00 */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 00 */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
/* 10 */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 10 */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
/* 20 */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 20 */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
/* 30 */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 30 */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
/* 40 */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 40 */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
/* 50 */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 50 */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
/* 60 */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 60 */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
/* 70 */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 70 */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
/* 80 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 80 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
/* 90 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 90 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
/* a0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* a0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
/* b0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* b0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
/* c0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, /* c0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
/* d0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, /* d0 */ -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
/* e0 */ -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, /* e0 */ -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3,
/* f0 */ -4, -4, -4, -4, -4, -4, -4, -4, -5, -5, -5, -5, -6, -6, -1, -1, /* f0 */ -4, -4, -4, -4, -4, -4, -4, -4, -1, -1, -1, -1, -1, -1, -1, -1,
}; };
@@ -85,15 +85,13 @@ int UTF8Encoding::convert(const unsigned char* bytes) const
{ {
int n = _charMap[*bytes]; int n = _charMap[*bytes];
int uc; int uc;
switch (n) switch (n)
{ {
case -6:
case -5:
case -1: case -1:
return -1; return -1;
case -4: case -4:
case -3: case -3:
case -2: case -2:
if (!isLegal(bytes, -n)) return -1; if (!isLegal(bytes, -n)) return -1;
uc = *bytes & ((0x07 << (n + 4)) | 0x03); uc = *bytes & ((0x07 << (n + 4)) | 0x03);
@@ -102,8 +100,8 @@ int UTF8Encoding::convert(const unsigned char* bytes) const
return n; return n;
} }
while (n++ < -1) while (n++ < -1)
{ {
uc <<= 6; uc <<= 6;
uc |= (*++bytes & 0x3F); uc |= (*++bytes & 0x3F);
} }
@@ -165,8 +163,6 @@ int UTF8Encoding::queryConvert(const unsigned char* bytes, int length) const
{ {
switch (n) switch (n)
{ {
case -6:
case -5:
case -1: case -1:
return -1; return -1;
case -4: case -4:
@@ -178,8 +174,8 @@ int UTF8Encoding::queryConvert(const unsigned char* bytes, int length) const
default: default:
return n; return n;
} }
while (n++ < -1) while (n++ < -1)
{ {
uc <<= 6; uc <<= 6;
uc |= (*++bytes & 0x3F); uc |= (*++bytes & 0x3F);
} }
@@ -207,7 +203,7 @@ bool UTF8Encoding::isLegal(const unsigned char *bytes, int length)
// Note: The following is loosely based on the isLegalUTF8 function // Note: The following is loosely based on the isLegalUTF8 function
// from ftp://ftp.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c // from ftp://ftp.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
// Excuse the ugliness... // Excuse the ugliness...
if (0 == bytes || 0 == length) return false; if (0 == bytes || 0 == length) return false;
unsigned char a; unsigned char a;
@@ -219,7 +215,7 @@ bool UTF8Encoding::isLegal(const unsigned char *bytes, int length)
// Everything else falls through when true. // Everything else falls through when true.
case 4: case 4:
if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
case 3: case 3:
if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
case 2: case 2:
a = (*--srcptr); a = (*--srcptr);

18
Foundation/testsuite/src/UTF8StringTest.cpp
View File

@@ -31,32 +31,32 @@ void UTF8StringTest::testCompare()
{ {
std::string a1("aaaaa"); std::string a1("aaaaa");
std::string b1("bbbbb"); std::string b1("bbbbb");
assert (UTF8::icompare(a1, b1) < 0); assert (UTF8::icompare(a1, b1) < 0);
std::string a2("aaaaa"); std::string a2("aaaaa");
std::string b2("BBBBB"); std::string b2("BBBBB");
assert (UTF8::icompare(a2, b2) < 0); assert (UTF8::icompare(a2, b2) < 0);
std::string a3("AAAAA"); std::string a3("AAAAA");
std::string b3("bbbbb"); std::string b3("bbbbb");
assert (UTF8::icompare(a3, b3) < 0); assert (UTF8::icompare(a3, b3) < 0);
std::string a4("aaaaa"); std::string a4("aaaaa");
std::string b4("AAAAA"); std::string b4("AAAAA");
assert (UTF8::icompare(a4, b4) == 0); assert (UTF8::icompare(a4, b4) == 0);
std::string a5("AAAAA"); std::string a5("AAAAA");
std::string b5("bbbbb"); std::string b5("bbbbb");
assert (UTF8::icompare(a5, b5) < 0); assert (UTF8::icompare(a5, b5) < 0);
std::string a6("\303\274\303\266\303\244"); // "u"o"a std::string a6("\303\274\303\266\303\244"); // "u"o"a
std::string b6("\303\234\303\226\303\204"); // "U"O"A std::string b6("\303\234\303\226\303\204"); // "U"O"A
assert (UTF8::icompare(a6, b6) == 0); assert (UTF8::icompare(a6, b6) == 0);
} }
@@ -72,14 +72,14 @@ void UTF8StringTest::testTransform()
assert (s2 == "ABCDE123"); assert (s2 == "ABCDE123");
std::string s3("\303\274\303\266\303\244"); // "u"o"a std::string s3("\303\274\303\266\303\244"); // "u"o"a
UTF8::toUpperInPlace(s3); UTF8::toUpperInPlace(s3);
assert (s3 == "\303\234\303\226\303\204"); // "U"O"A assert (s3 == "\303\234\303\226\303\204"); // "U"O"A
UTF8::toLowerInPlace(s3); UTF8::toLowerInPlace(s3);
assert (s3 == "\303\274\303\266\303\244"); // "u"o"a assert (s3 == "\303\274\303\266\303\244"); // "u"o"a
// a mix of invalid sequences // a mix of invalid sequences
std::string str = "\xC2\xE5\xF0\xF8\xE8\xED\xFB+-++"; std::string str = "\xC2\xE5\xF0\xF8\xE8\xED\xFB+-++";
assert ("???" == UTF8::toLower(str)); assert ("???+-++" == UTF8::toLower(str));
} }