218 lines
6.9 KiB
C++
218 lines
6.9 KiB
C++
// © 2017 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
|
|
// norms.h
|
|
// created: 2017jun04 Markus W. Scherer
|
|
// (pulled out of n2builder.cpp)
|
|
|
|
// Storing & manipulating Normalizer2 builder data.
|
|
|
|
#ifndef __NORMS_H__
|
|
#define __NORMS_H__
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_NORMALIZATION
|
|
|
|
#include "unicode/errorcode.h"
|
|
#include "unicode/uniset.h"
|
|
#include "unicode/unistr.h"
|
|
#include "unicode/utf16.h"
|
|
#include "normalizer2impl.h"
|
|
#include "toolutil.h"
|
|
#include "utrie2.h"
|
|
#include "uvectr32.h"
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
class BuilderReorderingBuffer {
|
|
public:
|
|
BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(FALSE) {}
|
|
void reset() {
|
|
fLength=0;
|
|
fLastStarterIndex=-1;
|
|
fDidReorder=FALSE;
|
|
}
|
|
int32_t length() const { return fLength; }
|
|
UBool isEmpty() const { return fLength==0; }
|
|
int32_t lastStarterIndex() const { return fLastStarterIndex; }
|
|
UChar32 charAt(int32_t i) const { return fArray[i]>>8; }
|
|
uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; }
|
|
UBool didReorder() const { return fDidReorder; }
|
|
|
|
void append(UChar32 c, uint8_t cc);
|
|
void toString(UnicodeString &dest) const;
|
|
|
|
private:
|
|
int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK];
|
|
int32_t fLength;
|
|
int32_t fLastStarterIndex;
|
|
UBool fDidReorder;
|
|
};
|
|
|
|
struct CompositionPair {
|
|
CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {}
|
|
UChar32 trail, composite;
|
|
};
|
|
|
|
struct Norm {
|
|
enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY };
|
|
|
|
UBool hasMapping() const { return mappingType>REMOVED; }
|
|
|
|
// Requires hasMapping() and well-formed mapping.
|
|
void setMappingCP() {
|
|
UChar32 c;
|
|
if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) {
|
|
mappingCP=c;
|
|
} else {
|
|
mappingCP=U_SENTINEL;
|
|
}
|
|
}
|
|
|
|
const CompositionPair *getCompositionPairs(int32_t &length) const {
|
|
if(compositions==nullptr) {
|
|
length=0;
|
|
return nullptr;
|
|
} else {
|
|
length=compositions->size()/2;
|
|
return reinterpret_cast<const CompositionPair *>(compositions->getBuffer());
|
|
}
|
|
}
|
|
UChar32 combine(UChar32 trail) const;
|
|
|
|
UnicodeString *mapping;
|
|
UnicodeString *rawMapping; // non-nullptr if the mapping is further decomposed
|
|
UChar32 mappingCP; // >=0 if mapping to 1 code point
|
|
int32_t mappingPhase;
|
|
MappingType mappingType;
|
|
|
|
UVector32 *compositions; // (trail, composite) pairs
|
|
uint8_t cc, leadCC, trailCC;
|
|
UBool combinesBack;
|
|
UBool hasCompBoundaryBefore, hasCompBoundaryAfter;
|
|
|
|
/**
|
|
* Overall type of normalization properties.
|
|
* Set after most processing is done.
|
|
*
|
|
* Corresponds to the rows in the chart on
|
|
* http://site.icu-project.org/design/normalization/custom
|
|
* in numerical (but reverse visual) order.
|
|
*
|
|
* YES_NO means composition quick check=yes, decomposition QC=no -- etc.
|
|
*/
|
|
enum Type {
|
|
/** Initial value until most processing is done. */
|
|
UNKNOWN,
|
|
/** No mapping, does not combine, ccc=0. */
|
|
INERT,
|
|
/** Starter, no mapping, has compositions. */
|
|
YES_YES_COMBINES_FWD,
|
|
/** Starter with a round-trip mapping and compositions. */
|
|
YES_NO_COMBINES_FWD,
|
|
/** Starter with a round-trip mapping but no compositions. */
|
|
YES_NO_MAPPING_ONLY,
|
|
/** Has a one-way mapping which is comp-normalized. */
|
|
NO_NO_COMP_YES,
|
|
/** Has a one-way mapping which is not comp-normalized but has a comp boundary before. */
|
|
NO_NO_COMP_BOUNDARY_BEFORE,
|
|
/** Has a one-way mapping which does not have a comp boundary before. */
|
|
NO_NO_COMP_NO_MAYBE_CC,
|
|
/** Has a one-way mapping to the empty string. */
|
|
NO_NO_EMPTY,
|
|
/** Has an algorithmic one-way mapping to a single code point. */
|
|
NO_NO_DELTA,
|
|
/**
|
|
* Combines both backward and forward, has compositions.
|
|
* Allowed, but not normally used.
|
|
*/
|
|
MAYBE_YES_COMBINES_FWD,
|
|
/** Combines only backward. */
|
|
MAYBE_YES_SIMPLE,
|
|
/** Non-zero ccc but does not combine backward. */
|
|
YES_YES_WITH_CC
|
|
} type;
|
|
/** Offset into the type's part of the extra data, or the algorithmic-mapping delta. */
|
|
int32_t offset;
|
|
|
|
/**
|
|
* Error string set by processing functions that do not have access
|
|
* to the code point, deferred for readable reporting.
|
|
*/
|
|
const char *error;
|
|
};
|
|
|
|
class Norms {
|
|
public:
|
|
Norms(UErrorCode &errorCode);
|
|
~Norms();
|
|
|
|
int32_t length() const { return utm_countItems(normMem); }
|
|
const Norm &getNormRefByIndex(int32_t i) const { return norms[i]; }
|
|
Norm &getNormRefByIndex(int32_t i) { return norms[i]; }
|
|
|
|
Norm *allocNorm();
|
|
/** Returns an existing Norm unit, or nullptr if c has no data. */
|
|
Norm *getNorm(UChar32 c);
|
|
const Norm *getNorm(UChar32 c) const;
|
|
/** Returns a Norm unit, creating a new one if necessary. */
|
|
Norm *createNorm(UChar32 c);
|
|
/** Returns an existing Norm unit, or an immutable empty object if c has no data. */
|
|
const Norm &getNormRef(UChar32 c) const;
|
|
uint8_t getCC(UChar32 c) const { return getNormRef(c).cc; }
|
|
UBool combinesBack(UChar32 c) const {
|
|
return Hangul::isJamoV(c) || Hangul::isJamoT(c) || getNormRef(c).combinesBack;
|
|
}
|
|
|
|
void reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const;
|
|
|
|
// int32_t highCC not uint8_t so that we can pass in 256 as the upper limit.
|
|
UBool combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const;
|
|
|
|
class Enumerator {
|
|
public:
|
|
Enumerator(Norms &n) : norms(n) {}
|
|
virtual ~Enumerator();
|
|
/** Called for enumerated value!=0. */
|
|
virtual void rangeHandler(UChar32 start, UChar32 end, Norm &norm) = 0;
|
|
/** @internal Public only for C callback. */
|
|
UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value);
|
|
protected:
|
|
Norms &norms;
|
|
};
|
|
|
|
void enumRanges(Enumerator &e);
|
|
|
|
UnicodeSet ccSet, mappingSet;
|
|
|
|
private:
|
|
Norms(const Norms &other) = delete;
|
|
Norms &operator=(const Norms &other) = delete;
|
|
|
|
UTrie2 *normTrie;
|
|
UToolMemory *normMem;
|
|
Norm *norms;
|
|
};
|
|
|
|
class CompositionBuilder : public Norms::Enumerator {
|
|
public:
|
|
CompositionBuilder(Norms &n) : Norms::Enumerator(n) {}
|
|
/** Adds a composition mapping for the first character in a round-trip mapping. */
|
|
void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE;
|
|
};
|
|
|
|
class Decomposer : public Norms::Enumerator {
|
|
public:
|
|
Decomposer(Norms &n) : Norms::Enumerator(n), didDecompose(FALSE) {}
|
|
/** Decomposes each character of the current mapping. Sets didDecompose if any. */
|
|
void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE;
|
|
UBool didDecompose;
|
|
};
|
|
|
|
U_NAMESPACE_END
|
|
|
|
#endif // #if !UCONFIG_NO_NORMALIZATION
|
|
|
|
#endif // __NORMS_H__
|