// © 2018 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html // characterproperties.cpp // created: 2018sep03 Markus W. Scherer #include "unicode/utypes.h" #include "unicode/localpointer.h" #include "unicode/uchar.h" #include "unicode/ucpmap.h" #include "unicode/ucptrie.h" #include "unicode/umutablecptrie.h" #include "unicode/uniset.h" #include "unicode/uscript.h" #include "unicode/uset.h" #include "cmemory.h" #include "emojiprops.h" #include "mutex.h" #include "normalizer2impl.h" #include "uassert.h" #include "ubidi_props.h" #include "ucase.h" #include "ucln_cmn.h" #include "umutex.h" #include "uprops.h" using icu::LocalPointer; #if !UCONFIG_NO_NORMALIZATION using icu::Normalizer2Factory; using icu::Normalizer2Impl; #endif using icu::UInitOnce; using icu::UnicodeSet; namespace { UBool U_CALLCONV characterproperties_cleanup(); constexpr int32_t NUM_INCLUSIONS = UPROPS_SRC_COUNT + (UCHAR_INT_LIMIT - UCHAR_INT_START); struct Inclusion { UnicodeSet *fSet = nullptr; UInitOnce fInitOnce {}; }; Inclusion gInclusions[NUM_INCLUSIONS]; // cached getInclusions() UnicodeSet *sets[UCHAR_BINARY_LIMIT] = {}; UCPMap *maps[UCHAR_INT_LIMIT - UCHAR_INT_START] = {}; icu::UMutex cpMutex; //---------------------------------------------------------------- // Inclusions list //---------------------------------------------------------------- // USetAdder implementation // Does not use uset.h to reduce code dependencies void U_CALLCONV _set_add(USet *set, UChar32 c) { ((UnicodeSet *)set)->add(c); } void U_CALLCONV _set_addRange(USet *set, UChar32 start, UChar32 end) { ((UnicodeSet *)set)->add(start, end); } void U_CALLCONV _set_addString(USet *set, const char16_t *str, int32_t length) { ((UnicodeSet *)set)->add(icu::UnicodeString((UBool)(length<0), str, length)); } UBool U_CALLCONV characterproperties_cleanup() { for (Inclusion &in: gInclusions) { delete in.fSet; in.fSet = nullptr; in.fInitOnce.reset(); } for (int32_t i = 0; i < UPRV_LENGTHOF(sets); ++i) { delete sets[i]; sets[i] = nullptr; } for (int32_t i = 0; i < UPRV_LENGTHOF(maps); ++i) { ucptrie_close(reinterpret_cast(maps[i])); maps[i] = nullptr; } return true; } void U_CALLCONV initInclusion(UPropertySource src, UErrorCode &errorCode) { // This function is invoked only via umtx_initOnce(). U_ASSERT(0 <= src && src < UPROPS_SRC_COUNT); if (src == UPROPS_SRC_NONE) { errorCode = U_INTERNAL_PROGRAM_ERROR; return; } U_ASSERT(gInclusions[src].fSet == nullptr); LocalPointer incl(new UnicodeSet()); if (incl.isNull()) { errorCode = U_MEMORY_ALLOCATION_ERROR; return; } USetAdder sa = { (USet *)incl.getAlias(), _set_add, _set_addRange, _set_addString, nullptr, // don't need remove() nullptr // don't need removeRange() }; switch(src) { case UPROPS_SRC_CHAR: uchar_addPropertyStarts(&sa, &errorCode); break; case UPROPS_SRC_PROPSVEC: upropsvec_addPropertyStarts(&sa, &errorCode); break; case UPROPS_SRC_CHAR_AND_PROPSVEC: uchar_addPropertyStarts(&sa, &errorCode); upropsvec_addPropertyStarts(&sa, &errorCode); break; #if !UCONFIG_NO_NORMALIZATION case UPROPS_SRC_CASE_AND_NORM: { const Normalizer2Impl *impl=Normalizer2Factory::getNFCImpl(errorCode); if(U_SUCCESS(errorCode)) { impl->addPropertyStarts(&sa, errorCode); } ucase_addPropertyStarts(&sa, &errorCode); break; } case UPROPS_SRC_NFC: { const Normalizer2Impl *impl=Normalizer2Factory::getNFCImpl(errorCode); if(U_SUCCESS(errorCode)) { impl->addPropertyStarts(&sa, errorCode); } break; } case UPROPS_SRC_NFKC: { const Normalizer2Impl *impl=Normalizer2Factory::getNFKCImpl(errorCode); if(U_SUCCESS(errorCode)) { impl->addPropertyStarts(&sa, errorCode); } break; } case UPROPS_SRC_NFKC_CF: { const Normalizer2Impl *impl=Normalizer2Factory::getNFKC_CFImpl(errorCode); if(U_SUCCESS(errorCode)) { impl->addPropertyStarts(&sa, errorCode); } break; } case UPROPS_SRC_NFC_CANON_ITER: { const Normalizer2Impl *impl=Normalizer2Factory::getNFCImpl(errorCode); if(U_SUCCESS(errorCode)) { impl->addCanonIterPropertyStarts(&sa, errorCode); } break; } #endif case UPROPS_SRC_CASE: ucase_addPropertyStarts(&sa, &errorCode); break; case UPROPS_SRC_BIDI: ubidi_addPropertyStarts(&sa, &errorCode); break; case UPROPS_SRC_INPC: case UPROPS_SRC_INSC: case UPROPS_SRC_VO: uprops_addPropertyStarts(src, &sa, &errorCode); break; case UPROPS_SRC_EMOJI: { const icu::EmojiProps *ep = icu::EmojiProps::getSingleton(errorCode); if (U_SUCCESS(errorCode)) { ep->addPropertyStarts(&sa, errorCode); } break; } case UPROPS_SRC_IDSU: // New in Unicode 15.1 for just two characters. sa.add(sa.set, 0x2FFE); sa.add(sa.set, 0x2FFF + 1); break; case UPROPS_SRC_ID_COMPAT_MATH: uprops_addPropertyStarts(src, &sa, &errorCode); break; default: errorCode = U_INTERNAL_PROGRAM_ERROR; break; } if (U_FAILURE(errorCode)) { return; } if (incl->isBogus()) { errorCode = U_MEMORY_ALLOCATION_ERROR; return; } // Compact for caching. incl->compact(); gInclusions[src].fSet = incl.orphan(); ucln_common_registerCleanup(UCLN_COMMON_CHARACTERPROPERTIES, characterproperties_cleanup); } const UnicodeSet *getInclusionsForSource(UPropertySource src, UErrorCode &errorCode) { if (U_FAILURE(errorCode)) { return nullptr; } if (src < 0 || UPROPS_SRC_COUNT <= src) { errorCode = U_ILLEGAL_ARGUMENT_ERROR; return nullptr; } Inclusion &i = gInclusions[src]; umtx_initOnce(i.fInitOnce, &initInclusion, src, errorCode); return i.fSet; } void U_CALLCONV initIntPropInclusion(UProperty prop, UErrorCode &errorCode) { // This function is invoked only via umtx_initOnce(). U_ASSERT(UCHAR_INT_START <= prop && prop < UCHAR_INT_LIMIT); int32_t inclIndex = UPROPS_SRC_COUNT + (prop - UCHAR_INT_START); U_ASSERT(gInclusions[inclIndex].fSet == nullptr); UPropertySource src = uprops_getSource(prop); const UnicodeSet *incl = getInclusionsForSource(src, errorCode); if (U_FAILURE(errorCode)) { return; } LocalPointer intPropIncl(new UnicodeSet(0, 0)); if (intPropIncl.isNull()) { errorCode = U_MEMORY_ALLOCATION_ERROR; return; } int32_t numRanges = incl->getRangeCount(); int32_t prevValue = 0; for (int32_t i = 0; i < numRanges; ++i) { UChar32 rangeEnd = incl->getRangeEnd(i); for (UChar32 c = incl->getRangeStart(i); c <= rangeEnd; ++c) { // TODO: Get a UCharacterProperty.IntProperty to avoid the property dispatch. int32_t value = u_getIntPropertyValue(c, prop); if (value != prevValue) { intPropIncl->add(c); prevValue = value; } } } if (intPropIncl->isBogus()) { errorCode = U_MEMORY_ALLOCATION_ERROR; return; } // Compact for caching. intPropIncl->compact(); gInclusions[inclIndex].fSet = intPropIncl.orphan(); ucln_common_registerCleanup(UCLN_COMMON_CHARACTERPROPERTIES, characterproperties_cleanup); } } // namespace U_NAMESPACE_BEGIN const UnicodeSet *CharacterProperties::getInclusionsForProperty( UProperty prop, UErrorCode &errorCode) { if (U_FAILURE(errorCode)) { return nullptr; } if (UCHAR_INT_START <= prop && prop < UCHAR_INT_LIMIT) { int32_t inclIndex = UPROPS_SRC_COUNT + (prop - UCHAR_INT_START); Inclusion &i = gInclusions[inclIndex]; umtx_initOnce(i.fInitOnce, &initIntPropInclusion, prop, errorCode); return i.fSet; } else { UPropertySource src = uprops_getSource(prop); return getInclusionsForSource(src, errorCode); } } U_NAMESPACE_END namespace { UnicodeSet *makeSet(UProperty property, UErrorCode &errorCode) { if (U_FAILURE(errorCode)) { return nullptr; } LocalPointer set(new UnicodeSet()); if (set.isNull()) { errorCode = U_MEMORY_ALLOCATION_ERROR; return nullptr; } if (UCHAR_BASIC_EMOJI <= property && property <= UCHAR_RGI_EMOJI) { // property of strings const icu::EmojiProps *ep = icu::EmojiProps::getSingleton(errorCode); if (U_FAILURE(errorCode)) { return nullptr; } USetAdder sa = { (USet *)set.getAlias(), _set_add, _set_addRange, _set_addString, nullptr, // don't need remove() nullptr // don't need removeRange() }; ep->addStrings(&sa, property, errorCode); if (property != UCHAR_BASIC_EMOJI && property != UCHAR_RGI_EMOJI) { // property of _only_ strings set->freeze(); return set.orphan(); } } const UnicodeSet *inclusions = icu::CharacterProperties::getInclusionsForProperty(property, errorCode); if (U_FAILURE(errorCode)) { return nullptr; } int32_t numRanges = inclusions->getRangeCount(); UChar32 startHasProperty = -1; for (int32_t i = 0; i < numRanges; ++i) { UChar32 rangeEnd = inclusions->getRangeEnd(i); for (UChar32 c = inclusions->getRangeStart(i); c <= rangeEnd; ++c) { // TODO: Get a UCharacterProperty.BinaryProperty to avoid the property dispatch. if (u_hasBinaryProperty(c, property)) { if (startHasProperty < 0) { // Transition from false to true. startHasProperty = c; } } else if (startHasProperty >= 0) { // Transition from true to false. set->add(startHasProperty, c - 1); startHasProperty = -1; } } } if (startHasProperty >= 0) { set->add(startHasProperty, 0x10FFFF); } set->freeze(); return set.orphan(); } UCPMap *makeMap(UProperty property, UErrorCode &errorCode) { if (U_FAILURE(errorCode)) { return nullptr; } uint32_t nullValue = property == UCHAR_SCRIPT ? USCRIPT_UNKNOWN : 0; icu::LocalUMutableCPTriePointer mutableTrie( umutablecptrie_open(nullValue, nullValue, &errorCode)); const UnicodeSet *inclusions = icu::CharacterProperties::getInclusionsForProperty(property, errorCode); if (U_FAILURE(errorCode)) { return nullptr; } int32_t numRanges = inclusions->getRangeCount(); UChar32 start = 0; uint32_t value = nullValue; for (int32_t i = 0; i < numRanges; ++i) { UChar32 rangeEnd = inclusions->getRangeEnd(i); for (UChar32 c = inclusions->getRangeStart(i); c <= rangeEnd; ++c) { // TODO: Get a UCharacterProperty.IntProperty to avoid the property dispatch. uint32_t nextValue = u_getIntPropertyValue(c, property); if (value != nextValue) { if (value != nullValue) { umutablecptrie_setRange(mutableTrie.getAlias(), start, c - 1, value, &errorCode); } start = c; value = nextValue; } } } if (value != 0) { umutablecptrie_setRange(mutableTrie.getAlias(), start, 0x10FFFF, value, &errorCode); } UCPTrieType type; if (property == UCHAR_BIDI_CLASS || property == UCHAR_GENERAL_CATEGORY) { type = UCPTRIE_TYPE_FAST; } else { type = UCPTRIE_TYPE_SMALL; } UCPTrieValueWidth valueWidth; // TODO: UCharacterProperty.IntProperty int32_t max = u_getIntPropertyMaxValue(property); if (max <= 0xff) { valueWidth = UCPTRIE_VALUE_BITS_8; } else if (max <= 0xffff) { valueWidth = UCPTRIE_VALUE_BITS_16; } else { valueWidth = UCPTRIE_VALUE_BITS_32; } return reinterpret_cast( umutablecptrie_buildImmutable(mutableTrie.getAlias(), type, valueWidth, &errorCode)); } } // namespace U_NAMESPACE_BEGIN const UnicodeSet *CharacterProperties::getBinaryPropertySet(UProperty property, UErrorCode &errorCode) { if (U_FAILURE(errorCode)) { return nullptr; } if (property < 0 || UCHAR_BINARY_LIMIT <= property) { errorCode = U_ILLEGAL_ARGUMENT_ERROR; return nullptr; } Mutex m(&cpMutex); UnicodeSet *set = sets[property]; if (set == nullptr) { sets[property] = set = makeSet(property, errorCode); } return set; } U_NAMESPACE_END U_NAMESPACE_USE U_CAPI const USet * U_EXPORT2 u_getBinaryPropertySet(UProperty property, UErrorCode *pErrorCode) { const UnicodeSet *set = CharacterProperties::getBinaryPropertySet(property, *pErrorCode); return U_SUCCESS(*pErrorCode) ? set->toUSet() : nullptr; } U_CAPI const UCPMap * U_EXPORT2 u_getIntPropertyMap(UProperty property, UErrorCode *pErrorCode) { if (U_FAILURE(*pErrorCode)) { return nullptr; } if (property < UCHAR_INT_START || UCHAR_INT_LIMIT <= property) { *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; return nullptr; } Mutex m(&cpMutex); UCPMap *map = maps[property - UCHAR_INT_START]; if (map == nullptr) { maps[property - UCHAR_INT_START] = map = makeMap(property, *pErrorCode); } return map; }