1301 lines
43 KiB
C++
1301 lines
43 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
***************************************************************************
|
|
* Copyright (C) 1999-2016 International Business Machines Corporation
|
|
* and others. All rights reserved.
|
|
***************************************************************************
|
|
*/
|
|
//
|
|
// file: rbbi.cpp Contains the implementation of the rule based break iterator
|
|
// runtime engine and the API implementation for
|
|
// class RuleBasedBreakIterator
|
|
//
|
|
|
|
#include "utypeinfo.h" // for 'typeid' to work
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_BREAK_ITERATION
|
|
|
|
#include <cinttypes>
|
|
|
|
#include "unicode/rbbi.h"
|
|
#include "unicode/schriter.h"
|
|
#include "unicode/uchriter.h"
|
|
#include "unicode/uclean.h"
|
|
#include "unicode/udata.h"
|
|
|
|
#include "brkeng.h"
|
|
#include "ucln_cmn.h"
|
|
#include "cmemory.h"
|
|
#include "cstring.h"
|
|
#include "localsvc.h"
|
|
#include "rbbidata.h"
|
|
#include "rbbi_cache.h"
|
|
#include "rbbirb.h"
|
|
#include "uassert.h"
|
|
#include "umutex.h"
|
|
#include "uvectr32.h"
|
|
|
|
#ifdef RBBI_DEBUG
|
|
static UBool gTrace = FALSE;
|
|
#endif
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
// The state number of the starting state
|
|
constexpr int32_t START_STATE = 1;
|
|
|
|
// The state-transition value indicating "stop"
|
|
constexpr int32_t STOP_STATE = 0;
|
|
|
|
|
|
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
|
|
|
|
|
|
//=======================================================================
|
|
// constructors
|
|
//=======================================================================
|
|
|
|
/**
|
|
* Constructs a RuleBasedBreakIterator that uses the already-created
|
|
* tables object that is passed in as a parameter.
|
|
*/
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
|
|
: fSCharIter(UnicodeString())
|
|
{
|
|
init(status);
|
|
fData = new RBBIDataWrapper(data, status); // status checked in constructor
|
|
if (U_FAILURE(status)) {return;}
|
|
if(fData == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
|
|
fLookAheadMatches = static_cast<int32_t *>(
|
|
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
|
|
if (fLookAheadMatches == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Construct from precompiled binary rules (tables). This constructor is public API,
|
|
// taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules().
|
|
//
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
|
|
uint32_t ruleLength,
|
|
UErrorCode &status)
|
|
: fSCharIter(UnicodeString())
|
|
{
|
|
init(status);
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
if (compiledRules == NULL || ruleLength < sizeof(RBBIDataHeader)) {
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return;
|
|
}
|
|
const RBBIDataHeader *data = (const RBBIDataHeader *)compiledRules;
|
|
if (data->fLength > ruleLength) {
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return;
|
|
}
|
|
fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status);
|
|
if (U_FAILURE(status)) {return;}
|
|
if(fData == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
|
|
fLookAheadMatches = static_cast<int32_t *>(
|
|
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
|
|
if (fLookAheadMatches == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Constructor from a UDataMemory handle to precompiled break rules
|
|
// stored in an ICU data file.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
|
|
: fSCharIter(UnicodeString())
|
|
{
|
|
init(status);
|
|
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
|
|
if (U_FAILURE(status)) {return;}
|
|
if(fData == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
|
|
fLookAheadMatches = static_cast<int32_t *>(
|
|
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
|
|
if (fLookAheadMatches == nullptr) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Constructor from a set of rules supplied as a string.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules,
|
|
UParseError &parseError,
|
|
UErrorCode &status)
|
|
: fSCharIter(UnicodeString())
|
|
{
|
|
init(status);
|
|
if (U_FAILURE(status)) {return;}
|
|
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
|
|
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
|
|
// Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that
|
|
// creates and returns a complete RBBI. From here, in a constructor, we
|
|
// can't just return the object created by the builder factory, hence
|
|
// the assignment of the factory created object to "this".
|
|
if (U_SUCCESS(status)) {
|
|
*this = *bi;
|
|
delete bi;
|
|
}
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Default Constructor. Create an empty shell that can be set up later.
|
|
// Used when creating a RuleBasedBreakIterator from a set
|
|
// of rules.
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator()
|
|
: fSCharIter(UnicodeString())
|
|
{
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
init(status);
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Copy constructor. Will produce a break iterator with the same behavior,
|
|
// and which iterates over the same text, as the one passed in.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
|
|
: BreakIterator(other),
|
|
fSCharIter(UnicodeString())
|
|
{
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
this->init(status);
|
|
*this = other;
|
|
}
|
|
|
|
|
|
/**
|
|
* Destructor
|
|
*/
|
|
RuleBasedBreakIterator::~RuleBasedBreakIterator() {
|
|
if (fCharIter != &fSCharIter) {
|
|
// fCharIter was adopted from the outside.
|
|
delete fCharIter;
|
|
}
|
|
fCharIter = nullptr;
|
|
|
|
utext_close(&fText);
|
|
|
|
if (fData != nullptr) {
|
|
fData->removeReference();
|
|
fData = nullptr;
|
|
}
|
|
delete fBreakCache;
|
|
fBreakCache = nullptr;
|
|
|
|
delete fDictionaryCache;
|
|
fDictionaryCache = nullptr;
|
|
|
|
delete fLanguageBreakEngines;
|
|
fLanguageBreakEngines = nullptr;
|
|
|
|
delete fUnhandledBreakEngine;
|
|
fUnhandledBreakEngine = nullptr;
|
|
|
|
uprv_free(fLookAheadMatches);
|
|
fLookAheadMatches = nullptr;
|
|
}
|
|
|
|
/**
|
|
* Assignment operator. Sets this iterator to have the same behavior,
|
|
* and iterate over the same text, as the one passed in.
|
|
* TODO: needs better handling of memory allocation errors.
|
|
*/
|
|
RuleBasedBreakIterator&
|
|
RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
|
|
if (this == &that) {
|
|
return *this;
|
|
}
|
|
BreakIterator::operator=(that);
|
|
|
|
if (fLanguageBreakEngines != NULL) {
|
|
delete fLanguageBreakEngines;
|
|
fLanguageBreakEngines = NULL; // Just rebuild for now
|
|
}
|
|
// TODO: clone fLanguageBreakEngines from "that"
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
utext_clone(&fText, &that.fText, FALSE, TRUE, &status);
|
|
|
|
if (fCharIter != &fSCharIter) {
|
|
delete fCharIter;
|
|
}
|
|
fCharIter = &fSCharIter;
|
|
|
|
if (that.fCharIter != NULL && that.fCharIter != &that.fSCharIter) {
|
|
// This is a little bit tricky - it will intially appear that
|
|
// this->fCharIter is adopted, even if that->fCharIter was
|
|
// not adopted. That's ok.
|
|
fCharIter = that.fCharIter->clone();
|
|
}
|
|
fSCharIter = that.fSCharIter;
|
|
if (fCharIter == NULL) {
|
|
fCharIter = &fSCharIter;
|
|
}
|
|
|
|
if (fData != NULL) {
|
|
fData->removeReference();
|
|
fData = NULL;
|
|
}
|
|
if (that.fData != NULL) {
|
|
fData = that.fData->addReference();
|
|
}
|
|
|
|
uprv_free(fLookAheadMatches);
|
|
fLookAheadMatches = nullptr;
|
|
if (fData && fData->fForwardTable->fLookAheadResultsSize > 0) {
|
|
fLookAheadMatches = static_cast<int32_t *>(
|
|
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
|
|
}
|
|
|
|
|
|
fPosition = that.fPosition;
|
|
fRuleStatusIndex = that.fRuleStatusIndex;
|
|
fDone = that.fDone;
|
|
|
|
// TODO: both the dictionary and the main cache need to be copied.
|
|
// Current position could be within a dictionary range. Trying to continue
|
|
// the iteration without the caches present would go to the rules, with
|
|
// the assumption that the current position is on a rule boundary.
|
|
fBreakCache->reset(fPosition, fRuleStatusIndex);
|
|
fDictionaryCache->reset();
|
|
|
|
return *this;
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// init() Shared initialization routine. Used by all the constructors.
|
|
// Initializes all fields, leaving the object in a consistent state.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
void RuleBasedBreakIterator::init(UErrorCode &status) {
|
|
fCharIter = nullptr;
|
|
fData = nullptr;
|
|
fPosition = 0;
|
|
fRuleStatusIndex = 0;
|
|
fDone = false;
|
|
fDictionaryCharCount = 0;
|
|
fLanguageBreakEngines = nullptr;
|
|
fUnhandledBreakEngine = nullptr;
|
|
fBreakCache = nullptr;
|
|
fDictionaryCache = nullptr;
|
|
fLookAheadMatches = nullptr;
|
|
|
|
// Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER.
|
|
// fText = UTEXT_INITIALIZER;
|
|
static const UText initializedUText = UTEXT_INITIALIZER;
|
|
uprv_memcpy(&fText, &initializedUText, sizeof(UText));
|
|
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
|
|
utext_openUChars(&fText, NULL, 0, &status);
|
|
fDictionaryCache = new DictionaryCache(this, status);
|
|
fBreakCache = new BreakCache(this, status);
|
|
if (U_SUCCESS(status) && (fDictionaryCache == NULL || fBreakCache == NULL)) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
}
|
|
|
|
#ifdef RBBI_DEBUG
|
|
static UBool debugInitDone = FALSE;
|
|
if (debugInitDone == FALSE) {
|
|
char *debugEnv = getenv("U_RBBIDEBUG");
|
|
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
|
|
gTrace = TRUE;
|
|
}
|
|
debugInitDone = TRUE;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// clone - Returns a newly-constructed RuleBasedBreakIterator with the same
|
|
// behavior, and iterating over the same text, as this one.
|
|
// Virtual function: does the right thing with subclasses.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
RuleBasedBreakIterator*
|
|
RuleBasedBreakIterator::clone() const {
|
|
return new RuleBasedBreakIterator(*this);
|
|
}
|
|
|
|
/**
|
|
* Equality operator. Returns TRUE if both BreakIterators are of the
|
|
* same class, have the same behavior, and iterate over the same text.
|
|
*/
|
|
UBool
|
|
RuleBasedBreakIterator::operator==(const BreakIterator& that) const {
|
|
if (typeid(*this) != typeid(that)) {
|
|
return FALSE;
|
|
}
|
|
if (this == &that) {
|
|
return TRUE;
|
|
}
|
|
|
|
// The base class BreakIterator carries no state that participates in equality,
|
|
// and does not implement an equality function that would otherwise be
|
|
// checked at this point.
|
|
|
|
const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that;
|
|
|
|
if (!utext_equals(&fText, &that2.fText)) {
|
|
// The two break iterators are operating on different text,
|
|
// or have a different iteration position.
|
|
// Note that fText's position is always the same as the break iterator's position.
|
|
return FALSE;
|
|
}
|
|
|
|
if (!(fPosition == that2.fPosition &&
|
|
fRuleStatusIndex == that2.fRuleStatusIndex &&
|
|
fDone == that2.fDone)) {
|
|
return FALSE;
|
|
}
|
|
|
|
if (that2.fData == fData ||
|
|
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
|
|
// The two break iterators are using the same rules.
|
|
return TRUE;
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
/**
|
|
* Compute a hash code for this BreakIterator
|
|
* @return A hash code
|
|
*/
|
|
int32_t
|
|
RuleBasedBreakIterator::hashCode(void) const {
|
|
int32_t hash = 0;
|
|
if (fData != NULL) {
|
|
hash = fData->hashCode();
|
|
}
|
|
return hash;
|
|
}
|
|
|
|
|
|
void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fBreakCache->reset();
|
|
fDictionaryCache->reset();
|
|
utext_clone(&fText, ut, FALSE, TRUE, &status);
|
|
|
|
// Set up a dummy CharacterIterator to be returned if anyone
|
|
// calls getText(). With input from UText, there is no reasonable
|
|
// way to return a characterIterator over the actual input text.
|
|
// Return one over an empty string instead - this is the closest
|
|
// we can come to signaling a failure.
|
|
// (GetText() is obsolete, this failure is sort of OK)
|
|
fSCharIter.setText(UnicodeString());
|
|
|
|
if (fCharIter != &fSCharIter) {
|
|
// existing fCharIter was adopted from the outside. Delete it now.
|
|
delete fCharIter;
|
|
}
|
|
fCharIter = &fSCharIter;
|
|
|
|
this->first();
|
|
}
|
|
|
|
|
|
UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const {
|
|
UText *result = utext_clone(fillIn, &fText, FALSE, TRUE, &status);
|
|
return result;
|
|
}
|
|
|
|
|
|
//=======================================================================
|
|
// BreakIterator overrides
|
|
//=======================================================================
|
|
|
|
/**
|
|
* Return a CharacterIterator over the text being analyzed.
|
|
*/
|
|
CharacterIterator&
|
|
RuleBasedBreakIterator::getText() const {
|
|
return *fCharIter;
|
|
}
|
|
|
|
/**
|
|
* Set the iterator to analyze a new piece of text. This function resets
|
|
* the current iteration position to the beginning of the text.
|
|
* @param newText An iterator over the text to analyze.
|
|
*/
|
|
void
|
|
RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
|
|
// If we are holding a CharacterIterator adopted from a
|
|
// previous call to this function, delete it now.
|
|
if (fCharIter != &fSCharIter) {
|
|
delete fCharIter;
|
|
}
|
|
|
|
fCharIter = newText;
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
fBreakCache->reset();
|
|
fDictionaryCache->reset();
|
|
if (newText==NULL || newText->startIndex() != 0) {
|
|
// startIndex !=0 wants to be an error, but there's no way to report it.
|
|
// Make the iterator text be an empty string.
|
|
utext_openUChars(&fText, NULL, 0, &status);
|
|
} else {
|
|
utext_openCharacterIterator(&fText, newText, &status);
|
|
}
|
|
this->first();
|
|
}
|
|
|
|
/**
|
|
* Set the iterator to analyze a new piece of text. This function resets
|
|
* the current iteration position to the beginning of the text.
|
|
* @param newText An iterator over the text to analyze.
|
|
*/
|
|
void
|
|
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
fBreakCache->reset();
|
|
fDictionaryCache->reset();
|
|
utext_openConstUnicodeString(&fText, &newText, &status);
|
|
|
|
// Set up a character iterator on the string.
|
|
// Needed in case someone calls getText().
|
|
// Can not, unfortunately, do this lazily on the (probably never)
|
|
// call to getText(), because getText is const.
|
|
fSCharIter.setText(newText);
|
|
|
|
if (fCharIter != &fSCharIter) {
|
|
// old fCharIter was adopted from the outside. Delete it.
|
|
delete fCharIter;
|
|
}
|
|
fCharIter = &fSCharIter;
|
|
|
|
this->first();
|
|
}
|
|
|
|
|
|
/**
|
|
* Provide a new UText for the input text. Must reference text with contents identical
|
|
* to the original.
|
|
* Intended for use with text data originating in Java (garbage collected) environments
|
|
* where the data may be moved in memory at arbitrary times.
|
|
*/
|
|
RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return *this;
|
|
}
|
|
if (input == NULL) {
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return *this;
|
|
}
|
|
int64_t pos = utext_getNativeIndex(&fText);
|
|
// Shallow read-only clone of the new UText into the existing input UText
|
|
utext_clone(&fText, input, FALSE, TRUE, &status);
|
|
if (U_FAILURE(status)) {
|
|
return *this;
|
|
}
|
|
utext_setNativeIndex(&fText, pos);
|
|
if (utext_getNativeIndex(&fText) != pos) {
|
|
// Sanity check. The new input utext is supposed to have the exact same
|
|
// contents as the old. If we can't set to the same position, it doesn't.
|
|
// The contents underlying the old utext might be invalid at this point,
|
|
// so it's not safe to check directly.
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
|
|
/**
|
|
* Sets the current iteration position to the beginning of the text, position zero.
|
|
* @return The new iterator position, which is zero.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::first(void) {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
if (!fBreakCache->seek(0)) {
|
|
fBreakCache->populateNear(0, status);
|
|
}
|
|
fBreakCache->current();
|
|
U_ASSERT(fPosition == 0);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Sets the current iteration position to the end of the text.
|
|
* @return The text's past-the-end offset.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::last(void) {
|
|
int32_t endPos = (int32_t)utext_nativeLength(&fText);
|
|
UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
|
|
(void)endShouldBeBoundary;
|
|
U_ASSERT(endShouldBeBoundary);
|
|
U_ASSERT(fPosition == endPos);
|
|
return endPos;
|
|
}
|
|
|
|
/**
|
|
* Advances the iterator either forward or backward the specified number of steps.
|
|
* Negative values move backward, and positive values move forward. This is
|
|
* equivalent to repeatedly calling next() or previous().
|
|
* @param n The number of steps to move. The sign indicates the direction
|
|
* (negative is backwards, and positive is forwards).
|
|
* @return The character offset of the boundary position n boundaries away from
|
|
* the current one.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::next(int32_t n) {
|
|
int32_t result = 0;
|
|
if (n > 0) {
|
|
for (; n > 0 && result != UBRK_DONE; --n) {
|
|
result = next();
|
|
}
|
|
} else if (n < 0) {
|
|
for (; n < 0 && result != UBRK_DONE; ++n) {
|
|
result = previous();
|
|
}
|
|
} else {
|
|
result = current();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Advances the iterator to the next boundary position.
|
|
* @return The position of the first boundary after this one.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::next(void) {
|
|
fBreakCache->next();
|
|
return fDone ? UBRK_DONE : fPosition;
|
|
}
|
|
|
|
/**
|
|
* Move the iterator backwards, to the boundary preceding the current one.
|
|
*
|
|
* Starts from the current position within fText.
|
|
* Starting position need not be on a boundary.
|
|
*
|
|
* @return The position of the boundary position immediately preceding the starting position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::previous(void) {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
fBreakCache->previous(status);
|
|
return fDone ? UBRK_DONE : fPosition;
|
|
}
|
|
|
|
/**
|
|
* Sets the iterator to refer to the first boundary position following
|
|
* the specified position.
|
|
* @param startPos The position from which to begin searching for a break position.
|
|
* @return The position of the first break after the current position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::following(int32_t startPos) {
|
|
// if the supplied position is before the beginning, return the
|
|
// text's starting offset
|
|
if (startPos < 0) {
|
|
return first();
|
|
}
|
|
|
|
// Move requested offset to a code point start. It might be on a trail surrogate,
|
|
// or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text.
|
|
utext_setNativeIndex(&fText, startPos);
|
|
startPos = (int32_t)utext_getNativeIndex(&fText);
|
|
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
fBreakCache->following(startPos, status);
|
|
return fDone ? UBRK_DONE : fPosition;
|
|
}
|
|
|
|
/**
|
|
* Sets the iterator to refer to the last boundary position before the
|
|
* specified position.
|
|
* @param offset The position to begin searching for a break from.
|
|
* @return The position of the last boundary before the starting position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
|
|
if (offset > utext_nativeLength(&fText)) {
|
|
return last();
|
|
}
|
|
|
|
// Move requested offset to a code point start. It might be on a trail surrogate,
|
|
// or on a trail byte if the input is UTF-8.
|
|
|
|
utext_setNativeIndex(&fText, offset);
|
|
int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText));
|
|
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
fBreakCache->preceding(adjustedOffset, status);
|
|
return fDone ? UBRK_DONE : fPosition;
|
|
}
|
|
|
|
/**
|
|
* Returns true if the specfied position is a boundary position. As a side
|
|
* effect, leaves the iterator pointing to the first boundary position at
|
|
* or after "offset".
|
|
*
|
|
* @param offset the offset to check.
|
|
* @return True if "offset" is a boundary position.
|
|
*/
|
|
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
|
|
// out-of-range indexes are never boundary positions
|
|
if (offset < 0) {
|
|
first(); // For side effects on current position, tag values.
|
|
return FALSE;
|
|
}
|
|
|
|
// Adjust offset to be on a code point boundary and not beyond the end of the text.
|
|
// Note that isBoundary() is always false for offsets that are not on code point boundaries.
|
|
// But we still need the side effect of leaving iteration at the following boundary.
|
|
|
|
utext_setNativeIndex(&fText, offset);
|
|
int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText));
|
|
|
|
bool result = false;
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) {
|
|
result = (fBreakCache->current() == offset);
|
|
}
|
|
|
|
if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) {
|
|
// Original offset is beyond the end of the text. Return FALSE, it's not a boundary,
|
|
// but the iteration position remains set to the end of the text, which is a boundary.
|
|
return FALSE;
|
|
}
|
|
if (!result) {
|
|
// Not on a boundary. isBoundary() must leave iterator on the following boundary.
|
|
// Cache->seek(), above, left us on the preceding boundary, so advance one.
|
|
next();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
/**
|
|
* Returns the current iteration position.
|
|
* @return The current iteration position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::current(void) const {
|
|
return fPosition;
|
|
}
|
|
|
|
|
|
//=======================================================================
|
|
// implementation
|
|
//=======================================================================
|
|
|
|
//
|
|
// RBBIRunMode - the state machine runs an extra iteration at the beginning and end
|
|
// of user text. A variable with this enum type keeps track of where we
|
|
// are. The state machine only fetches user input while in the RUN mode.
|
|
//
|
|
enum RBBIRunMode {
|
|
RBBI_START, // state machine processing is before first char of input
|
|
RBBI_RUN, // state machine processing is in the user text
|
|
RBBI_END // state machine processing is after end of user text.
|
|
};
|
|
|
|
|
|
// Wrapper functions to select the appropriate handleNext() or handleSafePrevious()
|
|
// instantiation, based on whether an 8 or 16 bit table is required.
|
|
//
|
|
// These Trie access functions will be inlined within the handleNext()/Previous() instantions.
|
|
static inline uint16_t TrieFunc8(const UCPTrie *trie, UChar32 c) {
|
|
return UCPTRIE_FAST_GET(trie, UCPTRIE_8, c);
|
|
}
|
|
|
|
static inline uint16_t TrieFunc16(const UCPTrie *trie, UChar32 c) {
|
|
return UCPTRIE_FAST_GET(trie, UCPTRIE_16, c);
|
|
}
|
|
|
|
int32_t RuleBasedBreakIterator::handleNext() {
|
|
const RBBIStateTable *statetable = fData->fForwardTable;
|
|
bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8;
|
|
if (statetable->fFlags & RBBI_8BITS_ROWS) {
|
|
if (use8BitsTrie) {
|
|
return handleNext<RBBIStateTableRow8, TrieFunc8>();
|
|
} else {
|
|
return handleNext<RBBIStateTableRow8, TrieFunc16>();
|
|
}
|
|
} else {
|
|
if (use8BitsTrie) {
|
|
return handleNext<RBBIStateTableRow16, TrieFunc8>();
|
|
} else {
|
|
return handleNext<RBBIStateTableRow16, TrieFunc16>();
|
|
}
|
|
}
|
|
}
|
|
|
|
int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
|
|
const RBBIStateTable *statetable = fData->fReverseTable;
|
|
bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8;
|
|
if (statetable->fFlags & RBBI_8BITS_ROWS) {
|
|
if (use8BitsTrie) {
|
|
return handleSafePrevious<RBBIStateTableRow8, TrieFunc8>(fromPosition);
|
|
} else {
|
|
return handleSafePrevious<RBBIStateTableRow8, TrieFunc16>(fromPosition);
|
|
}
|
|
} else {
|
|
if (use8BitsTrie) {
|
|
return handleSafePrevious<RBBIStateTableRow16, TrieFunc8>(fromPosition);
|
|
} else {
|
|
return handleSafePrevious<RBBIStateTableRow16, TrieFunc16>(fromPosition);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------------
|
|
//
|
|
// handleNext()
|
|
// Run the state machine to find a boundary
|
|
//
|
|
//-----------------------------------------------------------------------------------
|
|
template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc>
|
|
int32_t RuleBasedBreakIterator::handleNext() {
|
|
int32_t state;
|
|
uint16_t category = 0;
|
|
RBBIRunMode mode;
|
|
|
|
RowType *row;
|
|
UChar32 c;
|
|
int32_t result = 0;
|
|
int32_t initialPosition = 0;
|
|
const RBBIStateTable *statetable = fData->fForwardTable;
|
|
const char *tableData = statetable->fTableData;
|
|
uint32_t tableRowLen = statetable->fRowLen;
|
|
uint32_t dictStart = statetable->fDictCategoriesStart;
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPuts("Handle Next pos char state category");
|
|
}
|
|
#endif
|
|
|
|
// handleNext alway sets the break tag value.
|
|
// Set the default for it.
|
|
fRuleStatusIndex = 0;
|
|
|
|
fDictionaryCharCount = 0;
|
|
|
|
// if we're already at the end of the text, return DONE.
|
|
initialPosition = fPosition;
|
|
UTEXT_SETNATIVEINDEX(&fText, initialPosition);
|
|
result = initialPosition;
|
|
c = UTEXT_NEXT32(&fText);
|
|
if (c==U_SENTINEL) {
|
|
fDone = TRUE;
|
|
return UBRK_DONE;
|
|
}
|
|
|
|
// Set the initial state for the state machine
|
|
state = START_STATE;
|
|
row = (RowType *)
|
|
//(statetable->fTableData + (statetable->fRowLen * state));
|
|
(tableData + tableRowLen * state);
|
|
|
|
|
|
mode = RBBI_RUN;
|
|
if (statetable->fFlags & RBBI_BOF_REQUIRED) {
|
|
category = 2;
|
|
mode = RBBI_START;
|
|
}
|
|
|
|
|
|
// loop until we reach the end of the text or transition to state 0
|
|
//
|
|
for (;;) {
|
|
if (c == U_SENTINEL) {
|
|
// Reached end of input string.
|
|
if (mode == RBBI_END) {
|
|
// We have already run the loop one last time with the
|
|
// character set to the psueudo {eof} value. Now it is time
|
|
// to unconditionally bail out.
|
|
break;
|
|
}
|
|
// Run the loop one last time with the fake end-of-input character category.
|
|
mode = RBBI_END;
|
|
category = 1;
|
|
}
|
|
|
|
//
|
|
// Get the char category. An incoming category of 1 or 2 means that
|
|
// we are preset for doing the beginning or end of input, and
|
|
// that we shouldn't get a category from an actual text input character.
|
|
//
|
|
if (mode == RBBI_RUN) {
|
|
// look up the current character's character category, which tells us
|
|
// which column in the state table to look at.
|
|
category = trieFunc(fData->fTrie, c);
|
|
fDictionaryCharCount += (category >= dictStart);
|
|
}
|
|
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText));
|
|
if (0x20<=c && c<0x7f) {
|
|
RBBIDebugPrintf("\"%c\" ", c);
|
|
} else {
|
|
RBBIDebugPrintf("%5x ", c);
|
|
}
|
|
RBBIDebugPrintf("%3d %3d\n", state, category);
|
|
}
|
|
#endif
|
|
|
|
// State Transition - move machine to its next state
|
|
//
|
|
|
|
// fNextState is a variable-length array.
|
|
U_ASSERT(category<fData->fHeader->fCatCount);
|
|
state = row->fNextState[category]; /*Not accessing beyond memory*/
|
|
row = (RowType *)
|
|
// (statetable->fTableData + (statetable->fRowLen * state));
|
|
(tableData + tableRowLen * state);
|
|
|
|
|
|
uint16_t accepting = row->fAccepting;
|
|
if (accepting == ACCEPTING_UNCONDITIONAL) {
|
|
// Match found, common case.
|
|
if (mode != RBBI_START) {
|
|
result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
|
|
}
|
|
fRuleStatusIndex = row->fTagsIdx; // Remember the break status (tag) values.
|
|
} else if (accepting > ACCEPTING_UNCONDITIONAL) {
|
|
// Lookahead match is completed.
|
|
U_ASSERT(accepting < fData->fForwardTable->fLookAheadResultsSize);
|
|
int32_t lookaheadResult = fLookAheadMatches[accepting];
|
|
if (lookaheadResult >= 0) {
|
|
fRuleStatusIndex = row->fTagsIdx;
|
|
fPosition = lookaheadResult;
|
|
return lookaheadResult;
|
|
}
|
|
}
|
|
|
|
// If we are at the position of the '/' in a look-ahead (hard break) rule;
|
|
// record the current position, to be returned later, if the full rule matches.
|
|
// TODO: Move this check before the previous check of fAccepting.
|
|
// This would enable hard-break rules with no following context.
|
|
// But there are line break test failures when trying this. Investigate.
|
|
// Issue ICU-20837
|
|
uint16_t rule = row->fLookAhead;
|
|
U_ASSERT(rule == 0 || rule > ACCEPTING_UNCONDITIONAL);
|
|
U_ASSERT(rule == 0 || rule < fData->fForwardTable->fLookAheadResultsSize);
|
|
if (rule > ACCEPTING_UNCONDITIONAL) {
|
|
int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
|
|
fLookAheadMatches[rule] = pos;
|
|
}
|
|
|
|
if (state == STOP_STATE) {
|
|
// This is the normal exit from the lookup state machine.
|
|
// We have advanced through the string until it is certain that no
|
|
// longer match is possible, no matter what characters follow.
|
|
break;
|
|
}
|
|
|
|
// Advance to the next character.
|
|
// If this is a beginning-of-input loop iteration, don't advance
|
|
// the input position. The next iteration will be processing the
|
|
// first real input character.
|
|
if (mode == RBBI_RUN) {
|
|
c = UTEXT_NEXT32(&fText);
|
|
} else {
|
|
if (mode == RBBI_START) {
|
|
mode = RBBI_RUN;
|
|
}
|
|
}
|
|
}
|
|
|
|
// The state machine is done. Check whether it found a match...
|
|
|
|
// If the iterator failed to advance in the match engine, force it ahead by one.
|
|
// (This really indicates a defect in the break rules. They should always match
|
|
// at least one character.)
|
|
if (result == initialPosition) {
|
|
utext_setNativeIndex(&fText, initialPosition);
|
|
utext_next32(&fText);
|
|
result = (int32_t)utext_getNativeIndex(&fText);
|
|
fRuleStatusIndex = 0;
|
|
}
|
|
|
|
// Leave the iterator at our result position.
|
|
fPosition = result;
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPrintf("result = %d\n\n", result);
|
|
}
|
|
#endif
|
|
return result;
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------------
|
|
//
|
|
// handleSafePrevious()
|
|
//
|
|
// Iterate backwards using the safe reverse rules.
|
|
// The logic of this function is similar to handleNext(), but simpler
|
|
// because the safe table does not require as many options.
|
|
//
|
|
//-----------------------------------------------------------------------------------
|
|
template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc>
|
|
int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
|
|
|
|
int32_t state;
|
|
uint16_t category = 0;
|
|
RowType *row;
|
|
UChar32 c;
|
|
int32_t result = 0;
|
|
|
|
const RBBIStateTable *stateTable = fData->fReverseTable;
|
|
UTEXT_SETNATIVEINDEX(&fText, fromPosition);
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPuts("Handle Previous pos char state category");
|
|
}
|
|
#endif
|
|
|
|
// if we're already at the start of the text, return DONE.
|
|
if (fData == NULL || UTEXT_GETNATIVEINDEX(&fText)==0) {
|
|
return BreakIterator::DONE;
|
|
}
|
|
|
|
// Set the initial state for the state machine
|
|
c = UTEXT_PREVIOUS32(&fText);
|
|
state = START_STATE;
|
|
row = (RowType *)
|
|
(stateTable->fTableData + (stateTable->fRowLen * state));
|
|
|
|
// loop until we reach the start of the text or transition to state 0
|
|
//
|
|
for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) {
|
|
|
|
// look up the current character's character category, which tells us
|
|
// which column in the state table to look at.
|
|
//
|
|
// Off the dictionary flag bit. For reverse iteration it is not used.
|
|
category = trieFunc(fData->fTrie, c);
|
|
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText));
|
|
if (0x20<=c && c<0x7f) {
|
|
RBBIDebugPrintf("\"%c\" ", c);
|
|
} else {
|
|
RBBIDebugPrintf("%5x ", c);
|
|
}
|
|
RBBIDebugPrintf("%3d %3d\n", state, category);
|
|
}
|
|
#endif
|
|
|
|
// State Transition - move machine to its next state
|
|
//
|
|
// fNextState is a variable-length array.
|
|
U_ASSERT(category<fData->fHeader->fCatCount);
|
|
state = row->fNextState[category]; /*Not accessing beyond memory*/
|
|
row = (RowType *)
|
|
(stateTable->fTableData + (stateTable->fRowLen * state));
|
|
|
|
if (state == STOP_STATE) {
|
|
// This is the normal exit from the lookup state machine.
|
|
// Transistion to state zero means we have found a safe point.
|
|
break;
|
|
}
|
|
}
|
|
|
|
// The state machine is done. Check whether it found a match...
|
|
result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
|
|
#ifdef RBBI_DEBUG
|
|
if (gTrace) {
|
|
RBBIDebugPrintf("result = %d\n\n", result);
|
|
}
|
|
#endif
|
|
return result;
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// getRuleStatus() Return the break rule tag associated with the current
|
|
// iterator position. If the iterator arrived at its current
|
|
// position by iterating forwards, the value will have been
|
|
// cached by the handleNext() function.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
|
|
int32_t RuleBasedBreakIterator::getRuleStatus() const {
|
|
|
|
// fLastRuleStatusIndex indexes to the start of the appropriate status record
|
|
// (the number of status values.)
|
|
// This function returns the last (largest) of the array of status values.
|
|
int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex];
|
|
int32_t tagVal = fData->fRuleStatusTable[idx];
|
|
|
|
return tagVal;
|
|
}
|
|
|
|
|
|
int32_t RuleBasedBreakIterator::getRuleStatusVec(
|
|
int32_t *fillInVec, int32_t capacity, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return 0;
|
|
}
|
|
|
|
int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex];
|
|
int32_t numValsToCopy = numVals;
|
|
if (numVals > capacity) {
|
|
status = U_BUFFER_OVERFLOW_ERROR;
|
|
numValsToCopy = capacity;
|
|
}
|
|
int i;
|
|
for (i=0; i<numValsToCopy; i++) {
|
|
fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1];
|
|
}
|
|
return numVals;
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// getBinaryRules Access to the compiled form of the rules,
|
|
// for use by build system tools that save the data
|
|
// for standard iterator types.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) {
|
|
const uint8_t *retPtr = NULL;
|
|
length = 0;
|
|
|
|
if (fData != NULL) {
|
|
retPtr = (const uint8_t *)fData->fHeader;
|
|
length = fData->fHeader->fLength;
|
|
}
|
|
return retPtr;
|
|
}
|
|
|
|
|
|
RuleBasedBreakIterator *RuleBasedBreakIterator::createBufferClone(
|
|
void * /*stackBuffer*/, int32_t &bufferSize, UErrorCode &status) {
|
|
if (U_FAILURE(status)){
|
|
return NULL;
|
|
}
|
|
|
|
if (bufferSize == 0) {
|
|
bufferSize = 1; // preflighting for deprecated functionality
|
|
return NULL;
|
|
}
|
|
|
|
BreakIterator *clonedBI = clone();
|
|
if (clonedBI == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
} else {
|
|
status = U_SAFECLONE_ALLOCATED_WARNING;
|
|
}
|
|
return (RuleBasedBreakIterator *)clonedBI;
|
|
}
|
|
|
|
U_NAMESPACE_END
|
|
|
|
|
|
static icu::UStack *gLanguageBreakFactories = nullptr;
|
|
static const icu::UnicodeString *gEmptyString = nullptr;
|
|
static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER;
|
|
static icu::UInitOnce gRBBIInitOnce = U_INITONCE_INITIALIZER;
|
|
|
|
/**
|
|
* Release all static memory held by breakiterator.
|
|
*/
|
|
U_CDECL_BEGIN
|
|
UBool U_CALLCONV rbbi_cleanup(void) {
|
|
delete gLanguageBreakFactories;
|
|
gLanguageBreakFactories = nullptr;
|
|
delete gEmptyString;
|
|
gEmptyString = nullptr;
|
|
gLanguageBreakFactoriesInitOnce.reset();
|
|
gRBBIInitOnce.reset();
|
|
return TRUE;
|
|
}
|
|
U_CDECL_END
|
|
|
|
U_CDECL_BEGIN
|
|
static void U_CALLCONV _deleteFactory(void *obj) {
|
|
delete (icu::LanguageBreakFactory *) obj;
|
|
}
|
|
U_CDECL_END
|
|
U_NAMESPACE_BEGIN
|
|
|
|
static void U_CALLCONV rbbiInit() {
|
|
gEmptyString = new UnicodeString();
|
|
ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
|
|
}
|
|
|
|
static void U_CALLCONV initLanguageFactories() {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
U_ASSERT(gLanguageBreakFactories == NULL);
|
|
gLanguageBreakFactories = new UStack(_deleteFactory, NULL, status);
|
|
if (gLanguageBreakFactories != NULL && U_SUCCESS(status)) {
|
|
ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status);
|
|
gLanguageBreakFactories->push(builtIn, status);
|
|
#ifdef U_LOCAL_SERVICE_HOOK
|
|
LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status);
|
|
if (extra != NULL) {
|
|
gLanguageBreakFactories->push(extra, status);
|
|
}
|
|
#endif
|
|
}
|
|
ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
|
|
}
|
|
|
|
|
|
static const LanguageBreakEngine*
|
|
getLanguageBreakEngineFromFactory(UChar32 c)
|
|
{
|
|
umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories);
|
|
if (gLanguageBreakFactories == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
int32_t i = gLanguageBreakFactories->size();
|
|
const LanguageBreakEngine *lbe = NULL;
|
|
while (--i >= 0) {
|
|
LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i));
|
|
lbe = factory->getEngineFor(c);
|
|
if (lbe != NULL) {
|
|
break;
|
|
}
|
|
}
|
|
return lbe;
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the
|
|
// the character c.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
const LanguageBreakEngine *
|
|
RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
|
|
const LanguageBreakEngine *lbe = NULL;
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
|
|
if (fLanguageBreakEngines == NULL) {
|
|
fLanguageBreakEngines = new UStack(status);
|
|
if (fLanguageBreakEngines == NULL || U_FAILURE(status)) {
|
|
delete fLanguageBreakEngines;
|
|
fLanguageBreakEngines = 0;
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
int32_t i = fLanguageBreakEngines->size();
|
|
while (--i >= 0) {
|
|
lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i));
|
|
if (lbe->handles(c)) {
|
|
return lbe;
|
|
}
|
|
}
|
|
|
|
// No existing dictionary took the character. See if a factory wants to
|
|
// give us a new LanguageBreakEngine for this character.
|
|
lbe = getLanguageBreakEngineFromFactory(c);
|
|
|
|
// If we got one, use it and push it on our stack.
|
|
if (lbe != NULL) {
|
|
fLanguageBreakEngines->push((void *)lbe, status);
|
|
// Even if we can't remember it, we can keep looking it up, so
|
|
// return it even if the push fails.
|
|
return lbe;
|
|
}
|
|
|
|
// No engine is forthcoming for this character. Add it to the
|
|
// reject set. Create the reject break engine if needed.
|
|
if (fUnhandledBreakEngine == NULL) {
|
|
fUnhandledBreakEngine = new UnhandledEngine(status);
|
|
if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return nullptr;
|
|
}
|
|
// Put it last so that scripts for which we have an engine get tried
|
|
// first.
|
|
fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status);
|
|
// If we can't insert it, or creation failed, get rid of it
|
|
if (U_FAILURE(status)) {
|
|
delete fUnhandledBreakEngine;
|
|
fUnhandledBreakEngine = 0;
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
// Tell the reject engine about the character; at its discretion, it may
|
|
// add more than just the one character.
|
|
fUnhandledBreakEngine->handleCharacter(c);
|
|
|
|
return fUnhandledBreakEngine;
|
|
}
|
|
|
|
void RuleBasedBreakIterator::dumpCache() {
|
|
fBreakCache->dumpCache();
|
|
}
|
|
|
|
void RuleBasedBreakIterator::dumpTables() {
|
|
fData->printData();
|
|
}
|
|
|
|
/**
|
|
* Returns the description used to create this iterator
|
|
*/
|
|
|
|
const UnicodeString&
|
|
RuleBasedBreakIterator::getRules() const {
|
|
if (fData != NULL) {
|
|
return fData->getRuleSourceString();
|
|
} else {
|
|
umtx_initOnce(gRBBIInitOnce, &rbbiInit);
|
|
return *gEmptyString;
|
|
}
|
|
}
|
|
|
|
U_NAMESPACE_END
|
|
|
|
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
|