blob: 1bd631cbd85a88c14883844c73999c675233321c [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Implements a custom word iterator used for our spellchecker.
#include "components/spellcheck/renderer/spellcheck_worditerator.h"
#include <map>
#include <memory>
#include <string>
#include <utility>
#include "base/i18n/break_iterator.h"
#include "base/logging.h"
#include "base/macros.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "components/spellcheck/renderer/spellcheck.h"
#include "third_party/icu/source/common/unicode/normlzr.h"
#include "third_party/icu/source/common/unicode/schriter.h"
#include "third_party/icu/source/common/unicode/uscript.h"
#include "third_party/icu/source/i18n/unicode/ulocdata.h"
// SpellcheckCharAttribute implementation:
SpellcheckCharAttribute::SpellcheckCharAttribute()
: script_code_(USCRIPT_LATIN) {
}
SpellcheckCharAttribute::~SpellcheckCharAttribute() {
}
void SpellcheckCharAttribute::SetDefaultLanguage(const std::string& language) {
CreateRuleSets(language);
}
base::string16 SpellcheckCharAttribute::GetRuleSet(
bool allow_contraction) const {
return allow_contraction ?
ruleset_allow_contraction_ : ruleset_disallow_contraction_;
}
void SpellcheckCharAttribute::CreateRuleSets(const std::string& language) {
// The template for our custom rule sets, which is based on the word-break
// rules of ICU 4.0:
// <http://source.icu-project.org/repos/icu/icu/tags/release-4-0/source/data/brkitr/word.txt>.
// The major differences from the original one are listed below:
// * It discards comments in the original rules.
// * It discards characters not needed by our spellchecker (e.g. numbers,
// punctuation characters, Hiraganas, Katakanas, CJK Ideographs, and so on).
// * It allows customization of the $ALetter value (i.e. word characters).
// * It allows customization of the $ALetterPlus value (i.e. whether or not to
// use the dictionary data).
// * It allows choosing whether or not to split a text at contraction
// characters.
// This template only changes the forward-iteration rules. So, calling
// ubrk_prev() returns the same results as the original template.
static const char kRuleTemplate[] =
"!!chain;"
"$CR = [\\p{Word_Break = CR}];"
"$LF = [\\p{Word_Break = LF}];"
"$Newline = [\\p{Word_Break = Newline}];"
"$Extend = [\\p{Word_Break = Extend}];"
"$Format = [\\p{Word_Break = Format}];"
"$Katakana = [\\p{Word_Break = Katakana}];"
// Not all the characters in a given script are ALetter.
// For instance, U+05F4 is MidLetter. So, this may be
// better, but it leads to an empty set error in Thai.
// "$ALetter = [[\\p{script=%s}] & [\\p{Word_Break = ALetter}]];"
"$ALetter = [\\p{script=%s}%s];"
// U+0027 (single quote/apostrophe) is not in MidNumLet any more
// in UAX 29 rev 21 or later. For our purpose, U+0027
// has to be treated as MidNumLet. ( http://crbug.com/364072 )
"$MidNumLet = [\\p{Word_Break = MidNumLet} \\u0027];"
"$MidLetter = [\\p{Word_Break = MidLetter}%s];"
"$MidNum = [\\p{Word_Break = MidNum}];"
"$Numeric = [\\p{Word_Break = Numeric}];"
"$ExtendNumLet = [\\p{Word_Break = ExtendNumLet}];"
"$Control = [\\p{Grapheme_Cluster_Break = Control}]; "
"%s" // ALetterPlus
"$KatakanaEx = $Katakana ($Extend | $Format)*;"
"$ALetterEx = $ALetterPlus ($Extend | $Format)*;"
"$MidNumLetEx = $MidNumLet ($Extend | $Format)*;"
"$MidLetterEx = $MidLetter ($Extend | $Format)*;"
"$MidNumEx = $MidNum ($Extend | $Format)*;"
"$NumericEx = $Numeric ($Extend | $Format)*;"
"$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;"
"$Hiragana = [\\p{script=Hiragana}];"
"$Ideographic = [\\p{Ideographic}];"
"$HiraganaEx = $Hiragana ($Extend | $Format)*;"
"$IdeographicEx = $Ideographic ($Extend | $Format)*;"
"!!forward;"
"$CR $LF;"
"[^$CR $LF $Newline]? ($Extend | $Format)+;"
"$ALetterEx {200};"
"$ALetterEx $ALetterEx {200};"
"%s" // (Allow|Disallow) Contraction
"!!reverse;"
"$BackALetterEx = ($Format | $Extend)* $ALetterPlus;"
"$BackMidNumLetEx = ($Format | $Extend)* $MidNumLet;"
"$BackNumericEx = ($Format | $Extend)* $Numeric;"
"$BackMidNumEx = ($Format | $Extend)* $MidNum;"
"$BackMidLetterEx = ($Format | $Extend)* $MidLetter;"
"$BackKatakanaEx = ($Format | $Extend)* $Katakana;"
"$BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;"
"$LF $CR;"
"($Format | $Extend)* [^$CR $LF $Newline]?;"
"$BackALetterEx $BackALetterEx;"
"$BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;"
"$BackNumericEx $BackNumericEx;"
"$BackNumericEx $BackALetterEx;"
"$BackALetterEx $BackNumericEx;"
"$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx;"
"$BackKatakanaEx $BackKatakanaEx;"
"$BackExtendNumLetEx ($BackALetterEx | $BackNumericEx |"
" $BackKatakanaEx | $BackExtendNumLetEx);"
"($BackALetterEx | $BackNumericEx | $BackKatakanaEx)"
" $BackExtendNumLetEx;"
"!!safe_reverse;"
"($Extend | $Format)+ .?;"
"($MidLetter | $MidNumLet) $BackALetterEx;"
"($MidNum | $MidNumLet) $BackNumericEx;"
"!!safe_forward;"
"($Extend | $Format)+ .?;"
"($MidLetterEx | $MidNumLetEx) $ALetterEx;"
"($MidNumEx | $MidNumLetEx) $NumericEx;";
// Retrieve the script codes used by the given language from ICU. When the
// given language consists of two or more scripts, we just use the first
// script. The size of returned script codes is always < 8. Therefore, we use
// an array of size 8 so we can include all script codes without insufficient
// buffer errors.
UErrorCode error = U_ZERO_ERROR;
UScriptCode script_code[8];
int scripts = uscript_getCode(language.c_str(), script_code,
arraysize(script_code), &error);
if (U_SUCCESS(error) && scripts >= 1)
script_code_ = script_code[0];
// Retrieve the values for $ALetter and $ALetterPlus. We use the dictionary
// only for the languages which need it (i.e. Korean and Thai) to prevent ICU
// from returning dictionary words (i.e. Korean or Thai words) for languages
// which don't need them.
const char* aletter = uscript_getName(script_code_);
if (!aletter)
aletter = "Latin";
const char kWithDictionary[] =
"$dictionary = [:LineBreak = Complex_Context:];"
"$ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]];";
const char kWithoutDictionary[] = "$ALetterPlus = $ALetter;";
const char* aletter_plus = kWithoutDictionary;
if (script_code_ == USCRIPT_HANGUL || script_code_ == USCRIPT_THAI ||
script_code_ == USCRIPT_LAO || script_code_ == USCRIPT_KHMER)
aletter_plus = kWithDictionary;
// Treat numbers as word characters except for Arabic and Hebrew.
const char* aletter_extra = " [0123456789]";
if (script_code_ == USCRIPT_HEBREW)
aletter_extra = "";
else if (script_code_ == USCRIPT_ARABIC)
// When "script=Arabic", it does not include tatweel, which is
// "script=Common" so add it back. Otherwise, it creates unwanted
// word breaks.
aletter_extra = " [\\u0640]";
const char kMidLetterExtra[] = "";
// For Hebrew, treat single/double quoation marks as MidLetter.
const char kMidLetterExtraHebrew[] = "\"'";
const char* midletter_extra = kMidLetterExtra;
if (script_code_ == USCRIPT_HEBREW)
midletter_extra = kMidLetterExtraHebrew;
// Create two custom rule-sets: one allows contraction and the other does not.
// We save these strings in UTF-16 so we can use it without conversions. (ICU
// needs UTF-16 strings.)
const char kAllowContraction[] =
"$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};";
const char kDisallowContraction[] = "";
ruleset_allow_contraction_ = base::ASCIIToUTF16(
base::StringPrintf(kRuleTemplate,
aletter,
aletter_extra,
midletter_extra,
aletter_plus,
kAllowContraction));
ruleset_disallow_contraction_ = base::ASCIIToUTF16(
base::StringPrintf(kRuleTemplate,
aletter,
aletter_extra,
midletter_extra,
aletter_plus,
kDisallowContraction));
}
bool SpellcheckCharAttribute::OutputChar(UChar c,
base::string16* output) const {
// Call the language-specific function if necessary.
// Otherwise, we call the default one.
switch (script_code_) {
case USCRIPT_ARABIC:
return OutputArabic(c, output);
case USCRIPT_HANGUL:
return OutputHangul(c, output);
case USCRIPT_HEBREW:
return OutputHebrew(c, output);
default:
return OutputDefault(c, output);
}
}
bool SpellcheckCharAttribute::OutputArabic(UChar c,
base::string16* output) const {
// Include non-Arabic characters (which should trigger a spelling error)
// and Arabic characters excluding vowel marks and class "Lm".
// We filter the latter because, while they are "letters", they are
// optional and so don't affect the correctness of the rest of the word.
if (!(0x0600 <= c && c <= 0x06FF) || (u_isalpha(c) && c != 0x0640))
output->push_back(c);
return true;
}
bool SpellcheckCharAttribute::OutputHangul(UChar c,
base::string16* output) const {
// Decompose a Hangul character to a Hangul vowel and consonants used by our
// spellchecker. A Hangul character of Unicode is a ligature consisting of a
// Hangul vowel and consonants, e.g. U+AC01 "Gag" consists of U+1100 "G",
// U+1161 "a", and U+11A8 "g". That is, we can treat each Hangul character as
// a point of a cubic linear space consisting of (first consonant, vowel, last
// consonant). Therefore, we can compose a Hangul character from a vowel and
// two consonants with linear composition:
// character = 0xAC00 +
// (first consonant - 0x1100) * 28 * 21 +
// (vowel - 0x1161) * 28 +
// (last consonant - 0x11A7);
// We can also decompose a Hangul character with linear decomposition:
// first consonant = (character - 0xAC00) / 28 / 21;
// vowel = (character - 0xAC00) / 28 % 21;
// last consonant = (character - 0xAC00) % 28;
// This code is copied from Unicode Standard Annex #15
// <http://unicode.org/reports/tr15> and added some comments.
const int kSBase = 0xAC00; // U+AC00: the top of Hangul characters.
const int kLBase = 0x1100; // U+1100: the top of Hangul first consonants.
const int kVBase = 0x1161; // U+1161: the top of Hangul vowels.
const int kTBase = 0x11A7; // U+11A7: the top of Hangul last consonants.
const int kLCount = 19; // The number of Hangul first consonants.
const int kVCount = 21; // The number of Hangul vowels.
const int kTCount = 28; // The number of Hangul last consonants.
const int kNCount = kVCount * kTCount;
const int kSCount = kLCount * kNCount;
int index = c - kSBase;
if (index < 0 || index >= kSBase + kSCount) {
// This is not a Hangul syllable. Call the default output function since we
// should output this character when it is a Hangul syllable.
return OutputDefault(c, output);
}
// This is a Hangul character. Decompose this characters into Hangul vowels
// and consonants.
int l = kLBase + index / kNCount;
int v = kVBase + (index % kNCount) / kTCount;
int t = kTBase + index % kTCount;
output->push_back(l);
output->push_back(v);
if (t != kTBase)
output->push_back(t);
return true;
}
bool SpellcheckCharAttribute::OutputHebrew(UChar c,
base::string16* output) const {
// Discard characters except Hebrew alphabets. We also discard Hebrew niqquds
// to prevent our Hebrew dictionary from marking a Hebrew word including
// niqquds as misspelled. (Same as Arabic vowel marks, we need to check
// niqquds manually and filter them out since their script codes are
// USCRIPT_HEBREW.)
// Pass through ASCII single/double quotation marks and Hebrew Geresh and
// Gershayim.
if ((0x05D0 <= c && c <= 0x05EA) || c == 0x22 || c == 0x27 ||
c == 0x05F4 || c == 0x05F3)
output->push_back(c);
return true;
}
bool SpellcheckCharAttribute::OutputDefault(UChar c,
base::string16* output) const {
// Check the script code of this character and output only if it is the one
// used by the spellchecker language.
UErrorCode status = U_ZERO_ERROR;
UScriptCode script_code = uscript_getScript(c, &status);
if (script_code == script_code_ || script_code == USCRIPT_COMMON)
output->push_back(c);
return true;
}
// SpellcheckWordIterator implementation:
SpellcheckWordIterator::SpellcheckWordIterator()
: text_(nullptr), attribute_(nullptr), iterator_() {}
SpellcheckWordIterator::~SpellcheckWordIterator() {
Reset();
}
bool SpellcheckWordIterator::Initialize(
const SpellcheckCharAttribute* attribute,
bool allow_contraction) {
// Create a custom ICU break iterator with empty text used in this object. (We
// allow setting text later so we can re-use this iterator.)
DCHECK(attribute);
const base::string16 rule(attribute->GetRuleSet(allow_contraction));
// If there is no rule set, the attributes were invalid.
if (rule.empty())
return false;
std::unique_ptr<base::i18n::BreakIterator> iterator(
new base::i18n::BreakIterator(base::string16(), rule));
if (!iterator->Init()) {
// Since we're not passing in any text, the only reason this could fail
// is if we fail to parse the rules. Since the rules are hardcoded,
// that would be a bug in this class.
NOTREACHED() << "failed to open iterator (broken rules)";
return false;
}
iterator_ = std::move(iterator);
// Set the character attributes so we can normalize the words extracted by
// this iterator.
attribute_ = attribute;
return true;
}
bool SpellcheckWordIterator::IsInitialized() const {
// Return true iff we have an iterator.
return !!iterator_;
}
bool SpellcheckWordIterator::SetText(const base::char16* text, size_t length) {
DCHECK(!!iterator_);
// Set the text to be split by this iterator.
if (!iterator_->SetText(text, length)) {
LOG(ERROR) << "failed to set text";
return false;
}
text_ = text;
return true;
}
SpellcheckWordIterator::WordIteratorStatus SpellcheckWordIterator::GetNextWord(
base::string16* word_string,
int* word_start,
int* word_length) {
DCHECK(!!text_);
word_string->clear();
*word_start = 0;
*word_length = 0;
if (!text_) {
return IS_END_OF_TEXT;
}
// Find a word that can be checked for spelling or a character that can be
// skipped over. Rather than moving past a skippable character this returns
// IS_SKIPPABLE and defers handling the character to the calling function.
while (iterator_->Advance()) {
const size_t start = iterator_->prev();
const size_t length = iterator_->pos() - start;
switch (iterator_->GetWordBreakStatus()) {
case base::i18n::BreakIterator::IS_WORD_BREAK: {
if (Normalize(start, length, word_string)) {
*word_start = start;
*word_length = length;
return IS_WORD;
}
break;
}
case base::i18n::BreakIterator::IS_SKIPPABLE_WORD: {
*word_string = iterator_->GetString();
*word_start = start;
*word_length = length;
return IS_SKIPPABLE;
}
// |iterator_| is RULE_BASED so the break status should never be
// IS_LINE_OR_CHAR_BREAK.
case base::i18n::BreakIterator::IS_LINE_OR_CHAR_BREAK: {
NOTREACHED();
break;
}
}
}
// There aren't any more words in the given text.
return IS_END_OF_TEXT;
}
void SpellcheckWordIterator::Reset() {
iterator_.reset();
}
bool SpellcheckWordIterator::Normalize(int input_start,
int input_length,
base::string16* output_string) const {
// We use NFKC (Normalization Form, Compatible decomposition, followed by
// canonical Composition) defined in Unicode Standard Annex #15 to normalize
// this token because it it the most suitable normalization algorithm for our
// spellchecker. Nevertheless, it is not a perfect algorithm for our
// spellchecker and we need manual normalization as well. The normalized
// text does not have to be NUL-terminated since its characters are copied to
// string16, which adds a NUL character when we need.
icu::UnicodeString input(FALSE, &text_[input_start], input_length);
UErrorCode status = U_ZERO_ERROR;
icu::UnicodeString output;
icu::Normalizer::normalize(input, UNORM_NFKC, 0, output, status);
if (status != U_ZERO_ERROR && status != U_STRING_NOT_TERMINATED_WARNING)
return false;
// Copy the normalized text to the output.
icu::StringCharacterIterator it(output);
for (UChar c = it.first(); c != icu::CharacterIterator::DONE; c = it.next())
attribute_->OutputChar(c, output_string);
return !output_string->empty();
}