blob: ef400a5747dd2ffc1843effc76d6c7790a837b8b [file] [log] [blame]
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/url_formatter/idn_spoof_checker.h"
#include "base/numerics/safe_conversions.h"
#include "base/strings/string_piece.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/threading/thread_local_storage.h"
#include "build/build_config.h"
#include "net/base/lookup_string_in_fixed_set.h"
#include "third_party/icu/source/common/unicode/schriter.h"
#include "third_party/icu/source/common/unicode/unistr.h"
#include "third_party/icu/source/i18n/unicode/regex.h"
#include "third_party/icu/source/i18n/unicode/translit.h"
#include "third_party/icu/source/i18n/unicode/uspoof.h"
namespace url_formatter {
namespace {
base::ThreadLocalStorage::StaticSlot tls_index = TLS_INITIALIZER;
void OnThreadTermination(void* regex_matcher) {
delete reinterpret_cast<icu::RegexMatcher*>(regex_matcher);
#include "components/url_formatter/top_domains/"
// All the domains in the above file have 3 or fewer labels.
const size_t kNumberOfLabelsToCheck = 3;
const unsigned char* g_graph = kDafsa;
size_t g_graph_length = sizeof(kDafsa);
bool LookupMatchInTopDomains(base::StringPiece skeleton) {
DCHECK_NE(skeleton.back(), '.');
auto labels = base::SplitStringPiece(skeleton, ".", base::KEEP_WHITESPACE,
if (labels.size() > kNumberOfLabelsToCheck) {
labels.begin() + labels.size() - kNumberOfLabelsToCheck);
while (labels.size() > 1) {
std::string partial_skeleton = base::JoinString(labels, ".");
if (net::LookupStringInFixedSet(
g_graph, g_graph_length,,
partial_skeleton.length()) != net::kDafsaNotFound)
return true;
return false;
} // namespace
IDNSpoofChecker::IDNSpoofChecker() {
UErrorCode status = U_ZERO_ERROR;
checker_ = uspoof_open(&status);
if (U_FAILURE(status)) {
checker_ = nullptr;
// At this point, USpoofChecker has all the checks enabled except
// This default configuration is adjusted below as necessary.
// Set the restriction level to high. It allows mixing Latin with one logical
// CJK script (+ COMMON and INHERITED), but does not allow any other script
// mixing (e.g. Latin + Cyrillic, Latin + Armenian, Cyrillic + Greek). Note
// that each of {Han + Bopomofo} for Chinese, {Hiragana, Katakana, Han} for
// Japanese, and {Hangul, Han} for Korean is treated as a single logical
// script.
// See
uspoof_setRestrictionLevel(checker_, USPOOF_HIGHLY_RESTRICTIVE);
// Sets allowed characters in IDN labels and turns on USPOOF_CHAR_LIMIT.
// Enable the return of auxillary (non-error) information.
// We used to disable WHOLE_SCRIPT_CONFUSABLE check explicitly, but as of
// ICU 58.1, WSC is a no-op in a single string check API.
int32_t checks = uspoof_getChecks(checker_, &status) | USPOOF_AUX_INFO;
uspoof_setChecks(checker_, checks, &status);
// Four characters handled differently by IDNA 2003 and IDNA 2008. UTS46
// transitional processing treats them as IDNA 2003 does; maps U+00DF and
// U+03C2 and drops U+200[CD].
deviation_characters_ = icu::UnicodeSet(
UNICODE_STRING_SIMPLE("[\\u00df\\u03c2\\u200c\\u200d]"), status);
// Latin letters outside ASCII. 'Script_Extensions=Latin' is not necessary
// because additional characters pulled in with scx=Latn are not included in
// the allowed set.
non_ascii_latin_letters_ =
icu::UnicodeSet(UNICODE_STRING_SIMPLE("[[:Latin:] - [a-zA-Z]]"), status);
// The following two sets are parts of |dangerous_patterns_|.
kana_letters_exceptions_ = icu::UnicodeSet(
combining_diacritics_exceptions_ =
icu::UnicodeSet(UNICODE_STRING_SIMPLE("[\\u0300-\\u0339]"), status);
// These Cyrillic letters look like Latin. A domain label entirely made of
// these letters is blocked as a simplified whole-script-spoofable.
cyrillic_letters_latin_alike_ = icu::UnicodeSet(
icu::UnicodeString::fromUTF8("[асԁеһіјӏорԛѕԝхуъЬҽпгѵѡ]"), status);
cyrillic_letters_ =
icu::UnicodeSet(UNICODE_STRING_SIMPLE("[[:Cyrl:]]"), status);
// This set is used to determine whether or not to apply a slow
// transliteration to remove diacritics to a given hostname before the
// confusable skeleton calculation for comparison with top domain names. If
// it has any character outside the set, the expensive step will be skipped
// because it cannot match any of top domain names.
// The last ([\u0300-\u0339] is a shorthand for "[:Identifier_Status=Allowed:]
// & [:Script_Extensions=Inherited:] - [\\u200C\\u200D]". The latter is a
// subset of the former but it does not matter because hostnames with
// characters outside the latter set would be rejected in an earlier step.
lgc_letters_n_ascii_ = icu::UnicodeSet(
// Used for diacritics-removal before the skeleton calculation. Add
// "ł > l; ø > o; đ > d" that are not handled by "NFD; Nonspacing mark
// removal; NFC".
// TODO(jshin): Revisit "ł > l; ø > o" mapping.
UParseError parse_error;
icu::UnicodeString::fromUTF8("::NFD; ::[:Nonspacing Mark:] Remove; ::NFC;"
" ł > l; ø > o; đ > d;"),
UTRANS_FORWARD, parse_error, status));
// Supplement the Unicode confusable list by the following mapping.
// - {U+00FE (þ), U+03FC (ϼ), U+048F (ҏ)} => p
// - {U+0127 (ħ), U+043D (н), U+045B (ћ), U+04A3 (ң), U+04A5 (ҥ),
// U+04C8 (ӈ), U+0527 (ԧ), U+0529 (ԩ)} => h
// - {U+0138 (ĸ), U+03BA (κ), U+043A (к), U+049B (қ), U+049D (ҝ),
// U+049F (ҟ), U+04A1(ҡ), U+04C4 (ӄ), U+051F (ԟ)} => k
// - {U+0167 (ŧ), U+0442 (т), U+04AD (ҭ)} => t
// - {U+0185 (ƅ), U+044C (ь), U+048D (ҍ), U+0432 (в)} => b
// - {U+03C9 (ω), U+0448 (ш), U+0449 (щ)} => w
// - {U+043C (м), U+04CE (ӎ)} => m
// - U+043F(п) => n
// - U+0491 (ґ) => r
// - U+0493 (ғ) => f
// - U+04AB (ҫ) => c
// - U+04B1 (ұ) => y
// - U+03C7 (χ), U+04B3 (ҳ), U+04FD (ӽ), U+04FF (ӿ) => x
// - U+04BD (ҽ), U+04BF (ҿ) => e
// - U+04CF (ӏ) => i (on Windows), l (elsewhere)
// - U+0503 (ԃ) => d
// - U+050D (ԍ) => g
// - U+0D1F (ട) => s
icu::UnicodeString::fromUTF8("[þϼҏ] > p; [ħнћңҥӈԧԩ] > h;"
"[ĸκкқҝҟҡӄԟ] > k; [ŧтҭ] > t;"
"[ƅьҍв] > b; [ωшщ] > w; [мӎ] > m;"
"п > n; ћ > h; ґ > r; ғ > f; ҫ > c;"
"ұ > y; [χҳӽӿ] > x; [ҽҿ] > e;"
#if defined(OS_WIN)
"ӏ > i;"
"ӏ > l;"
"ԃ > d; ԍ > g; ട > s"),
UTRANS_FORWARD, parse_error, status));
<< "Spoofchecker initalization failed due to an error: "
<< u_errorName(status);
IDNSpoofChecker::~IDNSpoofChecker() {
bool IDNSpoofChecker::SafeToDisplayAsUnicode(base::StringPiece16 label,
bool is_tld_ascii) {
UErrorCode status = U_ZERO_ERROR;
int32_t result =
base::checked_cast<int32_t>(label.size()), nullptr, &status);
// If uspoof_check fails (due to library failure), or if any of the checks
// fail, treat the IDN as unsafe.
if (U_FAILURE(status) || (result & USPOOF_ALL_CHECKS))
return false;
icu::UnicodeString label_string(FALSE,,
// A punycode label with 'xn--' prefix is not subject to the URL
// canonicalization and is stored as it is in GURL. If it encodes a deviation
// character (UTS 46; e.g. U+00DF/sharp-s), it should be still shown in
// punycode instead of Unicode. Without this check, xn--fu-hia for
// 'fu<sharp-s>' would be converted to 'fu<sharp-s>' for display because
// "UTS 46 section 4 Processing step 4" applies validity criteria for
// non-transitional processing (i.e. do not map deviation characters) to any
// punycode labels regardless of whether transitional or non-transitional is
// chosen. On the other hand, 'fu<sharp-s>' typed or copy and pasted
// as Unicode would be canonicalized to 'fuss' by GURL and is displayed as
// such. See .
if (deviation_characters_.containsSome(label_string))
return false;
// If there's no script mixing, the input is regarded as safe without any
// extra check unless it falls into one of three categories:
// - contains Kana letter exceptions
// - the TLD is ASCII and the input is made entirely of Cyrillic letters
// that look like Latin letters.
// - it has combining diacritic marks.
// Note that the following combinations of scripts are treated as a 'logical'
// single script.
// - Chinese: Han, Bopomofo, Common
// - Japanese: Han, Hiragana, Katakana, Common
// - Korean: Hangul, Han, Common
if (result == USPOOF_ASCII)
return true;
kana_letters_exceptions_.containsNone(label_string) &&
combining_diacritics_exceptions_.containsNone(label_string)) {
// Check Cyrillic confusable only for ASCII TLDs.
return !is_tld_ascii || !IsMadeOfLatinAlikeCyrillic(label_string);
// Additional checks for |label| with multiple scripts, one of which is Latin.
// Disallow non-ASCII Latin letters to mix with a non-Latin script.
// Note that the non-ASCII Latin check should not be applied when the entire
// label is made of Latin. Checking with lgc_letters set here should be fine
// because script mixing of LGC is already rejected.
if (non_ascii_latin_letters_.containsSome(label_string) &&
return false;
if (!tls_index.initialized())
icu::RegexMatcher* dangerous_pattern =
if (!dangerous_pattern) {
// Disallow the katakana no, so, zo, or n, as they may be mistaken for
// slashes when they're surrounded by non-Japanese scripts (i.e. scripts
// other than Katakana, Hiragana or Han). If {no, so, zo, n} next to a
// non-Japanese script on either side is disallowed, legitimate cases like
// '{vitamin in Katakana}b6' are blocked. Note that trying to block those
// characters when used alone as a label is futile because those cases
// would not reach here.
// Also disallow what used to be blocked by mixed-script-confusable (MSC)
// detection. ICU 58 does not detect MSC any more for a single input string.
// See .
// TODO(jshin): adjust the pattern once the above ICU bug is fixed.
// - Disallow U+30FB (Katakana Middle Dot) and U+30FC (Hiragana-Katakana
// Prolonged Sound) used out-of-context.
// - Dislallow U+30FD/E (Katakana iteration mark/voiced iteration mark)
// unless they're preceded by a Katakana.
// - Disallow three Hiragana letters (U+307[8-A]) or Katakana letters
// (U+30D[8-A]) that look exactly like each other when they're used in a
// label otherwise entirely in Katakna or Hiragana.
// - Disallow U+0585 (Armenian Small Letter Oh) and U+0581 (Armenian Small
// Letter Co) to be next to Latin.
// - Disallow combining diacritical mark (U+0300-U+0339) after a non-LGC
// character. Other combining diacritical marks are not in the allowed
// character set.
// - Disallow dotless i (U+0131) followed by a combining mark.
// - Disallow U+0307 (dot above) after 'i', 'j', 'l' or dotless i (U+0131).
// Dotless j (U+0237) is not in the allowed set to begin with.
dangerous_pattern = new icu::RegexMatcher(
-1, US_INV),
0, status);
return !dangerous_pattern->find();
bool IDNSpoofChecker::SimilarToTopDomains(base::StringPiece16 hostname) {
size_t hostname_length = hostname.length() - (hostname.back() == '.' ? 1 : 0);
icu::UnicodeString ustr_host(FALSE,, hostname_length);
// If input has any characters outside Latin-Greek-Cyrillic and [0-9._-],
// there is no point in getting rid of diacritics because combining marks
// attached to non-LGC characters are already blocked.
if (lgc_letters_n_ascii_.span(ustr_host, 0, USET_SPAN_CONTAINED) ==
UErrorCode status = U_ZERO_ERROR;
icu::UnicodeString ustr_skeleton;
uspoof_getSkeletonUnicodeString(checker_, 0, ustr_host, ustr_skeleton,
if (U_FAILURE(status))
return false;
std::string skeleton;
return LookupMatchInTopDomains(ustr_skeleton.toUTF8String(skeleton));
bool IDNSpoofChecker::IsMadeOfLatinAlikeCyrillic(
const icu::UnicodeString& label) {
// Collect all the Cyrillic letters in |label_string| and see if they're
// a subset of |cyrillic_letters_latin_alike_|.
// A shortcut of defining cyrillic_letters_latin_alike_ to include [0-9] and
// [_-] and checking if the set contains all letters of |label|
// would work in most cases, but not if a label has non-letters outside
icu::UnicodeSet cyrillic_in_label;
icu::StringCharacterIterator it(label);
for (it.setToStart(); it.hasNext();) {
const UChar32 c = it.next32PostInc();
if (cyrillic_letters_.contains(c))
return !cyrillic_in_label.isEmpty() &&
void IDNSpoofChecker::SetAllowedUnicodeSet(UErrorCode* status) {
if (U_FAILURE(*status))
// The recommended set is a set of characters for identifiers in a
// security-sensitive environment taken from UTR 39
// ( and
// .
// The inclusion set comes from "Candidate Characters for Inclusion
// in idenfiers" of UTR 31 ( The list
// may change over the time and will be updated whenever the version of ICU
// used in Chromium is updated.
const icu::UnicodeSet* recommended_set =
icu::UnicodeSet allowed_set;
const icu::UnicodeSet* inclusion_set = uspoof_getInclusionUnicodeSet(status);
// The sections below refer to Mozilla's IDN blacklist:
// U+0338 (Combining Long Solidus Overlay) is included in the recommended set,
// but is blacklisted by Mozilla. It is dropped because it can look like a
// slash when rendered with a broken font.
// U+05F4 (Hebrew Punctuation Gershayim) is in the inclusion set, but is
// blacklisted by Mozilla. We keep it, even though it can look like a double
// quotation mark. Using it in Hebrew should be safe. When used with a
// non-Hebrew script, it'd be filtered by other checks in place.
// The following 5 characters are disallowed because they're in NV8 (invalid
// in IDNA 2008).
allowed_set.remove(0x58au); // Armenian Hyphen
// U+2010 (Hyphen) is in the inclusion set, but we drop it because it can be
// confused with an ASCII U+002D (Hyphen-Minus).
// U+2019 is hard to notice when sitting next to a regular character.
allowed_set.remove(0x2019u); // Right Single Quotation Mark
// U+2027 (Hyphenation Point) is in the inclusion set, but is blacklisted by
// Mozilla. It is dropped, as it can be confused with U+30FB (Katakana Middle
// Dot).
allowed_set.remove(0x30a0u); // Katakana-Hiragana Double Hyphen
// Block {Single,double}-quotation-mark look-alikes.
allowed_set.remove(0x2bbu); // Modifier Letter Turned Comma
allowed_set.remove(0x2bcu); // Modifier Letter Apostrophe
// No need to block U+144A (Canadian Syllabics West-Cree P) separately
// because it's blocked from mixing with other scripts including Latin.
#if defined(OS_MACOSX)
// The following characters are reported as present in the default macOS
// system UI font, but they render as blank. Remove them from the allowed
// set to prevent spoofing until the font issue is resolved.
// Arabic letter KASHMIRI YEH. Not used in Arabic and Persian.
// Tibetan characters used for transliteration of ancient texts:
// Disallow extremely rarely used LGC character blocks.
// Cyllic Ext A is not in the allowed set. Neither are Latin Ext-{C,E}.
allowed_set.remove(0x01CDu, 0x01DCu); // Latin Ext B; Pinyin
allowed_set.remove(0x1C80u, 0x1C8Fu); // Cyrillic Extended-C
allowed_set.remove(0x1E00u, 0x1E9Bu); // Latin Extended Additional
allowed_set.remove(0x1F00u, 0x1FFFu); // Greek Extended
allowed_set.remove(0xA640u, 0xA69Fu); // Cyrillic Extended-B
allowed_set.remove(0xA720u, 0xA7FFu); // Latin Extended-D
uspoof_setAllowedUnicodeSet(checker_, &allowed_set, status);
void IDNSpoofChecker::RestoreTopDomainGraphToDefault() {
g_graph = kDafsa;
g_graph_length = sizeof(kDafsa);
void IDNSpoofChecker::SetTopDomainGraph(base::StringPiece domain_graph) {
DCHECK_NE(0u, domain_graph.length());
g_graph = reinterpret_cast<const unsigned char*>(;
g_graph_length = domain_graph.length();
} // namespace url_formatter