| // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| // This file extends lang_enc.cc with additional languages and extended routines |
| // It is current with Unicode 5.1 (beta Jan 2008) |
| // |
| |
| #include <stdlib.h> |
| #include <stdio.h> |
| #include <string.h> |
| |
| #include "encodings/compact_lang_det/ext_lang_enc.h" |
| #include "encodings/compact_lang_det/win/cld_macros.h" |
| #include "encodings/compact_lang_det/win/cld_strtoint.h" |
| |
| // Language names above NUM_LANGUAGES |
| // These are also the C enum declared names |
| static const char* const kExtLanguageName[] = { |
| "X_BORK_BORK_BORK", "X_PIG_LATIN", "X_HACKER", "X_KLINGON", "X_ELMER_FUDD", |
| |
| // Pseudo-languages for Unicode scripts that express a single language |
| "X_OGHAM", "X_RUNIC", "X_YI", "X_OLD_ITALIC", "X_GOTHIC", |
| "X_DESERET", "X_HANUNOO", "X_BUHID", "X_TAGBANWA", "X_TAI_LE", |
| "X_LINEAR_B", "X_UGARITIC", "X_SHAVIAN", "X_OSMANYA", "X_CYPRIOT", |
| "X_BUGINESE", "X_COPTIC", "X_NEW_TAI_LUE", "X_GLAGOLITIC", "X_TIFINAGH", |
| "X_SYLOTI_NAGRI", "X_OLD_PERSIAN", "X_KHAROSHTHI", "X_BALINESE", "X_CUNEIFORM", |
| "X_PHOENICIAN", "X_PHAGS_PA", "X_NKO", |
| |
| // Unicode 5.1 |
| "X_SUDANESE", "X_LEPCHA", "X_OL_CHIKI", "X_VAI", "X_SAURASHTRA", |
| "X_KAYAH_LI", "X_REJANG", "X_LYCIAN", "X_CARIAN", "X_LYDIAN", |
| "X_CHAM", |
| }; |
| |
| |
| // These are the C enum declared names, for programs creating C code |
| static const char* const kExtLangDeclaredName[] = { |
| "ENGLISH", /* 0 */ |
| "DANISH", /* 1 */ |
| "DUTCH", /* 2 */ |
| "FINNISH", /* 3 */ |
| "FRENCH", /* 4 */ |
| "GERMAN", /* 5 */ |
| "HEBREW", /* 6 */ |
| "ITALIAN", /* 7 */ |
| "JAPANESE", /* 8 */ |
| "KOREAN", /* 9 */ |
| "NORWEGIAN", /* 10 */ |
| "POLISH", /* 11 */ |
| "PORTUGUESE", /* 12 */ |
| "RUSSIAN", /* 13 */ |
| "SPANISH", /* 14 */ |
| "SWEDISH", /* 15 */ |
| "CHINESE", /* 16 */ |
| "CZECH", /* 17 */ |
| "GREEK", /* 18 */ |
| "ICELANDIC", /* 19 */ |
| "LATVIAN", /* 20 */ |
| "LITHUANIAN", /* 21 */ |
| "ROMANIAN", /* 22 */ |
| "HUNGARIAN", /* 23 */ |
| "ESTONIAN", /* 24 */ |
| "TG_UNKNOWN_LANGUAGE", /* 25 */ |
| "UNKNOWN_LANGUAGE", /* 26 */ |
| "BULGARIAN", /* 27 */ |
| "CROATIAN", /* 28 */ |
| "SERBIAN", /* 29 */ |
| "IRISH", /* 30 */ |
| "GALICIAN", /* 31 */ |
| "TAGALOG", /* 32 */ |
| "TURKISH", /* 33 */ |
| "UKRAINIAN", /* 34 */ |
| "HINDI", /* 35 */ |
| "MACEDONIAN", /* 36 */ |
| "BENGALI", /* 37 */ |
| "INDONESIAN", /* 38 */ |
| "LATIN", /* 39 */ |
| "MALAY", /* 40 */ |
| "MALAYALAM", /* 41 */ |
| "WELSH", /* 42 */ |
| "NEPALI", /* 43 */ |
| "TELUGU", /* 44 */ |
| "ALBANIAN", /* 45 */ |
| "TAMIL", /* 46 */ |
| "BELARUSIAN", /* 47 */ |
| "JAVANESE", /* 48 */ |
| "OCCITAN", /* 49 */ |
| "URDU", /* 50 */ |
| "BIHARI", /* 51 */ |
| "GUJARATI", /* 52 */ |
| "THAI", /* 53 */ |
| "ARABIC", /* 54 */ |
| "CATALAN", /* 55 */ |
| "ESPERANTO", /* 56 */ |
| "BASQUE", /* 57 */ |
| "INTERLINGUA", /* 58 */ |
| "KANNADA", /* 59 */ |
| "PUNJABI", /* 60 */ |
| "SCOTS_GAELIC", /* 61 */ |
| "SWAHILI", /* 62 */ |
| "SLOVENIAN", /* 63 */ |
| "MARATHI", /* 64 */ |
| "MALTESE", /* 65 */ |
| "VIETNAMESE", /* 66 */ |
| "FRISIAN", /* 67 */ |
| "SLOVAK", /* 68 */ |
| "CHINESE_T", /* 69 */ |
| "FAROESE", /* 70 */ |
| "SUNDANESE", /* 71 */ |
| "UZBEK", /* 72 */ |
| "AMHARIC", /* 73 */ |
| "AZERBAIJANI", /* 74 */ |
| "GEORGIAN", /* 75 */ |
| "TIGRINYA", /* 76 */ |
| "PERSIAN", /* 77 */ |
| "BOSNIAN", /* 78 */ |
| "SINHALESE", /* 79 */ |
| "NORWEGIAN_N", /* 80 */ |
| "PORTUGUESE_P", /* 81 */ |
| "PORTUGUESE_B", /* 82 */ |
| "XHOSA", /* 83 */ |
| "ZULU", /* 84 */ |
| "GUARANI", /* 85 */ |
| "SESOTHO", /* 86 */ |
| "TURKMEN", /* 87 */ |
| "KYRGYZ", /* 88 */ |
| "BRETON", /* 89 */ |
| "TWI", /* 90 */ |
| "YIDDISH", /* 91 */ |
| "SERBO_CROATIAN", /* 92 */ |
| "SOMALI", /* 93 */ |
| "UIGHUR", /* 94 */ |
| "KURDISH", /* 95 */ |
| "MONGOLIAN", /* 96 */ |
| "ARMENIAN", /* 97 */ |
| "LAOTHIAN", /* 98 */ |
| "SINDHI", /* 99 */ |
| "RHAETO_ROMANCE", /* 100 */ |
| "AFRIKAANS", /* 101 */ |
| "LUXEMBOURGISH", /* 102 */ |
| "BURMESE", /* 103 */ |
| "KHMER", /* 104 */ |
| "TIBETAN", /* 105 */ |
| "DHIVEHI", /* 106 */ // sometimes spelled Divehi; lang of Maldives |
| "CHEROKEE", /* 107 */ |
| "SYRIAC", /* 108 */ |
| "LIMBU", /* 109 */ |
| "ORIYA", /* 110 */ |
| "ASSAMESE", /* 111 */ |
| "CORSICAN", /* 112 */ |
| "INTERLINGUE", /* 113 */ |
| "KAZAKH", /* 114 */ |
| "LINGALA", /* 115 */ |
| "MOLDAVIAN", /* 116 */ |
| "PASHTO", /* 117 */ |
| "QUECHUA", /* 118 */ |
| "SHONA", /* 119 */ |
| "TAJIK", /* 120 */ |
| "TATAR", /* 121 */ |
| "TONGA", /* 122 */ |
| "YORUBA", /* 123 */ |
| "CREOLES_AND_PIDGINS_ENGLISH_BASED", /* 124 */ |
| "CREOLES_AND_PIDGINS_FRENCH_BASED", /* 125 */ |
| "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", /* 126 */ |
| "CREOLES_AND_PIDGINS_OTHER", /* 127 */ |
| "MAORI", /* 128 */ |
| "WOLOF", /* 129 */ |
| "ABKHAZIAN", /* 130 */ |
| "AFAR", /* 131 */ |
| "AYMARA", /* 132 */ |
| "BASHKIR", /* 133 */ |
| "BISLAMA", /* 134 */ |
| "DZONGKHA", /* 135 */ |
| "FIJIAN", /* 136 */ |
| "GREENLANDIC", /* 137 */ |
| "HAUSA", /* 138 */ |
| "HAITIAN_CREOLE", /* 139 */ |
| "INUPIAK", /* 140 */ |
| "INUKTITUT", /* 141 */ |
| "KASHMIRI", /* 142 */ |
| "KINYARWANDA", /* 143 */ |
| "MALAGASY", /* 144 */ |
| "NAURU", /* 145 */ |
| "OROMO", /* 146 */ |
| "RUNDI", /* 147 */ |
| "SAMOAN", /* 148 */ |
| "SANGO", /* 149 */ |
| "SANSKRIT", /* 150 */ |
| "SISWANT", /* 151 */ |
| "TSONGA", /* 152 */ |
| "TSWANA", /* 153 */ |
| "VOLAPUK", /* 154 */ |
| "ZHUANG", /* 155 */ |
| "KHASI", /* 156 */ |
| "SCOTS", /* 157 */ |
| "GANDA", /* 158 */ |
| "MANX", /* 159 */ |
| "MONTENEGRIN", /* 160 */ |
| // Add new language declared names just before here |
| }; |
| |
| COMPILE_ASSERT(arraysize(kExtLangDeclaredName) == NUM_LANGUAGES, |
| kExtLangDeclaredName_has_incorrect_length); |
| |
| |
| // Language codes above NUM_LANGUAGES |
| // I made all these up, except Klingon from ISO-639-2 (dsites) |
| // NOTE: zza is a standard name |
| static const char* const kExtLanguageCode[] = { |
| // "X_BORK_BORK_BORK", "X_PIG_LATIN", "X_HACKER", "X_KLINGON", "X_ELMER_FUDD", |
| // All Latin script |
| "zzb", "zzp", "zzh", "tlh", "zze", |
| |
| // Pseudo-languages for Unicode scripts that express a single language |
| "xx-Ogam", "xx-Runr", "xx-Yiii", "xx-Ital", "xx-Goth", |
| "xx-Dsrt", "xx-Hano", "xx-Buhd", "xx-Tagb", "xx-Tale", |
| "xx-Linb", "xx-Ugar", "xx-Shaw", "xx-Osma", "xx-Cprt", |
| "xx-Bugi", "xx-Copt", "xx-Talu", "xx-Glag", "xx-Tfng", |
| "xx-Sylo", "xx-Xpeo", "xx-Khar", "xx-Bali", "xx-Xsux", |
| "xx-Phnx", "xx-Phag", "xx-Nkoo", |
| |
| // Unicode 5.1 |
| "xx-Sund", "xx-Lepc", "xx-Olck", "xx-Vaii", "xx-Saur", |
| "xx-Kali", "xx-Rjng", "xx-Lyci", "xx-Cari", "xx-Lydi", |
| "xx-Cham", |
| }; |
| |
| |
| // Given the Language, returns its string name used as the output by |
| // the lang/enc identifier, e.g. "Korean" |
| // "invalid_language" if the input is invalid. |
| // TG_UNKNOWN_LANGUAGE is used as a placeholder for the "ignore me" language, |
| // used to subtract out HTML, link farms, DNA strings, and alittle English porn |
| const char* ExtLanguageName(const Language lang) { |
| if (lang < 0) { |
| // No-text-at-all result from a Tote |
| return ""; |
| } |
| // CompactLanguageDetect extension |
| if (lang == TG_UNKNOWN_LANGUAGE) { |
| return "Ignore"; |
| } |
| if ((0 <= lang) && (lang < NUM_LANGUAGES)) { |
| return LanguageName(lang); |
| } |
| if ((EXT_LANGUAGE_BASE <= lang) && (lang < EXT_NUM_LANGUAGES)) { |
| return kExtLanguageName[lang - EXT_LANGUAGE_BASE]; |
| } |
| return invalid_language_name(); |
| } |
| |
| |
| // Given the Language, returns its Language enum spelling, for use by |
| // programs that create C declarations, e.g. "KOREAN" |
| // "UNKNOWN_LANGUAGE" if the input is invalid. |
| const char* ExtLanguageDeclaredName(const Language lang) { |
| if ((0 <= lang) && (lang < NUM_LANGUAGES)) { |
| return kExtLangDeclaredName[lang]; |
| } |
| if ((EXT_LANGUAGE_BASE <= lang) && (lang < EXT_NUM_LANGUAGES)) { |
| return kExtLanguageName[lang - EXT_LANGUAGE_BASE]; |
| } |
| return "UNKNOWN_LANGUAGE"; |
| } |
| |
| // Given the Language, return the language code, e.g. "ko" |
| const char* ExtLanguageCode(const Language lang) { |
| // Hack for ignore/porn pseudo-language |
| if (lang == TG_UNKNOWN_LANGUAGE) { |
| return "xxx"; |
| } |
| if ((0 <= lang) && (lang < NUM_LANGUAGES)) { |
| return LanguageCode(lang); |
| } |
| if ((EXT_LANGUAGE_BASE <= lang) && (lang < EXT_NUM_LANGUAGES)) { |
| return kExtLanguageCode[lang - EXT_LANGUAGE_BASE]; |
| } |
| return "??"; |
| } |
| |
| |
| // Convert "en-Latn-GB" to ENGLISH |
| // Normalize to PORTUGUESE, not PORTUGUESE_B nor PORTUGUESE_P |
| // Consider for later: NORWEGIAN, NORWEGIAN_N |
| // Consider for later: SCOTS, SCOTS_GAELIC |
| // Consider for later: SERBO_CROATIAN, SERBIAN, CROATIAN, BOSNIAN |
| // |
| Language GetLanguageFromNumberOrName(const char* src) { |
| if (strspn(src, "0123456789") == strlen(src)) { |
| // All digits |
| return static_cast<Language>(strto32(src, NULL, 10)); |
| } |
| |
| Language retlang = UNKNOWN_LANGUAGE; |
| size_t len = strlen(src); |
| |
| if (true /*FLAGS_mergepairs*/) { |
| // Merge sets of langauges pt-xx en-xx fr-xx, NOT bs/hr/sr |
| if (memcmp(src, "pt-", 3) == 0) {return PORTUGUESE;} |
| if (memcmp(src, "en-", 3) == 0) {return ENGLISH;} |
| if (memcmp(src, "fr-", 3) == 0) {return FRENCH;} |
| // Use NormalizeLanguage instead |
| if (memcmp(src, "bs-", 3) == 0) {return CROATIAN;} |
| if (memcmp(src, "hr-", 3) == 0) {return CROATIAN;} |
| if (memcmp(src, "sr-Latn", 7) == 0) {return CROATIAN;} |
| if (memcmp(src, "sh-Latn", 7) == 0) {return CROATIAN;} |
| if (memcmp(src, "sr-Cyrl", 7) == 0) {return SERBIAN;} |
| if (memcmp(src, "sh-Cyrl", 7) == 0) {return SERBIAN;} |
| } |
| |
| // Extensions |
| if (len >= 3) { |
| // Standin for ignore/porn "language" |
| if (memcmp(src, "xxx", 3) == 0) {return TG_UNKNOWN_LANGUAGE;} |
| |
| if (memcmp(src, "zzb", 3) == 0) {return X_BORK_BORK_BORK;} |
| if (memcmp(src, "zzp", 3) == 0) {return X_PIG_LATIN;} |
| if (memcmp(src, "zzh", 3) == 0) {return X_HACKER;} |
| if (memcmp(src, "tlh", 3) == 0) {return X_KLINGON;} |
| if (memcmp(src, "zze", 3) == 0) {return X_ELMER_FUDD;} |
| } |
| |
| // We have a name like en-Latn-GB or pt-BR |
| // First, get rid of some special cases |
| if (len <= 3) { |
| LanguageFromCode(src, &retlang); |
| } else if (len == 7) { |
| // More Extensions |
| if (memcmp(src, "xx-", 3) == 0) { |
| if (memcmp(src, "xx-Ogam", 7) == 0) {return X_OGHAM;} |
| if (memcmp(src, "xx-Runr", 7) == 0) {return X_RUNIC;} |
| if (memcmp(src, "xx-Yiii", 7) == 0) {return X_YI;} |
| if (memcmp(src, "xx-Ital", 7) == 0) {return X_OLD_ITALIC;} |
| if (memcmp(src, "xx-Goth", 7) == 0) {return X_GOTHIC;} |
| if (memcmp(src, "xx-Dsrt", 7) == 0) {return X_DESERET;} |
| if (memcmp(src, "xx-Hano", 7) == 0) {return X_HANUNOO;} |
| if (memcmp(src, "xx-Buhd", 7) == 0) {return X_BUHID;} |
| if (memcmp(src, "xx-Tagb", 7) == 0) {return X_TAGBANWA;} |
| if (memcmp(src, "xx-Tale", 7) == 0) {return X_TAI_LE;} |
| if (memcmp(src, "xx-Linb", 7) == 0) {return X_LINEAR_B;} |
| if (memcmp(src, "xx-Ugar", 7) == 0) {return X_UGARITIC;} |
| if (memcmp(src, "xx-Shaw", 7) == 0) {return X_SHAVIAN;} |
| if (memcmp(src, "xx-Osma", 7) == 0) {return X_OSMANYA;} |
| if (memcmp(src, "xx-Cprt", 7) == 0) {return X_CYPRIOT;} |
| if (memcmp(src, "xx-Bugi", 7) == 0) {return X_BUGINESE;} |
| if (memcmp(src, "xx-Copt", 7) == 0) {return X_COPTIC;} |
| if (memcmp(src, "xx-Talu", 7) == 0) {return X_NEW_TAI_LUE;} |
| if (memcmp(src, "xx-Glag", 7) == 0) {return X_GLAGOLITIC;} |
| if (memcmp(src, "xx-Tfng", 7) == 0) {return X_TIFINAGH;} |
| if (memcmp(src, "xx-Sylo", 7) == 0) {return X_SYLOTI_NAGRI;} |
| if (memcmp(src, "xx-Xpeo", 7) == 0) {return X_OLD_PERSIAN;} |
| if (memcmp(src, "xx-Khar", 7) == 0) {return X_KHAROSHTHI;} |
| if (memcmp(src, "xx-Bali", 7) == 0) {return X_BALINESE;} |
| if (memcmp(src, "xx-Xsux", 7) == 0) {return X_CUNEIFORM;} |
| if (memcmp(src, "xx-Phnx", 7) == 0) {return X_PHOENICIAN;} |
| if (memcmp(src, "xx-Phag", 7) == 0) {return X_PHAGS_PA;} |
| if (memcmp(src, "xx-Nkoo", 7) == 0) {return X_NKO;} |
| |
| // Unicode 5.1 |
| if (memcmp(src, "xx-Sund", 7) == 0) {return X_SUDANESE;} |
| if (memcmp(src, "xx-Lepc", 7) == 0) {return X_LEPCHA;} |
| if (memcmp(src, "xx-Olck", 7) == 0) {return X_OL_CHIKI;} |
| if (memcmp(src, "xx-Vaii", 7) == 0) {return X_VAI;} |
| if (memcmp(src, "xx-Saur", 7) == 0) {return X_SAURASHTRA;} |
| if (memcmp(src, "xx-Kali", 7) == 0) {return X_KAYAH_LI;} |
| if (memcmp(src, "xx-Rjng", 7) == 0) {return X_REJANG;} |
| if (memcmp(src, "xx-Lyci", 7) == 0) {return X_LYCIAN;} |
| if (memcmp(src, "xx-Cari", 7) == 0) {return X_CARIAN;} |
| if (memcmp(src, "xx-Lydi", 7) == 0) {return X_LYDIAN;} |
| if (memcmp(src, "xx-Cham", 7) == 0) {return X_CHAM;} |
| } |
| } |
| // Some other weird ones |
| // Could be Latn or Limb; all our current training data is Latn |
| if (strcmp(src, "sit-NP") == 0) {return LIMBU;} |
| if (strcmp(src, "un-Latn") == 0) {return UNKNOWN_LANGUAGE;} |
| |
| // Multi-country langauges |
| if (memcmp(src, "zh", 2) == 0) { |
| if (memcmp(&src[len - 2], "TW", 2) == 0) {return CHINESE_T;} |
| if (memcmp(&src[len - 2], "HK", 2) == 0) {return CHINESE_T;} |
| return CHINESE; |
| } |
| if (memcmp(src, "pt", 2) == 0) { |
| if (memcmp(&src[len - 2], "BR", 2) == 0) {return PORTUGUESE;} |
| return PORTUGUESE; |
| } |
| if (memcmp(src, "fr", 2) == 0) { |
| if (memcmp(&src[len -2], "CA", 2) == 0) {return FRENCH;} |
| return FRENCH; |
| } |
| |
| // None of the special cases matched |
| if (src[2] == '-') { |
| char temp[4]; |
| memcpy(temp, src, 4); |
| temp[2] = '\0'; |
| LanguageFromCode(temp, &retlang); |
| } |
| if (src[3] == '-') { |
| char temp[4]; |
| memcpy(temp, src, 4); |
| temp[3] = '\0'; |
| LanguageFromCode(temp, &retlang); |
| } |
| if (retlang != UNKNOWN_LANGUAGE) { |
| return retlang; |
| } |
| |
| return retlang; |
| } |
| |
| typedef struct { |
| const char* name; |
| UnicodeLScript lscript; |
| } NameScriptPair; |
| |
| // In alphabetic order for binary search |
| static const NameScriptPair kNameScriptPair[] = { |
| // Unicode 5.1 additional scripts |
| {"Arab", ULScript_Arabic}, |
| {"Armn", ULScript_Armenian}, |
| {"Bali", ULScript_Balinese}, |
| {"Beng", ULScript_Bengali}, |
| {"Bugi", ULScript_Buginese}, |
| {"Buhd", ULScript_Buhid}, |
| {"Cans", ULScript_Canadian_Aboriginal}, |
| {"Cari", ULScript_Carian}, // Unicode 5.1 |
| {"Cham", ULScript_Cham}, // Unicode 5.1 |
| {"Cher", ULScript_Cherokee}, |
| {"Copt", ULScript_Coptic}, |
| {"Cprt", ULScript_Cypriot}, |
| {"Cyrl", ULScript_Cyrillic}, |
| {"Deva", ULScript_Devanagari}, |
| {"Dsrt", ULScript_Deseret}, |
| {"Ethi", ULScript_Ethiopic}, |
| {"Geor", ULScript_Georgian}, |
| {"Glag", ULScript_Glagolitic}, |
| {"Goth", ULScript_Gothic}, |
| {"Grek", ULScript_Greek}, |
| {"Gujr", ULScript_Gujarati}, |
| {"Guru", ULScript_Gurmukhi}, |
| {"Hani", ULScript_HanCJK}, |
| {"Hano", ULScript_Hanunoo}, |
| {"Hebr", ULScript_Hebrew}, |
| {"Ital", ULScript_Old_Italic}, |
| {"Kali", ULScript_Kayah_Li}, // Unicode 5.1 |
| {"Khar", ULScript_Kharoshthi}, |
| {"Khmr", ULScript_Khmer}, |
| {"Knda", ULScript_Kannada}, |
| {"Laoo", ULScript_Lao}, |
| {"Latn", ULScript_Latin}, |
| {"Lepc", ULScript_Lepcha}, // Unicode 5.1 |
| {"Limb", ULScript_Limbu}, |
| {"Linb", ULScript_Linear_B}, |
| {"Lyci", ULScript_Lycian}, // Unicode 5.1 |
| {"Lydi", ULScript_Lydian}, // Unicode 5.1 |
| {"Mlym", ULScript_Malayalam}, |
| {"Mong", ULScript_Mongolian}, |
| {"Mymr", ULScript_Myanmar}, |
| {"Nkoo", ULScript_Nko}, |
| {"Ogam", ULScript_Ogham}, |
| {"Olck", ULScript_Ol_Chiki}, // Unicode 5.1 |
| {"Orya", ULScript_Oriya}, |
| {"Osma", ULScript_Osmanya}, |
| {"Phag", ULScript_Phags_Pa}, |
| {"Phnx", ULScript_Phoenician}, |
| {"Rjng", ULScript_Rejang}, // Unicode 5.1 |
| {"Runr", ULScript_Runic}, |
| {"Saur", ULScript_Saurashtra}, // Unicode 5.1 |
| {"Shaw", ULScript_Shavian}, |
| {"Sinh", ULScript_Sinhala}, |
| {"Sund", ULScript_Sundanese}, // Unicode 5.1 |
| {"Sylo", ULScript_Syloti_Nagri}, |
| {"Syrc", ULScript_Syriac}, |
| {"Tagb", ULScript_Tagbanwa}, |
| {"Tale", ULScript_Tai_Le}, |
| {"Talu", ULScript_New_Tai_Lue}, |
| {"Taml", ULScript_Tamil}, |
| {"Telu", ULScript_Telugu}, |
| {"Tfng", ULScript_Tifinagh}, |
| {"Tglg", ULScript_Tagalog}, |
| {"Thaa", ULScript_Thaana}, |
| {"Thai", ULScript_Thai}, |
| {"Tibt", ULScript_Tibetan}, |
| {"Ugar", ULScript_Ugaritic}, |
| {"Vaii", ULScript_Vai}, // Unicode 5.1 // NOTE: apparently 'Vai ' |
| {"Xpeo", ULScript_Old_Persian}, |
| {"Xsux", ULScript_Cuneiform}, |
| {"Yiii", ULScript_Yi}, |
| {"Zyyy", ULScript_Common}, |
| {"Zzzz", ULScript_Inherited}, |
| }; |
| |
| // Convert "en-Latn-GB" to ULScript_Latin |
| UnicodeLScript GetLScriptFromNumberOrName(const char* src) { |
| if (strspn(src, "0123456789") == strlen(src)) { |
| // All digits |
| return static_cast<UnicodeLScript>(strto32(src, NULL, 10)); |
| } |
| |
| if (strcmp(src, "zh-TW") == 0) {return ULScript_HanCJK;} |
| if (strcmp(src, "zh-CN") == 0) {return ULScript_HanCJK;} |
| if (strcmp(src, "pt-BR") == 0) {return ULScript_Latin;} |
| if (strcmp(src, "pt-PT") == 0) {return ULScript_Latin;} |
| // Could be Latn or Limb; all our current training data is Latn |
| if (strcmp(src, "sit-NP") == 0) {return ULScript_Latin;} |
| |
| // Isolate just the script field |
| char temp[5]; |
| const char* src2 = strchr(src, '-'); |
| if (src2 == NULL) {return ULScript_Latin;} |
| src2 += 1; // over the - |
| memcpy(temp, src2, 4); |
| temp[4] = '\0'; |
| |
| int lo = 0; |
| int hi = ULScript_NUM_SCRIPTS; |
| while (lo < hi) { |
| int mid = (lo + hi) >> 1; |
| if (strcmp(temp, kNameScriptPair[mid].name) < 0) { |
| hi = mid; |
| } else if (strcmp(temp, kNameScriptPair[mid].name) > 0) { |
| lo = mid + 1; |
| } else { |
| return kNameScriptPair[mid].lscript; |
| } |
| } |
| return ULScript_Latin; |
| } |
| |
| |
| // Merge together some languages, such as bo/hr/sr |
| // Croatian Latin and Serbian Cyrillic now. |
| Language NormalizeLanguage(Language lang) { |
| if (lang == BOSNIAN) {return CROATIAN;} |
| if (lang == SERBO_CROATIAN) {return SERBIAN;} |
| |
| if (lang == PORTUGUESE_P) {return PORTUGUESE;} |
| if (lang == PORTUGUESE_B) {return PORTUGUESE;} |
| |
| return lang; |
| } |
| |
