| // Copyright 2004-2009 Google Inc. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| // ======================================================================== |
| // |
| // This file is for i18n. It contains two enums, namely Language and |
| // Encoding, where Language is the linguistic convention, and Encoding |
| // contains information on both language encoding and character set. |
| // |
| // The language and encoding are both based on Teragram's conventions, |
| // except for some common ISO-8859 encodings that are not detected by |
| // Teragram but might be in the future. |
| // |
| // This file also includes functions that do mappings among |
| // Language/Encoding enums, language/encoding string names (typically |
| // the output from Language Encoding identifier), and language codes |
| // (iso 639), and two-letter country codes (iso 3166) |
| // |
| // NOTE: Both Language and Encoding enums should always start from |
| // zero value. This assumption has been made and used. |
| |
| #ifndef OMAHA_BASE_LANG_ENC_H_ |
| #define OMAHA_BASE_LANG_ENC_H_ |
| |
| #include <windows.h> |
| |
| // some of the popular encoding aliases |
| #define LATIN1 ISO_8859_1 |
| #define LATIN2 ISO_8859_2 |
| #define LATIN3 ISO_8859_3 |
| #define LATIN4 ISO_8859_4 |
| #define CYRILLIC ISO_8859_5 |
| #define ARABIC_ENCODING ISO_8859_6 // avoiding the same name as language |
| #define GREEK_ENCODING ISO_8859_7 // avoiding the same name as language |
| #define HEBREW_ENCODING ISO_8859_8 // avoiding the same name as language |
| #define LATIN5 ISO_8859_9 |
| #define LATIN6 ISO_8859_10 |
| #define KOREAN_HANGUL KOREAN_EUC_KR |
| |
| // NOTE: Only add new languages to the end of this list (but before |
| // NUM_LANGUAGES). |
| enum Language { |
| ENGLISH = 0, /* 0 */ |
| DANISH, /* 1 */ |
| DUTCH, /* 2 */ |
| FINNISH, /* 3 */ |
| FRENCH, /* 4 */ |
| GERMAN, /* 5 */ |
| HEBREW, /* 6 */ |
| ITALIAN, /* 7 */ |
| JAPANESE, /* 8 */ |
| KOREAN, /* 9 */ |
| NORWEGIAN, /* 10 */ |
| POLISH, /* 11 */ |
| PORTUGUESE, /* 12 */ |
| RUSSIAN, /* 13 */ |
| SPANISH, /* 14 */ |
| SWEDISH, /* 15 */ |
| CHINESE, /* 16 */ |
| CZECH, /* 17 */ |
| GREEK, /* 18 */ |
| ICELANDIC, /* 19 */ |
| LATVIAN, /* 20 */ |
| LITHUANIAN, /* 21 */ |
| ROMANIAN, /* 22 */ |
| HUNGARIAN, /* 23 */ |
| ESTONIAN, /* 24 */ |
| TG_UNKNOWN_LANGUAGE, /* 25 */ |
| UNKNOWN_LANGUAGE, /* 26 */ |
| BULGARIAN, /* 27 */ |
| CROATIAN, /* 28 */ |
| SERBIAN, /* 29 */ |
| IRISH, /* 30 */ |
| GALICIAN, /* 31 */ |
| TAGALOG, /* 32 */ |
| TURKISH, /* 33 */ |
| UKRAINIAN, /* 34 */ |
| HINDI, /* 35 */ |
| MACEDONIAN, /* 36 */ |
| BENGALI, /* 37 */ |
| INDONESIAN, /* 38 */ |
| LATIN, /* 39 */ |
| MALAY, /* 40 */ |
| MALAYALAM, /* 41 */ |
| WELSH, /* 42 */ |
| NEPALI, /* 43 */ |
| TELUGU, /* 44 */ |
| ALBANIAN, /* 45 */ |
| TAMIL, /* 46 */ |
| BELARUSIAN, /* 47 */ |
| JAVANESE, /* 48 */ |
| OCCITAN, /* 49 */ |
| URDU, /* 50 */ |
| BIHARI, /* 51 */ |
| GUJARATI, /* 52 */ |
| THAI, /* 53 */ |
| ARABIC, /* 54 */ |
| CATALAN, /* 55 */ |
| ESPERANTO, /* 56 */ |
| BASQUE, /* 57 */ |
| INTERLINGUA, /* 58 */ |
| KANNADA, /* 59 */ |
| PUNJABI, /* 60 */ |
| SCOTS_GAELIC, /* 61 */ |
| SWAHILI, /* 62 */ |
| SLOVENIAN, /* 63 */ |
| MARATHI, /* 64 */ |
| MALTESE, /* 65 */ |
| VIETNAMESE, /* 66 */ |
| FRISIAN, /* 67 */ |
| SLOVAK, /* 68 */ |
| CHINESE_T, /* 69 */ // This is added to solve the problem of |
| // distinguishing Traditional and Simplified |
| // Chinese when the encoding is UTF8. |
| FAROESE, /* 70 */ |
| SUNDANESE, /* 71 */ |
| UZBEK, /* 72 */ |
| AMHARIC, /* 73 */ |
| AZERBAIJANI, /* 74 */ |
| GEORGIAN, /* 75 */ |
| TIGRINYA, /* 76 */ |
| PERSIAN, /* 77 */ |
| BOSNIAN, /* 78 */ |
| SINHALESE, /* 79 */ |
| NORWEGIAN_N, /* 80 */ |
| PORTUGUESE_P, /* 81 */ |
| PORTUGUESE_B, /* 82 */ |
| XHOSA, /* 83 */ |
| ZULU, /* 84 */ |
| GUARANI, /* 85 */ |
| SESOTHO, /* 86 */ |
| TURKMEN, /* 87 */ |
| KYRGYZ, /* 88 */ |
| BRETON, /* 89 */ |
| TWI, /* 90 */ |
| YIDDISH, /* 91 */ |
| ORIYA, /* 92 */ |
| SERBO_CROATIAN, /* 93 */ |
| SOMALI, /* 94 */ |
| UIGHUR, /* 95 */ |
| KURDISH, /* 96 */ |
| MONGOLIAN, /* 97 */ |
| ARMENIAN, /* 98 */ |
| LAOTHIAN, /* 99 */ |
| SINDHI, /* 100! */ |
| RHAETO_ROMANCE, /* 101 */ |
| CHINESE_JAPANESE_KOREAN, /* 103 */ // Not really a language |
| PSEUDOTRANSLATION, /* 104 */ // Not really a language |
| NUM_LANGUAGES, // Always keep this at the end. It is not a |
| // valid Language enum, it is only used to |
| // indicate the total number of Languages. |
| }; |
| |
| |
| // Language codes for those languages we support, used to map to IDs from |
| // the Language enumeration. We could have used the Rfc1766ToLcid from the |
| // Win32 system's mlang.dll to map these to LCIDs, but a) we don't want to |
| // have to load mlang.dll and b) we are using our own language IDs. |
| const TCHAR* const kLangCodeChinesePrc = _T("zh_cn"); |
| const TCHAR* const kLangCodeChineseTaiwan = _T("zh_tw"); |
| const TCHAR* const kLangCodeCjk = _T("cjk"); |
| const TCHAR* const kLangCodeDutch = _T("nl"); |
| const TCHAR* const kLangCodeEnglish = _T("en"); |
| const TCHAR* const kLangCodeFrench = _T("fr"); |
| const TCHAR* const kLangCodeGerman = _T("de"); |
| const TCHAR* const kLangCodeItalian = _T("it"); |
| const TCHAR* const kLangCodeJapanese = _T("ja"); |
| const TCHAR* const kLangCodeKorean = _T("ko"); |
| const TCHAR* const kLangCodePseudo = _T("x"); |
| const TCHAR* const kLangCodeSpanish = _T("es"); |
| |
| |
| // Maps language codes to languages. Terminated by a { NULL, UNKNOWN_LANGUAGE } |
| // item. |
| struct CodeToLanguage { |
| const TCHAR* code; |
| Language language; |
| }; |
| |
| SELECTANY CodeToLanguage codes_to_languages[] = { |
| { kLangCodeChinesePrc, CHINESE }, |
| { kLangCodeChineseTaiwan, CHINESE_T }, |
| { kLangCodeCjk, CHINESE_JAPANESE_KOREAN }, |
| { kLangCodeDutch, DUTCH }, |
| { kLangCodeEnglish, ENGLISH }, |
| { kLangCodeFrench, FRENCH }, |
| { kLangCodeGerman, GERMAN }, |
| { kLangCodeItalian, ITALIAN }, |
| { kLangCodeJapanese, JAPANESE }, |
| { kLangCodeKorean, KOREAN }, |
| { kLangCodePseudo, PSEUDOTRANSLATION }, |
| { kLangCodeSpanish, SPANISH }, |
| { NULL, UNKNOWN_LANGUAGE } |
| }; |
| |
| |
| |
| // Macro to wrap the notion of "unknown language". |
| #define IS_LANGUAGE_UNKNOWN(l) \ |
| ((l) == TG_UNKNOWN_LANGUAGE || (l) == UNKNOWN_LANGUAGE) |
| |
| // NOTE: Only add new encodings to the end of this list (but before |
| // NUM_ENCODINGS). |
| // NOTE: If you add an encoding here, you must also modify basistech_encoding() |
| // and google2/com/google/i18n/Encoding.java |
| enum Encoding { |
| ISO_8859_1 = 0, // 0: Teragram ASCII |
| ISO_8859_2, // 1: Teragram Latin2 |
| ISO_8859_3, // 2: in BasisTech but not in Teragram |
| ISO_8859_4, // 3: Teragram Latin4 |
| ISO_8859_5, // 4: Teragram ISO-8859-5 |
| ISO_8859_6, // 5: Teragram Arabic |
| ISO_8859_7, // 6: Teragram Greek |
| ISO_8859_8, // 7: Teragram Hebrew |
| ISO_8859_9, // 8: in BasisTech but not in Teragram |
| ISO_8859_10, // 9: in BasisTech but not in Teragram |
| JAPANESE_EUC_JP, // 10: Teragram EUC_JP |
| JAPANESE_SHIFT_JIS, // 11: Teragram SJS |
| JAPANESE_JIS, // 12: Teragram JIS |
| CHINESE_BIG5, // 13: Teragram BIG5 |
| CHINESE_GB, // 14: Teragram GB |
| CHINESE_EUC_CN, // 15: Teragram EUC-CN |
| KOREAN_EUC_KR, // 16: Teragram KSC |
| UNICODE_ENCODING, // 17: Teragram Unicode, changed to UNICODE_ENCODING |
| // from UNICODE, which is predefined by WINDOW |
| CHINESE_EUC_DEC, // 18: Teragram EUC |
| CHINESE_CNS, // 19: Teragram CNS |
| CHINESE_BIG5_CP950, // 20: Teragram BIG5_CP950 |
| JAPANESE_CP932, // 21: Teragram CP932 |
| UTF8, // 22 |
| UNKNOWN_ENCODING, // 23 |
| ASCII_7BIT, // 24: ISO_8859_1 with all characters <= 127. |
| // Should be present only in the crawler |
| // and in the repository, |
| // *never* as a result of Document::encoding(). |
| RUSSIAN_KOI8_R, // 25: Teragram KOI8R |
| RUSSIAN_CP1251, // 26: Teragram CP1251 |
| |
| //---------------------------------------------------------- |
| // These are _not_ output from teragram. Instead, they are as |
| // detected in the headers of usenet articles. |
| MSFT_CP1252, // 27: CP1252 aka MSFT euro ascii |
| RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, used for Ukrainian |
| MSFT_CP1250, // 29: CP1250 aka MSFT eastern european |
| ISO_8859_15, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized |
| //---------------------------------------------------------- |
| |
| //---------------------------------------------------------- |
| // These are in BasisTech but not in Teragram. They are |
| // needed for new interface languages. Now detected by |
| // research langid |
| MSFT_CP1254, // 31: used for Turkish |
| MSFT_CP1257, // 32: used in Baltic countries |
| //---------------------------------------------------------- |
| |
| //---------------------------------------------------------- |
| //---------------------------------------------------------- |
| // New encodings detected by Teragram |
| ISO_8859_11, // 33: aka TIS-620, used for Thai |
| MSFT_CP874, // 34: used for Thai |
| MSFT_CP1256, // 35: used for Arabic |
| |
| //---------------------------------------------------------- |
| // Detected as ISO_8859_8 by Teragram, but can be found in META tags |
| MSFT_CP1255, // 36: Logical Hebrew Microsoft |
| ISO_8859_8_I, // 37: Iso Hebrew Logical |
| HEBREW_VISUAL, // 38: Iso Hebrew Visual |
| //---------------------------------------------------------- |
| |
| //---------------------------------------------------------- |
| // Detected by research langid |
| CZECH_CP852, // 39 |
| CZECH_CSN_369103, // 40: aka ISO_IR_139 aka KOI8_CS |
| MSFT_CP1253, // 41: used for Greek |
| RUSSIAN_CP866, // 42 |
| //---------------------------------------------------------- |
| HZ_ENCODING, |
| ISO2022_CN, |
| ISO2022_KR, |
| |
| NUM_ENCODINGS // Always keep this at the end. It is not a |
| // valid Encoding enum, it is only used to |
| // indicate the total number of Encodings. |
| }; |
| |
| const int kNumLanguages = NUM_LANGUAGES; |
| const int kNumEncodings = NUM_ENCODINGS; |
| |
| #endif // OMAHA_BASE_LANG_ENC_H_ |
