| // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #ifndef ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_ |
| #define ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_ |
| |
| #include <string> |
| #include "encodings/compact_lang_det/ext_lang_enc.h" |
| #include "encodings/compact_lang_det/tote.h" |
| #include "encodings/compact_lang_det/win/cld_basictypes.h" |
| #include "encodings/compact_lang_det/win/cld_commandlineflags.h" |
| #include "encodings/compact_lang_det/win/cld_utf8statetable.h" |
| |
| namespace cld { |
| |
| // Hash bucket for four-way associative lookup with < 64K buckets |
| // 32 bytes per bucket, 8-byte entries |
| typedef struct { |
| uint32 key[4]; // hashed word to look up |
| uint32 value[4]; // packed three lang numbers and probability subscript |
| } SmallWordProbBucket4; |
| |
| // Hash bucket for fouro-way associative lookup with >= 64K buckets |
| // 24 bytes per bucket, 6-byte entries |
| typedef struct { |
| uint16 key[4]; // Half of hashed word to look up; other |
| // half is used to pick the bucket |
| uint32 value[4]; // packed three lang numbers and probability subscript |
| } LargeQuadProbBucket4; |
| |
| // Hash bucket for four-way associative lookup, indirect probabilities |
| // 16 bytes per bucket, 4-byte entries |
| typedef struct { |
| uint32 keyvalue[4]; // Upper part of word is hash, lower is indirect prob |
| } IndirectProbBucket4; |
| |
| |
| // This describes a complete CLD table, consisting of |
| // a main lookup table, an indirect language/probability table, and |
| // three constants. |
| // The main table key is a quadgram, bigram, or longword hash, with |
| // part of the key used to select a bucket modulo kCLDTableSize, |
| // and the rest matched against the key portion of four entries in a bucket, |
| // defined by kCLDTableKeyMask. The remaining bits of an entry, defined |
| // by ~kCLDTableKeyMask, are usually a subscript in the indirect table. |
| // |
| // By using part of the key to select a bucket, those key bits do not need |
| // to be stored in the main table entries, saving space (typically 2 bytes). |
| // |
| // By using an indirect table for lang/prob triples, only the subscript needs |
| // to be stored in the main table entires, saving space (typically 2 bytes). |
| // |
| // Each entry in the indirect table has three languages and three |
| // corresponding probabilities, packed into four bytes. |
| // |
| // The build date constant is included just for version tracking and is not |
| // otherwise used. |
| // |
| // Different-size tables can be linked in for different production |
| // environments. By going indirect through this struct, the runtime code is |
| // insensitive to the actual sizes. |
| // |
| // An empty placeholder table can be described by a table size of 1 |
| // bucket, a keymask of 0xffffffff, a degenerate bucket of four no-match |
| // entries, and a degenerate indirect table of one no-languages entry. |
| // |
| // |
| struct CLDTableSummary { |
| const IndirectProbBucket4* kCLDTable; |
| // Each bucket has four entries, part |
| // key and part indirect subscript |
| const uint32* kCLDTableInd; // Each entry is three packed lang/prob |
| const int kCLDTableSize; // Bucket count |
| const int kCLDTableIndSize; // Entries count |
| const int kCLDTableKeyMask; // Mask hash key |
| const int kCLDTableBuildDate; // yyyymmdd |
| }; |
| |
| |
| // Keeps per-character 0-12 language probabilities for CTJKVZ-- in that order. |
| // Chinese ChineseT Japanese Korean Vietnamese Zhuang |
| // (2 bytes unused, for alignment padding and future) |
| typedef struct { |
| uint8 probs[8]; |
| } UnigramProbArray; |
| |
| // Map 8-bit subscript to CTJKVZ probabilities |
| // Target runtime probabilities for CTJK + VZ |
| // Hand-generated to cover a reasonable range of choices |
| static const int kTargetCTJKVZProbsSize = 242; |
| static const UnigramProbArray kTargetCTJKVZProbs[kTargetCTJKVZProbsSize] = { |
| {{0,0,0,0,0,0,0,0}}, |
| {{0,0,0,0,0,12,0,0}}, |
| {{0,0,0,0,12,0,0,0}}, |
| {{0,0,0,12,0,0,0,0}}, |
| {{0,0,12,0,0,0,0,0}}, |
| {{0,12,0,0,0,0,0,0}}, |
| {{12,0,0,0,0,0,0,0}}, |
| |
| {{8,0,0,0,4,0,0,0}}, |
| {{8,0,0,4,0,0,0,0}}, |
| {{8,0,4,0,0,0,0,0}}, |
| {{8,4,0,0,0,0,0,0}}, |
| {{8,2,0,2,0,0,0,0}}, |
| {{0,0,0,0,0,8,0,0}}, |
| {{0,4,8,0,0,0,0,0}}, |
| {{4,0,0,0,0,8,0,0}}, |
| {{0,0,8,0,0,0,0,0}}, |
| {{8,2,2,0,0,0,0,0}}, |
| {{0,8,4,0,0,0,0,0}}, |
| {{8,0,0,0,0,4,0,0}}, |
| {{0,8,2,0,0,0,0,0}}, |
| {{4,8,0,0,0,0,0,0}}, |
| {{2,8,0,2,0,0,0,0}}, |
| {{2,2,8,0,0,0,0,0}}, |
| {{0,8,0,0,0,0,0,0}}, |
| {{0,2,8,0,0,0,0,0}}, |
| {{2,8,2,0,0,0,0,0}}, |
| {{8,0,0,0,0,0,0,0}}, |
| {{2,8,0,0,0,0,0,0}}, |
| {{8,2,0,0,0,0,0,0}}, |
| |
| {{0,6,2,0,2,0,0,0}}, |
| {{2,0,0,0,6,0,0,0}}, |
| {{4,0,0,0,6,0,0,0}}, |
| {{4,6,0,0,4,0,0,0}}, |
| {{4,6,2,0,2,0,0,0}}, |
| {{4,6,4,0,2,0,0,0}}, |
| {{5,4,6,0,0,0,0,0}}, |
| {{6,0,0,0,4,0,0,0}}, |
| {{6,0,2,0,4,0,0,0}}, |
| {{6,0,4,0,4,0,0,0}}, |
| {{6,2,0,0,4,0,0,0}}, |
| {{6,2,2,0,4,0,0,0}}, |
| {{6,2,4,0,2,0,0,0}}, |
| {{6,4,0,0,2,0,0,0}}, |
| {{6,4,2,0,2,0,0,0}}, |
| {{0,0,6,2,0,0,0,0}}, |
| {{0,6,2,0,0,2,0,0}}, |
| {{2,2,2,0,0,6,0,0}}, |
| {{2,2,6,4,0,0,0,0}}, |
| {{2,4,0,0,0,6,0,0}}, |
| {{2,6,0,4,0,0,0,0}}, |
| {{2,6,2,4,0,0,0,0}}, |
| {{2,6,4,4,0,0,0,0}}, |
| {{4,0,2,0,0,6,0,0}}, |
| {{4,2,6,2,0,0,0,0}}, |
| {{4,4,2,0,0,6,0,0}}, |
| {{4,6,4,0,0,2,0,0}}, |
| {{6,0,2,0,0,2,0,0}}, |
| {{6,2,0,0,0,2,0,0}}, |
| {{6,2,2,0,0,4,0,0}}, |
| {{6,2,4,0,0,2,0,0}}, |
| {{4,6,2,0,0,4,0,0}}, |
| {{6,4,2,0,0,4,0,0}}, |
| {{2,0,0,0,0,6,0,0}}, |
| {{6,2,0,2,0,0,0,0}}, |
| {{2,2,0,0,0,6,0,0}}, |
| {{6,2,6,0,0,0,0,0}}, |
| {{6,4,2,0,0,2,0,0}}, |
| {{6,4,2,2,0,0,0,0}}, |
| {{4,6,4,2,0,0,0,0}}, |
| {{6,0,2,0,0,4,0,0}}, |
| {{6,0,4,0,0,2,0,0}}, |
| {{6,0,6,0,0,0,0,0}}, |
| {{6,2,2,0,0,0,0,0}}, |
| {{6,4,0,0,0,2,0,0}}, |
| {{6,4,5,0,0,0,0,0}}, |
| {{0,6,0,2,0,0,0,0}}, |
| {{0,6,2,2,0,0,0,0}}, |
| {{2,6,0,2,0,0,0,0}}, |
| {{2,6,2,2,0,0,0,0}}, |
| {{4,2,0,0,0,6,0,0}}, |
| {{6,4,0,0,0,4,0,0}}, |
| {{6,4,0,2,0,0,0,0}}, |
| {{6,6,0,2,0,0,0,0}}, |
| {{6,0,4,0,0,4,0,0}}, |
| {{6,2,0,0,0,4,0,0}}, |
| {{6,6,2,2,0,0,0,0}}, |
| {{4,6,0,0,0,2,0,0}}, |
| {{2,6,6,0,0,0,0,0}}, |
| {{4,5,6,0,0,0,0,0}}, |
| {{4,6,0,2,0,0,0,0}}, |
| {{6,2,0,0,0,6,0,0}}, |
| {{0,6,4,2,0,0,0,0}}, |
| {{4,0,6,0,0,0,0,0}}, |
| {{2,6,4,2,0,0,0,0}}, |
| {{4,6,0,0,0,4,0,0}}, |
| {{6,2,2,0,0,0,0,0}}, |
| {{4,6,2,2,0,0,0,0}}, |
| {{4,6,5,0,0,0,0,0}}, |
| {{6,0,2,0,0,0,0,0}}, |
| {{6,4,4,0,0,0,0,0}}, |
| {{4,2,6,0,0,0,0,0}}, |
| {{2,0,6,0,0,0,0,0}}, |
| {{4,4,0,0,0,6,0,0}}, |
| {{4,4,6,0,0,0,0,0}}, |
| {{4,6,2,0,0,2,0,0}}, |
| {{2,2,6,0,0,0,0,0}}, |
| {{2,4,6,0,0,0,0,0}}, |
| {{0,6,6,0,0,0,0,0}}, |
| {{6,2,4,0,0,0,0,0}}, |
| {{0,4,6,0,0,0,0,0}}, |
| {{4,0,0,0,0,6,0,0}}, |
| {{4,6,4,0,0,0,0,0}}, |
| {{6,0,0,0,0,6,0,0}}, |
| {{6,0,0,0,0,2,0,0}}, |
| {{6,0,4,0,0,0,0,0}}, |
| {{6,5,4,0,0,0,0,0}}, |
| {{0,2,6,0,0,0,0,0}}, |
| {{0,0,6,0,0,0,0,0}}, |
| {{6,6,2,0,0,0,0,0}}, |
| {{2,6,4,0,0,0,0,0}}, |
| {{6,4,2,0,0,0,0,0}}, |
| {{2,6,2,0,0,0,0,0}}, |
| {{2,6,0,0,0,0,0,0}}, |
| {{6,0,0,0,0,4,0,0}}, |
| {{6,4,0,0,0,0,0,0}}, |
| {{6,6,0,0,0,0,0,0}}, |
| {{5,6,4,0,0,0,0,0}}, |
| {{0,6,0,0,0,0,0,0}}, |
| {{6,2,0,0,0,0,0,0}}, |
| {{0,6,2,0,0,0,0,0}}, |
| {{4,6,2,0,0,0,0,0}}, |
| {{0,6,4,0,0,0,0,0}}, |
| {{4,6,0,0,0,0,0,0}}, |
| {{6,0,0,0,0,0,0,0}}, |
| {{6,6,5,0,0,0,0,0}}, |
| {{6,5,6,0,0,0,0,0}}, |
| {{5,6,6,0,0,0,0,0}}, |
| {{5,5,6,0,0,0,0,0}}, |
| {{5,6,5,0,0,0,0,0}}, |
| {{6,5,5,0,0,0,0,0}}, |
| {{6,6,6,0,0,0,0,0}}, |
| {{6,5,0,0,0,0,0,0}}, |
| {{6,0,5,0,0,0,0,0}}, |
| {{0,6,5,0,0,0,0,0}}, |
| {{5,6,0,0,0,0,0,0}}, |
| {{5,0,6,0,0,0,0,0}}, |
| {{0,5,6,0,0,0,0,0}}, |
| |
| {{0,0,0,0,4,0,0,0}}, |
| {{0,0,0,4,0,0,0,0}}, |
| {{2,2,0,0,4,0,0,0}}, |
| {{2,2,2,0,4,0,0,0}}, |
| {{2,4,0,0,2,0,0,0}}, |
| {{2,4,2,0,2,0,0,0}}, |
| {{2,4,4,0,2,0,0,0}}, |
| {{4,0,2,0,4,0,0,0}}, |
| {{4,0,4,0,2,0,0,0}}, |
| {{4,2,0,0,2,0,0,0}}, |
| {{4,2,2,0,2,0,0,0}}, |
| {{4,4,0,0,2,0,0,0}}, |
| {{4,4,2,0,2,0,0,0}}, |
| {{4,4,4,0,2,0,0,0}}, |
| {{0,2,2,4,0,0,0,0}}, |
| {{2,2,4,2,0,0,0,0}}, |
| {{2,4,4,0,0,2,0,0}}, |
| {{2,4,4,2,0,0,0,0}}, |
| {{4,0,4,0,0,2,0,0}}, |
| {{4,0,4,0,0,4,0,0}}, |
| {{4,2,2,4,0,0,0,0}}, |
| {{4,4,0,2,0,0,0,0}}, |
| {{2,2,0,4,0,0,0,0}}, |
| {{2,4,2,2,0,0,0,0}}, |
| {{4,4,2,2,0,0,0,0}}, |
| {{4,0,4,0,0,0,0,0}}, |
| {{4,4,4,0,0,4,0,0}}, |
| {{0,4,0,2,0,0,0,0}}, |
| {{0,4,2,2,0,0,0,0}}, |
| {{4,0,2,0,0,2,0,0}}, |
| {{4,2,0,0,0,4,0,0}}, |
| {{2,2,2,0,0,4,0,0}}, |
| {{4,0,0,2,0,0,0,0}}, |
| {{4,4,4,0,0,2,0,0}}, |
| {{4,0,0,0,0,4,0,0}}, |
| {{4,0,2,0,0,4,0,0}}, |
| {{4,2,0,0,0,2,0,0}}, |
| {{4,2,2,0,0,2,0,0}}, |
| {{2,4,0,2,0,0,0,0}}, |
| {{2,2,0,0,0,4,0,0}}, |
| {{2,4,0,0,0,4,0,0}}, |
| {{2,4,2,0,0,4,0,0}}, |
| {{4,2,4,0,0,0,0,0}}, |
| {{2,0,4,0,0,0,0,0}}, |
| {{4,0,2,0,0,0,0,0}}, |
| {{4,4,0,0,0,4,0,0}}, |
| {{4,4,2,0,0,4,0,0}}, |
| {{0,4,4,0,0,0,0,0}}, |
| {{4,4,0,0,0,2,0,0}}, |
| {{2,4,0,0,0,2,0,0}}, |
| {{2,2,4,0,0,0,0,0}}, |
| {{0,2,4,0,0,0,0,0}}, |
| {{4,2,2,0,0,0,0,0}}, |
| {{2,4,2,0,0,2,0,0}}, |
| {{4,4,4,0,0,0,0,0}}, |
| {{2,4,4,0,0,0,0,0}}, |
| {{0,0,4,0,0,0,0,0}}, |
| {{0,4,2,0,0,0,0,0}}, |
| {{4,4,2,0,0,2,0,0}}, |
| {{2,4,2,0,0,0,0,0}}, |
| {{4,2,0,0,0,0,0,0}}, |
| {{4,4,0,0,0,0,0,0}}, |
| {{4,4,2,0,0,0,0,0}}, |
| {{2,4,0,0,0,0,0,0}}, |
| {{0,4,0,0,0,0,0,0}}, |
| {{4,0,0,0,0,0,0,0}}, |
| {{0,0,0,4,4,0,0,0}}, |
| {{0,0,4,0,4,0,0,0}}, |
| {{0,0,4,4,0,0,0,0}}, |
| {{0,4,0,0,4,0,0,0}}, |
| {{0,4,0,4,0,0,0,0}}, |
| {{4,0,0,0,4,0,0,0}}, |
| {{4,0,0,4,0,0,0,0}}, |
| |
| {{2,0,0,0,0,0,0,0}}, |
| {{0,2,0,0,0,0,0,0}}, |
| {{0,2,0,2,2,0,0,0}}, |
| {{0,2,2,0,2,0,0,0}}, |
| {{2,0,0,2,2,0,0,0}}, |
| {{2,0,2,0,2,0,0,0}}, |
| {{2,0,2,2,0,0,0,0}}, |
| {{2,2,0,0,2,0,0,0}}, |
| {{2,2,2,2,0,0,0,0}}, |
| {{2,2,0,2,0,0,0,0}}, |
| {{2,2,0,0,0,0,0,0}}, |
| {{0,0,2,0,0,0,0,0}}, |
| {{0,2,2,0,0,0,0,0}}, |
| {{2,2,2,0,0,0,0,0}}, |
| {{0,0,0,2,0,0,0,0}}, |
| {{2,0,2,0,0,0,0,0}}, |
| {{0,2,0,2,0,0,0,0}}, |
| {{0,0,2,2,0,0,0,0}}, |
| {{0,2,2,2,0,0,0,0}}, |
| }; |
| |
| |
| |
| |
| // 1 to skip ASCII space, vowels AEIOU aeiou and UTF-8 continuation bytes 80-BF |
| static const uint8 kSkipSpaceVowelContinue[256] = { |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0, |
| 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| }; |
| |
| // 1 to skip ASCII space, and UTF-8 continuation bytes 80-BF |
| static const uint8 kSkipSpaceContinue[256] = { |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| }; |
| |
| |
| // If != UNKNOWN, use nilgrams to determine language of this script |
| static const Language kOnlyLanguagePerLScript[] = { |
| ENGLISH, // ULScript_Common, [no words should be in this script] |
| UNKNOWN_LANGUAGE, // ULScript_Latin, |
| //UNKNOWN_LANGUAGE, // ULScript_Greek, Jan 2009: change so we can score quads |
| GREEK, // ULScript_Greek, Mar 2009: change back; do gibberish separately |
| UNKNOWN_LANGUAGE, // ULScript_Cyrillic, |
| ARMENIAN, // ULScript_Armenian, |
| UNKNOWN_LANGUAGE, // ULScript_Hebrew, |
| UNKNOWN_LANGUAGE, // ULScript_Arabic, |
| SYRIAC, // ULScript_Syriac, |
| DHIVEHI, // ULScript_Thaana, |
| UNKNOWN_LANGUAGE, // ULScript_Devanagari, |
| UNKNOWN_LANGUAGE, // ULScript_Bengali, |
| PUNJABI, // ULScript_Gurmukhi, |
| GUJARATI, // ULScript_Gujarati, |
| ORIYA, // ULScript_Oriya, |
| TAMIL, // ULScript_Tamil, |
| TELUGU, // ULScript_Telugu, |
| KANNADA, // ULScript_Kannada, |
| MALAYALAM, // ULScript_Malayalam, |
| SINHALESE, // ULScript_Sinhala, |
| THAI, // ULScript_Thai, |
| LAOTHIAN, // ULScript_Lao, |
| UNKNOWN_LANGUAGE, // ULScript_Tibetan, |
| BURMESE, // ULScript_Myanmar, |
| GEORGIAN, // ULScript_Georgian, |
| UNKNOWN_LANGUAGE, // ULScript_HanCJK, |
| UNKNOWN_LANGUAGE, // ULScript_Ethiopic, |
| CHEROKEE, // ULScript_Cherokee, |
| INUKTITUT, // ULScript_Canadian_Aboriginal, |
| X_OGHAM, // ULScript_Ogham, |
| X_RUNIC, // ULScript_Runic, |
| KHMER, // ULScript_Khmer, |
| MONGOLIAN, // ULScript_Mongolian, |
| X_YI, // ULScript_Yi, |
| X_OLD_ITALIC, // ULScript_Old_Italic, |
| X_GOTHIC, // ULScript_Gothic, |
| X_DESERET, // ULScript_Deseret, |
| ENGLISH, // ULScript_Inherited, [no words should be in this script] |
| TAGALOG, // ULScript_Tagalog, |
| X_HANUNOO, // ULScript_Hanunoo, |
| X_BUHID, // ULScript_Buhid, |
| X_TAGBANWA, // ULScript_Tagbanwa, |
| LIMBU, // ULScript_Limbu, |
| X_TAI_LE, // ULScript_Tai_Le, |
| X_LINEAR_B, // ULScript_Linear_B, |
| X_UGARITIC, // ULScript_Ugaritic, |
| X_SHAVIAN, // ULScript_Shavian, |
| X_OSMANYA, // ULScript_Osmanya, |
| X_CYPRIOT, // ULScript_Cypriot, |
| X_BUGINESE, // ULScript_Buginese, |
| X_COPTIC, // ULScript_Coptic, |
| X_NEW_TAI_LUE, // ULScript_New_Tai_Lue, |
| X_GLAGOLITIC, // ULScript_Glagolitic, |
| X_TIFINAGH, // ULScript_Tifinagh, |
| X_SYLOTI_NAGRI, // ULScript_Syloti_Nagri, |
| X_OLD_PERSIAN, // ULScript_Old_Persian, |
| X_KHAROSHTHI, // ULScript_Kharoshthi, |
| X_BALINESE, // ULScript_Balinese, |
| X_CUNEIFORM, // ULScript_Cuneiform, |
| X_PHOENICIAN, // ULScript_Phoenician, |
| X_PHAGS_PA, // ULScript_Phags_Pa, |
| X_NKO, // ULScript_Nko, |
| |
| // Unicode 5.1 |
| X_SUDANESE, // ULScript_Sundanese, |
| X_LEPCHA, // ULScript_Lepcha, |
| X_OL_CHIKI, // ULScript_Ol_Chiki, |
| X_VAI, // ULScript_Vai, |
| X_SAURASHTRA, // ULScript_Saurashtra, |
| X_KAYAH_LI, // ULScript_Kayah_Li, |
| X_REJANG, // ULScript_Rejang, |
| X_LYCIAN, // ULScript_Lycian, |
| X_CARIAN, // ULScript_Carian, |
| X_LYDIAN, // ULScript_Lydian, |
| X_CHAM, // ULScript_Cham, |
| }; |
| |
| COMPILE_ASSERT(arraysize(kOnlyLanguagePerLScript) == ULScript_NUM_SCRIPTS, |
| kOnlyLanguagePerLScript_has_incorrect_length); |
| |
| |
| // This is, in a sense, the complement of the table above |
| // If != UNKNOWN, determines a default language of this script |
| static const Language kDefaultLanguagePerLScript[] = { |
| UNKNOWN_LANGUAGE, // ULScript_Common, [no words should be in this script] |
| ENGLISH, // ULScript_Latin, |
| UNKNOWN_LANGUAGE, // ULScript_Greek, |
| RUSSIAN, // ULScript_Cyrillic, |
| UNKNOWN_LANGUAGE, // ULScript_Armenian, |
| HEBREW, // ULScript_Hebrew, |
| ARABIC, // ULScript_Arabic, |
| UNKNOWN_LANGUAGE, // ULScript_Syriac, |
| UNKNOWN_LANGUAGE, // ULScript_Thaana, |
| HINDI, // ULScript_Devanagari, |
| BENGALI, // ULScript_Bengali, |
| UNKNOWN_LANGUAGE, // ULScript_Gurmukhi, |
| UNKNOWN_LANGUAGE, // ULScript_Gujarati, |
| UNKNOWN_LANGUAGE, // ULScript_Oriya, |
| UNKNOWN_LANGUAGE, // ULScript_Tamil, |
| UNKNOWN_LANGUAGE, // ULScript_Telugu, |
| UNKNOWN_LANGUAGE, // ULScript_Kannada, |
| UNKNOWN_LANGUAGE, // ULScript_Malayalam, |
| UNKNOWN_LANGUAGE, // ULScript_Sinhala, |
| UNKNOWN_LANGUAGE, // ULScript_Thai, |
| UNKNOWN_LANGUAGE, // ULScript_Lao, |
| TIBETAN, // ULScript_Tibetan, |
| UNKNOWN_LANGUAGE, // ULScript_Myanmar, |
| UNKNOWN_LANGUAGE, // ULScript_Georgian, |
| CHINESE, // ULScript_HanCJK, |
| AMHARIC, // ULScript_Ethiopic, |
| UNKNOWN_LANGUAGE, // ULScript_Cherokee, |
| UNKNOWN_LANGUAGE, // ULScript_Canadian_Aboriginal, |
| UNKNOWN_LANGUAGE, // ULScript_Ogham, |
| UNKNOWN_LANGUAGE, // ULScript_Runic, |
| UNKNOWN_LANGUAGE, // ULScript_Khmer, |
| UNKNOWN_LANGUAGE, // ULScript_Mongolian, |
| UNKNOWN_LANGUAGE, // ULScript_Yi, |
| UNKNOWN_LANGUAGE, // ULScript_Old_Italic, |
| UNKNOWN_LANGUAGE, // ULScript_Gothic, |
| UNKNOWN_LANGUAGE, // ULScript_Deseret, |
| UNKNOWN_LANGUAGE, // ULScript_Inherited, [no words should be in this script] |
| UNKNOWN_LANGUAGE, // ULScript_Tagalog, |
| UNKNOWN_LANGUAGE, // ULScript_Hanunoo, |
| UNKNOWN_LANGUAGE, // ULScript_Buhid, |
| UNKNOWN_LANGUAGE, // ULScript_Tagbanwa, |
| UNKNOWN_LANGUAGE, // ULScript_Limbu, |
| UNKNOWN_LANGUAGE, // ULScript_Tai_Le, |
| UNKNOWN_LANGUAGE, // ULScript_Linear_B, |
| UNKNOWN_LANGUAGE, // ULScript_Ugaritic, |
| UNKNOWN_LANGUAGE, // ULScript_Shavian, |
| UNKNOWN_LANGUAGE, // ULScript_Osmanya, |
| UNKNOWN_LANGUAGE, // ULScript_Cypriot, |
| UNKNOWN_LANGUAGE, // ULScript_Buginese, |
| UNKNOWN_LANGUAGE, // ULScript_Coptic, |
| UNKNOWN_LANGUAGE, // ULScript_New_Tai_Lue, |
| UNKNOWN_LANGUAGE, // ULScript_Glagolitic, |
| UNKNOWN_LANGUAGE, // ULScript_Tifinagh, |
| UNKNOWN_LANGUAGE, // ULScript_Syloti_Nagri, |
| UNKNOWN_LANGUAGE, // ULScript_Old_Persian, |
| UNKNOWN_LANGUAGE, // ULScript_Kharoshthi, |
| UNKNOWN_LANGUAGE, // ULScript_Balinese, |
| UNKNOWN_LANGUAGE, // ULScript_Cuneiform, |
| UNKNOWN_LANGUAGE, // ULScript_Phoenician, |
| UNKNOWN_LANGUAGE, // ULScript_Phags_Pa, |
| UNKNOWN_LANGUAGE, // ULScript_Nko, |
| |
| // Unicode 5.1 |
| UNKNOWN_LANGUAGE, // ULScript_Sundanese, |
| UNKNOWN_LANGUAGE, // ULScript_Lepcha, |
| UNKNOWN_LANGUAGE, // ULScript_Ol_Chiki, |
| UNKNOWN_LANGUAGE, // ULScript_Vai, |
| UNKNOWN_LANGUAGE, // ULScript_Saurashtra, |
| UNKNOWN_LANGUAGE, // ULScript_Kayah_Li, |
| UNKNOWN_LANGUAGE, // ULScript_Rejang, |
| UNKNOWN_LANGUAGE, // ULScript_Lycian, |
| UNKNOWN_LANGUAGE, // ULScript_Carian, |
| UNKNOWN_LANGUAGE, // ULScript_Lydian, |
| UNKNOWN_LANGUAGE, // ULScript_Cham, |
| }; |
| |
| COMPILE_ASSERT(arraysize(kDefaultLanguagePerLScript) == ULScript_NUM_SCRIPTS, |
| kDefaultLanguagePerLScript_has_incorrect_length); |
| |
| |
| // True for standalone languages (only lang in a script) |
| // Subscripted by packed language number |
| // If 1, we will use nilgrams to determine language |
| static const uint8 kIsStandaloneLang[EXT_NUM_LANGUAGES + 1] = { |
| 0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,0, // GREEK |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| 0,1,0,0,1, 0,1,0,0,0, 0,0,1,1,0, 0,0,0,0,1, // MALAYALAM..KANNADA |
| 1,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,1, // PUNJABI..SINHALESE |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,1,1,0, // ARMENIAN..LAOTHIAN |
| |
| 0,0,0,0,1, 0,1,1,1,0, 1,0,0,0,0, 0,0,0,0,0, // KHMER..ORIYA |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| 0,1,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // INUKTITUT |
| |
| 0,0,0,0,0, // [160..164] |
| // Add new language standalone bit just before here |
| 0,0,0,0,0, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, |
| 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, |
| |
| 1,1,1,1, |
| }; |
| |
| // True for ULScript_HanCJK |
| // (Vietnamese and Zhuang also have Latin script quadgrams) |
| // Subscripted by packed language number |
| static const uint8 kIsUnigramLang[EXT_NUM_LANGUAGES + 1] = { |
| 0, |
| 0,0,0,0,0, 0,0,0,1,1, 0,0,0,0,0, 0,1,0,0,0, // JAPANESE KOREAN CHINESE |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| 0,0,0,0,0, 0,1,0,0,1, 0,0,0,0,0, 0,0,0,0,0, // VIETNAMESE CHINESE_T |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,0, // ZHUANG |
| |
| 0,0,0,0,0, // [160..164] |
| // Add new language unigram bit just before here |
| |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // |
| |
| 0,0,0,0, |
| }; |
| |
| |
| // True for ULScript_HanCJK |
| // Subscripted by lscript number |
| static const uint8 kScoreUniPerLScript[] = { |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, |
| }; |
| |
| COMPILE_ASSERT(arraysize(kScoreUniPerLScript) == ULScript_NUM_SCRIPTS, |
| kScoreUniPerLScript_has_incorrect_length); |
| |
| |
| // Defines Top40 packed languages |
| |
| // Tier 0/1 Language enum list (16) |
| // ENGLISH, /*no en_GB,*/ FRENCH, ITALIAN, GERMAN, SPANISH, // E - FIGS |
| // DUTCH, CHINESE, CHINESE_T, JAPANESE, KOREAN, |
| // PORTUGUESE, RUSSIAN, POLISH, TURKISH, THAI, |
| // ARABIC, |
| // |
| // Tier 2 Language enum list (22) |
| // SWEDISH, FINNISH, DANISH, /*no pt-PT,*/ ROMANIAN, HUNGARIAN, |
| // HEBREW, INDONESIAN, CZECH, GREEK, NORWEGIAN, |
| // VIETNAMESE, BULGARIAN, CROATIAN, LITHUANIAN, SLOVAK, |
| // TAGALOG, SLOVENIAN, SERBIAN, CATALAN, LATVIAN, |
| // UKRAINIAN, HINDI, |
| // |
| // use SERBO_CROATIAN instead of BOSNIAN, SERBIAN, CROATIAN, MONTENEGRIN(21) |
| // |
| // Include IgnoreMe (TG_UNKNOWN_LANGUAGE, 25+1) as a top 40 |
| |
| // NOTE: packed, i.e. Language enum + 1 |
| static const uint8 kIsPackedTop40[EXT_NUM_LANGUAGES + 1] = { |
| 0, |
| 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,0, |
| 1,1,1,1,0, 1,0,1,0,0, 0,0,1,1,1, 1,0,0,1,0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,1, 1,0,0,0,0, |
| 0,0,0,1,0, 0,1,0,1,1, 0,0,0,0,0, 0,0,0,0,0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,1,0,0, 0,0,0,0,0, |
| |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| |
| 0,0,0,0,0, // [160..164] |
| // Add new language top40 bit just before here |
| |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, |
| |
| 0,0,0,0, |
| }; |
| |
| |
| |
| // Table has 234 eight-byte entries. Each entry has a five-byte array and |
| // a three-byte array of log base 2 probabilities in the range 0..11. |
| // The intended use is to express five or three probabilities in a single-byte |
| // subscript, then decode via this table. These probabilities are |
| // intended to go with an array of five or three language numbers. |
| // |
| // The corresponding language numbers will have to be sorted by descending |
| // probability, then the actual probability subscript chosen to match the |
| // closest available entry in this table. |
| // |
| // Pattern of probability values: |
| // hi 3/4 1/2 1/4 lo hi mid lo |
| // where "3/4" is (hi*3+lo)/4, "1/2" is (hi+lo)/2, and "1/4" is (hi+lo*3)/4 and |
| // mid is one of 3/4 1/2 or 1/4. |
| // There are three groups of 78 (=12*13/2) entries, with hi running 0..11 and |
| // lo running 0..hi. Only the first group is used for five-entry lookups. |
| // The mid value in the first group is 1/2, the second group 3/4, and the |
| // third group 1/4. For three-entry lookups, this allows the mid entry to be |
| // somewhat higher or lower than the midpoint, to allow a better match to the |
| // original probabilities. |
| static const int kLgProbV2TblSize = 234; |
| static const uint8 kLgProbV2Tbl[kLgProbV2TblSize * 8] = { |
| 1,1,1,1,1, 1,1,1, // [0] |
| 2,2,2,1,1, 2,2,1, // [1] |
| 2,2,2,2,2, 2,2,2, |
| 3,3,2,2,1, 3,2,1, // [3] |
| 3,3,3,2,2, 3,3,2, |
| 3,3,3,3,3, 3,3,3, |
| 4,3,3,2,1, 4,3,1, // [6] |
| 4,4,3,3,2, 4,3,2, |
| 4,4,4,3,3, 4,4,3, |
| 4,4,4,4,4, 4,4,4, |
| 5,4,3,2,1, 5,3,1, // [10] |
| 5,4,4,3,2, 5,4,2, |
| 5,5,4,4,3, 5,4,3, |
| 5,5,5,4,4, 5,5,4, |
| 5,5,5,5,5, 5,5,5, |
| 6,5,4,2,1, 6,4,1, // [15] |
| 6,5,4,3,2, 6,4,2, |
| 6,5,5,4,3, 6,5,3, |
| 6,6,5,5,4, 6,5,4, |
| 6,6,6,5,5, 6,6,5, |
| 6,6,6,6,6, 6,6,6, |
| 7,6,4,3,1, 7,4,1, // [21] |
| 7,6,5,3,2, 7,5,2, |
| 7,6,5,4,3, 7,5,3, |
| 7,6,6,5,4, 7,6,4, |
| 7,7,6,6,5, 7,6,5, |
| 7,7,7,6,6, 7,7,6, |
| 7,7,7,7,7, 7,7,7, |
| 8,6,5,3,1, 8,5,1, // [28] |
| 8,7,5,4,2, 8,5,2, |
| 8,7,6,4,3, 8,6,3, |
| 8,7,6,5,4, 8,6,4, |
| 8,7,7,6,5, 8,7,5, |
| 8,8,7,7,6, 8,7,6, |
| 8,8,8,7,7, 8,8,7, |
| 8,8,8,8,8, 8,8,8, |
| 9,7,5,3,1, 9,5,1, // [36] |
| 9,7,6,4,2, 9,6,2, |
| 9,8,6,5,3, 9,6,3, |
| 9,8,7,5,4, 9,7,4, |
| 9,8,7,6,5, 9,7,5, |
| 9,8,8,7,6, 9,8,6, |
| 9,9,8,8,7, 9,8,7, |
| 9,9,9,8,8, 9,9,8, |
| 9,9,9,9,9, 9,9,9, |
| 10,8,6,3,1, 10,6,1, // [45] |
| 10,8,6,4,2, 10,6,2, |
| 10,8,7,5,3, 10,7,3, |
| 10,9,7,6,4, 10,7,4, |
| 10,9,8,6,5, 10,8,5, |
| 10,9,8,7,6, 10,8,6, |
| 10,9,9,8,7, 10,9,7, |
| 10,10,9,9,8, 10,9,8, |
| 10,10,10,9,9, 10,10,9, |
| 10,10,10,10,10, 10,10,10, |
| 11,9,6,4,1, 11,6,1, // [55] |
| 11,9,7,4,2, 11,7,2, |
| 11,9,7,5,3, 11,7,3, |
| 11,9,8,6,4, 11,8,4, |
| 11,10,8,7,5, 11,8,5, |
| 11,10,9,7,6, 11,9,6, |
| 11,10,9,8,7, 11,9,7, |
| 11,10,10,9,8, 11,10,8, |
| 11,11,10,10,9, 11,10,9, |
| 11,11,11,10,10, 11,11,10, |
| 11,11,11,11,11, 11,11,11, |
| 12,9,7,4,1, 12,7,1, // [66] |
| 12,10,7,5,2, 12,7,2, |
| 12,10,8,5,3, 12,8,3, |
| 12,10,8,6,4, 12,8,4, |
| 12,10,9,7,5, 12,9,5, |
| 12,11,9,8,6, 12,9,6, |
| 12,11,10,8,7, 12,10,7, |
| 12,11,10,9,8, 12,10,8, |
| 12,11,11,10,9, 12,11,9, |
| 12,12,11,11,10, 12,11,10, |
| 12,12,12,11,11, 12,12,11, |
| 12,12,12,12,12, 12,12,12, |
| |
| 1,1,1,1,1, 1,1,1, |
| 2,2,2,1,1, 2,2,1, |
| 2,2,2,2,2, 2,2,2, |
| 3,3,2,2,1, 3,3,1, |
| 3,3,3,2,2, 3,3,2, |
| 3,3,3,3,3, 3,3,3, |
| 4,3,3,2,1, 4,3,1, |
| 4,4,3,3,2, 4,4,2, |
| 4,4,4,3,3, 4,4,3, |
| 4,4,4,4,4, 4,4,4, |
| 5,4,3,2,1, 5,4,1, |
| 5,4,4,3,2, 5,4,2, |
| 5,5,4,4,3, 5,5,3, |
| 5,5,5,4,4, 5,5,4, |
| 5,5,5,5,5, 5,5,5, |
| 6,5,4,2,1, 6,5,1, |
| 6,5,4,3,2, 6,5,2, |
| 6,5,5,4,3, 6,5,3, |
| 6,6,5,5,4, 6,6,4, |
| 6,6,6,5,5, 6,6,5, |
| 6,6,6,6,6, 6,6,6, |
| 7,6,4,3,1, 7,6,1, |
| 7,6,5,3,2, 7,6,2, |
| 7,6,5,4,3, 7,6,3, |
| 7,6,6,5,4, 7,6,4, |
| 7,7,6,6,5, 7,7,5, |
| 7,7,7,6,6, 7,7,6, |
| 7,7,7,7,7, 7,7,7, |
| 8,6,5,3,1, 8,6,1, |
| 8,7,5,4,2, 8,7,2, |
| 8,7,6,4,3, 8,7,3, |
| 8,7,6,5,4, 8,7,4, |
| 8,7,7,6,5, 8,7,5, |
| 8,8,7,7,6, 8,8,6, |
| 8,8,8,7,7, 8,8,7, |
| 8,8,8,8,8, 8,8,8, |
| 9,7,5,3,1, 9,7,1, |
| 9,7,6,4,2, 9,7,2, |
| 9,8,6,5,3, 9,8,3, |
| 9,8,7,5,4, 9,8,4, |
| 9,8,7,6,5, 9,8,5, |
| 9,8,8,7,6, 9,8,6, |
| 9,9,8,8,7, 9,9,7, |
| 9,9,9,8,8, 9,9,8, |
| 9,9,9,9,9, 9,9,9, |
| 10,8,6,3,1, 10,8,1, |
| 10,8,6,4,2, 10,8,2, |
| 10,8,7,5,3, 10,8,3, |
| 10,9,7,6,4, 10,9,4, |
| 10,9,8,6,5, 10,9,5, |
| 10,9,8,7,6, 10,9,6, |
| 10,9,9,8,7, 10,9,7, |
| 10,10,9,9,8, 10,10,8, |
| 10,10,10,9,9, 10,10,9, |
| 10,10,10,10,10, 10,10,10, |
| 11,9,6,4,1, 11,9,1, |
| 11,9,7,4,2, 11,9,2, |
| 11,9,7,5,3, 11,9,3, |
| 11,9,8,6,4, 11,9,4, |
| 11,10,8,7,5, 11,10,5, |
| 11,10,9,7,6, 11,10,6, |
| 11,10,9,8,7, 11,10,7, |
| 11,10,10,9,8, 11,10,8, |
| 11,11,10,10,9, 11,11,9, |
| 11,11,11,10,10, 11,11,10, |
| 11,11,11,11,11, 11,11,11, |
| 12,9,7,4,1, 12,9,1, |
| 12,10,7,5,2, 12,10,2, |
| 12,10,8,5,3, 12,10,3, |
| 12,10,8,6,4, 12,10,4, |
| 12,10,9,7,5, 12,10,5, |
| 12,11,9,8,6, 12,11,6, |
| 12,11,10,8,7, 12,11,7, |
| 12,11,10,9,8, 12,11,8, |
| 12,11,11,10,9, 12,11,9, |
| 12,12,11,11,10, 12,12,10, |
| 12,12,12,11,11, 12,12,11, |
| 12,12,12,12,12, 12,12,12, |
| |
| 1,1,1,1,1, 1,1,1, |
| 2,2,2,1,1, 2,1,1, |
| 2,2,2,2,2, 2,2,2, |
| 3,3,2,2,1, 3,2,1, |
| 3,3,3,2,2, 3,2,2, |
| 3,3,3,3,3, 3,3,3, |
| 4,3,3,2,1, 4,2,1, |
| 4,4,3,3,2, 4,3,2, |
| 4,4,4,3,3, 4,3,3, |
| 4,4,4,4,4, 4,4,4, |
| 5,4,3,2,1, 5,2,1, |
| 5,4,4,3,2, 5,3,2, |
| 5,5,4,4,3, 5,4,3, |
| 5,5,5,4,4, 5,4,4, |
| 5,5,5,5,5, 5,5,5, |
| 6,5,4,2,1, 6,2,1, |
| 6,5,4,3,2, 6,3,2, |
| 6,5,5,4,3, 6,4,3, |
| 6,6,5,5,4, 6,5,4, |
| 6,6,6,5,5, 6,5,5, |
| 6,6,6,6,6, 6,6,6, |
| 7,6,4,3,1, 7,3,1, |
| 7,6,5,3,2, 7,3,2, |
| 7,6,5,4,3, 7,4,3, |
| 7,6,6,5,4, 7,5,4, |
| 7,7,6,6,5, 7,6,5, |
| 7,7,7,6,6, 7,6,6, |
| 7,7,7,7,7, 7,7,7, |
| 8,6,5,3,1, 8,3,1, |
| 8,7,5,4,2, 8,4,2, |
| 8,7,6,4,3, 8,4,3, |
| 8,7,6,5,4, 8,5,4, |
| 8,7,7,6,5, 8,6,5, |
| 8,8,7,7,6, 8,7,6, |
| 8,8,8,7,7, 8,7,7, |
| 8,8,8,8,8, 8,8,8, |
| 9,7,5,3,1, 9,3,1, |
| 9,7,6,4,2, 9,4,2, |
| 9,8,6,5,3, 9,5,3, |
| 9,8,7,5,4, 9,5,4, |
| 9,8,7,6,5, 9,6,5, |
| 9,8,8,7,6, 9,7,6, |
| 9,9,8,8,7, 9,8,7, |
| 9,9,9,8,8, 9,8,8, |
| 9,9,9,9,9, 9,9,9, |
| 10,8,6,3,1, 10,3,1, |
| 10,8,6,4,2, 10,4,2, |
| 10,8,7,5,3, 10,5,3, |
| 10,9,7,6,4, 10,6,4, |
| 10,9,8,6,5, 10,6,5, |
| 10,9,8,7,6, 10,7,6, |
| 10,9,9,8,7, 10,8,7, |
| 10,10,9,9,8, 10,9,8, |
| 10,10,10,9,9, 10,9,9, |
| 10,10,10,10,10, 10,10,10, |
| 11,9,6,4,1, 11,4,1, |
| 11,9,7,4,2, 11,4,2, |
| 11,9,7,5,3, 11,5,3, |
| 11,9,8,6,4, 11,6,4, |
| 11,10,8,7,5, 11,7,5, |
| 11,10,9,7,6, 11,7,6, |
| 11,10,9,8,7, 11,8,7, |
| 11,10,10,9,8, 11,9,8, |
| 11,11,10,10,9, 11,10,9, |
| 11,11,11,10,10, 11,10,10, |
| 11,11,11,11,11, 11,11,11, |
| 12,9,7,4,1, 12,4,1, |
| 12,10,7,5,2, 12,5,2, |
| 12,10,8,5,3, 12,5,3, |
| 12,10,8,6,4, 12,6,4, |
| 12,10,9,7,5, 12,7,5, |
| 12,11,9,8,6, 12,8,6, |
| 12,11,10,8,7, 12,8,7, |
| 12,11,10,9,8, 12,9,8, |
| 12,11,11,10,9, 12,10,9, |
| 12,12,11,11,10, 12,11,10, |
| 12,12,12,11,11, 12,11,11, |
| 12,12,12,12,12, 12,12,12, |
| }; |
| |
| // Backmap a single desired probability into an entry in kLgProbV2Tbl |
| static const uint8 kLgProbV2TblBackmap[13] = { |
| 0, |
| 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, |
| }; |
| |
| |
| // Always advances one UTF-8 character |
| static const uint8 kAdvanceOneChar[256] = { |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, |
| 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, |
| }; |
| |
| // Does not advance past space or cr/lf/nul |
| static const uint8 kAdvanceOneCharButSpace[256] = { |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, |
| 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, |
| }; |
| |
| // Advances *only* on space or ASCII vowel (or illegal byte) |
| static const uint8 kAdvanceOneCharSpaceVowel[256] = { |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0, |
| 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| }; |
| |
| // Advances *only* on space (or illegal byte) |
| static const uint8 kAdvanceOneCharSpace[256] = { |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, |
| }; |
| |
| |
| //------------------------------------------------------------------------------ |
| // General |
| //------------------------------------------------------------------------------ |
| static inline int minint(int a, int b) {return (a < b) ? a: b;} |
| static inline int maxint(int a, int b) {return (a > b) ? a: b;} |
| |
| // Here to make available for debugging |
| int ReliabilityDelta(int value1, int value2, int count); |
| int ReliabilityMainstream(int topscore, int len, int mean_score); |
| |
| // Returns "0" for too small |
| inline const char* MyExtLanguageCode(Language lang) { |
| return ExtLanguageCode(lang); |
| } |
| |
| // Map script into Latin, Cyrillic, Arabic, Other. Used in keeping track of |
| // amount of training data for language-script combinations |
| inline int LScript4(UnicodeLScript lscript) { |
| if (lscript == ULScript_Latin) {return 0;} |
| if (lscript == ULScript_Cyrillic) {return 1;} |
| if (lscript == ULScript_Arabic) {return 2;} |
| return 3; |
| } |
| |
| |
| // Routines to access 3 or 5 log probabilities in a single byte. |
| |
| // Return address of 8-byte entry[i] |
| inline const uint8* LgProb2TblEntry(int i) { |
| return &kLgProbV2Tbl[i * 8]; |
| } |
| |
| // Return one of five probabilities in an entry |
| // CURRENTLY UNUSED |
| inline uint8 LgProb5(const uint8* entry, int j) { |
| return entry[j]; |
| } |
| |
| // Return one of three probabilities in an entry |
| inline uint8 LgProb3(const uint8* entry, int j) { |
| return entry[j + 5]; |
| } |
| |
| |
| |
| //------------------------------------------------------------------------------ |
| // Hashing groups of 1/2/4/8 letters, perhaps with spaces or underscores |
| //------------------------------------------------------------------------------ |
| |
| // Pick up 1..12 bytes and hash them via mask/shift/add. NO pre/post |
| // OVERSHOOTS up to 3 bytes |
| uint32 BiHashV25(const char* word_ptr, int bytecount); |
| |
| // Pick up 1..12 bytes plus pre/post space and hash them via mask/shift/add |
| // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes |
| uint32 QuadHashV25(const char* word_ptr, int bytecount); |
| |
| // Pick up 1..12 bytes plus pre/post '_' and hash them via mask/shift/add |
| // OVERSHOOTS up to 3 bytes |
| uint32 QuadHashV25Underscore(const char* word_ptr, int bytecount); |
| |
| |
| // Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add |
| // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes |
| // For runtime use of tables V3 |
| uint64 OctaHash40(const char* word_ptr, int bytecount); |
| |
| uint64 OctaHash40underscore(const char* word_ptr, int bytecount); |
| |
| |
| // From 32-bit gram FP, return hash table subscript and remaining key |
| inline void QuadFPJustHash(uint32 quadhash, |
| uint32 keymask, |
| int bucketcount, |
| uint32* subscr, uint32* hashkey) { |
| *subscr = (quadhash + (quadhash >> 12)) & (bucketcount - 1); |
| *hashkey = quadhash & keymask; |
| } |
| |
| // Look up 32-bit gram FP in caller-passed table |
| // Typical size 256K entries (1.5MB) |
| // Two-byte hashkey |
| inline const uint32 QuadHashV3Lookup4(const cld::CLDTableSummary* gram_obj, |
| uint32 quadhash) { |
| |
| uint32 subscr, hashkey; |
| const IndirectProbBucket4* quadtable = gram_obj->kCLDTable; |
| uint32 keymask = gram_obj->kCLDTableKeyMask; |
| int bucketcount = gram_obj->kCLDTableSize; |
| QuadFPJustHash(quadhash, keymask, bucketcount, &subscr, &hashkey); |
| const IndirectProbBucket4* bucket_ptr = &quadtable[subscr]; |
| // Four-way associative, 4 compares |
| if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[0]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[1]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[2]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[3]; |
| } |
| return 0; |
| } |
| |
| |
| // Map 40 bits to subscript, hashkey, expected 18-22 bit subscript (min 16) |
| // wwwwwwww xxxxxxxx xxxxxxxx yyyyyyyy yyyyyyyy |
| // + ........ ....wwww wwwwxxxx xxxxxxxx xxxxyyyy |
| // 00000000 00000000 00000011 11111111 11111111 (18-bit bucketcount-1) |
| // |
| // hashkey: |
| // wwwwxxxx xxxxxxxx xxxx.... ........ (20-bit keymask) |
| // 12-bit shift in subscript mixes in ~4 letters x 4 bits each |
| |
| // From 40-bit gram FP, return hash table subscript and remaining key |
| inline void OctaFPJustHash(uint64 longwordhash, |
| uint32 keymask, |
| int bucketcount, |
| uint32* subscr, uint32* hashkey) { |
| uint32 temp = (longwordhash + (longwordhash >> 12)) & (bucketcount - 1); |
| *subscr = temp; |
| temp = longwordhash >> 4; |
| *hashkey = temp & keymask; |
| } |
| |
| // Look up 40-bit gram FP in caller-passed table |
| // Typical size 256K-4M entries (1-16MB) |
| // 24-12 bit hashkey packed with 8-20 bit indirect lang/probs |
| // keymask is 0xfffff000 for 20-bit hashkey and 12-bit indirect |
| inline const uint32 OctaHashV3Lookup4(const cld::CLDTableSummary* gram_obj, |
| uint64 longwordhash) { |
| uint32 subscr, hashkey; |
| const IndirectProbBucket4* octatable = gram_obj->kCLDTable; |
| uint32 keymask = gram_obj->kCLDTableKeyMask; |
| int bucketcount = gram_obj->kCLDTableSize; |
| OctaFPJustHash(longwordhash, keymask, bucketcount, |
| &subscr, &hashkey); |
| const IndirectProbBucket4* bucket_ptr = &octatable[subscr]; |
| // Four-way associative, 4 compares |
| if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[0]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[1]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[2]; |
| } |
| if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) { |
| return bucket_ptr->keyvalue[3]; |
| } |
| return 0; |
| } |
| |
| |
| |
| //------------------------------------------------------------------------------ |
| // Scoring single groups of letters |
| //------------------------------------------------------------------------------ |
| |
| // UNIGRAM score one => tote |
| // Input: 1-byte entry of subscript into unigram probs, plus |
| // an accumulator tote. |
| // Output: running sums in tote updated |
| void ProcessProbV25UniTote(int propval, Tote* tote); |
| |
| // BIGRAM, QUADGRAM, OCTAGRAM score one => tote |
| // Input: 4-byte entry of 3 language numbers and one probability subscript, |
| // plus an accumulator tote. (language 0 means unused entry) |
| // Output: running sums in tote updated |
| void ProcessProbV25Tote(uint32 probs, Tote* tote); |
| |
| |
| //------------------------------------------------------------------------------ |
| // Routines to accumulate probabilities |
| //------------------------------------------------------------------------------ |
| |
| // Score up to n=gram_limit unigrams, returning number of bytes consumed |
| // Caller supplies table, such as compact_lang_det_generated_ctjkvz_b1_obj |
| int DoUniScoreV3(const UTF8PropObj* unigram_obj, |
| const char* isrc, int srclen, int advance_by, |
| int* tote_grams, int gram_limit, Tote* chunk_tote); |
| |
| |
| // Score all words in isrc, using languages that have bigrams (CJK) |
| // Caller supplies table, such as &kCjkBiTable_obj or &kGibberishTable_obj |
| // Return number of bigrams that hit in the hash table |
| int DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj, |
| const char* isrc, int srclen, Tote* chunk_tote); |
| |
| |
| // Score up to n=gram_limit quadgrams, returning number of bytes consumed |
| // Caller supplies table, such as &kQuadTable_obj or &kGibberishTable_obj |
| int DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj, |
| const char* isrc, int srclen, int advance_by, |
| int* tote_grams, int gram_limit, Tote* chunk_tote); |
| |
| // Score all octagrams (words) in isrc, using languages that have quadgrams |
| // Caller supplies table, such as &kLongWord8Table_obj |
| // Return number of words that hit in the hash table |
| int DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj, |
| const char* isrc, int srclen, Tote* chunk_tote); |
| |
| //------------------------------------------------------------------------------ |
| // Reliability calculations, for single language and between languages |
| //------------------------------------------------------------------------------ |
| |
| // Reliability = 0..100 |
| static const int kMinReliable = 75; |
| |
| // Calculate ratio of score per 1KB vs. expected score per 1KB |
| double GetNormalizedScore(Language lang, UnicodeLScript lscript, |
| int bytes, int score); |
| |
| // Calculate reliablity of len bytes of script lscript with chunk_tote |
| int GetReliability(int len, UnicodeLScript lscript, const Tote* chunk_tote); |
| |
| |
| //------------------------------------------------------------------------------ |
| // Miscellaneous |
| //------------------------------------------------------------------------------ |
| |
| // Make languages packed into uint32 values non-zero |
| // These routines later could remap so languages not in QuadHash tables are not |
| // represented, and so that any thrashing in accumulation is eliminated |
| uint8 inline PackLanguage(Language lang) { |
| return static_cast<uint8>(lang + 1);} |
| |
| Language inline UnpackLanguage(int ilang) { |
| return static_cast<Language>(ilang - 1);} |
| |
| // Useful single-byte tests |
| bool inline IsUTF8ContinueByte(char c) { |
| return static_cast<signed char>(c) < -64;} |
| bool inline IsUTF8HighByte(char c) { |
| return static_cast<signed char>(c) < 0;} |
| |
| |
| // Demote all languages except Top40 and plus_one |
| // Do this just before sorting |
| void DemoteNotTop40(Tote* chunk_tote, int packed_plus_one); |
| |
| } // End namespace cld |
| |
| |
| #endif // ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_ |
