1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef LATINIME_DEFINES_H 18#define LATINIME_DEFINES_H 19 20#ifdef __GNUC__ 21#define AK_FORCE_INLINE __attribute__((always_inline)) __inline__ 22#else // __GNUC__ 23#define AK_FORCE_INLINE inline 24#endif // __GNUC__ 25 26#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 27#undef AK_FORCE_INLINE 28#define AK_FORCE_INLINE inline 29#endif // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 30 31// Must be equal to Constants.Dictionary.MAX_WORD_LENGTH in Java 32#define MAX_WORD_LENGTH 48 33// Must be equal to BinaryDictionary.MAX_RESULTS in Java 34#define MAX_RESULTS 18 35// Must be equal to ProximityInfo.MAX_PROXIMITY_CHARS_SIZE in Java 36#define MAX_PROXIMITY_CHARS_SIZE 16 37#define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2 38 39#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 40#include <android/log.h> 41#ifndef LOG_TAG 42#define LOG_TAG "LatinIME: " 43#endif // LOG_TAG 44#define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__) 45#define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__) 46 47#define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0) 48#define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0) 49#define INTS_TO_CHARS(input, length, output) do { \ 50 intArrayToCharArray(input, length, output); } while (0) 51 52// TODO: Support full UTF-8 conversion 53AK_FORCE_INLINE static int intArrayToCharArray(const int *source, const int sourceSize, 54 char *dest) { 55 int si = 0; 56 int di = 0; 57 while (si < sourceSize && di < MAX_WORD_LENGTH - 1 && 0 != source[si]) { 58 const int codePoint = source[si++]; 59 if (codePoint < 0x7F) { 60 dest[di++] = codePoint; 61 } else if (codePoint < 0x7FF) { 62 dest[di++] = 0xC0 + (codePoint >> 6); 63 dest[di++] = 0x80 + (codePoint & 0x3F); 64 } else if (codePoint < 0xFFFF) { 65 dest[di++] = 0xE0 + (codePoint >> 12); 66 dest[di++] = 0x80 + ((codePoint & 0xFC0) >> 6); 67 dest[di++] = 0x80 + (codePoint & 0x3F); 68 } 69 } 70 dest[di] = 0; 71 return di; 72} 73 74static inline void dumpWordInfo(const int *word, const int length, const int rank, 75 const int probability) { 76 static char charBuf[50]; 77 const int N = intArrayToCharArray(word, length, charBuf); 78 if (N > 1) { 79 AKLOGI("%2d [ %s ] (%d)", rank, charBuf, probability); 80 } 81} 82 83static inline void dumpResult(const int *outWords, const int *frequencies) { 84 AKLOGI("--- DUMP RESULT ---------"); 85 for (int i = 0; i < MAX_RESULTS; ++i) { 86 dumpWordInfo(&outWords[i * MAX_WORD_LENGTH], MAX_WORD_LENGTH, i, frequencies[i]); 87 } 88 AKLOGI("-------------------------"); 89} 90 91static AK_FORCE_INLINE void dumpWord(const int *word, const int length) { 92 static char charBuf[50]; 93 const int N = intArrayToCharArray(word, length, charBuf); 94 if (N > 1) { 95 AKLOGI("[ %s ]", charBuf); 96 } 97} 98 99#ifndef __ANDROID__ 100#include <cassert> 101#include <execinfo.h> 102#include <stdlib.h> 103 104#define DO_ASSERT_TEST 105#define ASSERT(success) do { if (!(success)) { showStackTrace(); assert(success);} } while (0) 106#define SHOW_STACK_TRACE do { showStackTrace(); } while (0) 107 108static inline void showStackTrace() { 109 void *callstack[128]; 110 int i, frames = backtrace(callstack, 128); 111 char **strs = backtrace_symbols(callstack, frames); 112 for (i = 0; i < frames; ++i) { 113 if (i == 0) { 114 AKLOGI("=== Trace ==="); 115 continue; 116 } 117 AKLOGI("%s", strs[i]); 118 } 119 free(strs); 120} 121#else // __ANDROID__ 122#include <cassert> 123#define DO_ASSERT_TEST 124#define ASSERT(success) assert(success) 125#define SHOW_STACK_TRACE 126#endif // __ANDROID__ 127 128#else // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 129#define AKLOGE(fmt, ...) 130#define AKLOGI(fmt, ...) 131#define DUMP_RESULT(words, frequencies) 132#define DUMP_WORD(word, length) 133#undef DO_ASSERT_TEST 134#define ASSERT(success) 135#define SHOW_STACK_TRACE 136#define INTS_TO_CHARS(input, length, output) 137#endif // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 138 139#ifdef FLAG_DO_PROFILE 140// Profiler 141#include <time.h> 142 143#define PROF_BUF_SIZE 100 144static float profile_buf[PROF_BUF_SIZE]; 145static float profile_old[PROF_BUF_SIZE]; 146static unsigned int profile_counter[PROF_BUF_SIZE]; 147 148#define PROF_RESET prof_reset() 149#define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id] 150#define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while (0) 151#define PROF_START(prof_buf_id) do { \ 152 PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while (0) 153#define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while (0) 154#define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id]) 155#define PROF_CLOCKOUT(prof_buf_id) \ 156 AKLOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id])) 157#define PROF_OUTALL do { AKLOGI("--- %s ---", __FUNCTION__); prof_out(); } while (0) 158 159static inline void prof_reset(void) { 160 for (int i = 0; i < PROF_BUF_SIZE; ++i) { 161 profile_buf[i] = 0; 162 profile_old[i] = 0; 163 profile_counter[i] = 0; 164 } 165} 166 167static inline void prof_out(void) { 168 if (profile_counter[PROF_BUF_SIZE - 1] != 1) { 169 AKLOGI("Error: You must call PROF_OPEN before PROF_CLOSE."); 170 } 171 AKLOGI("Total time is %6.3f ms.", 172 profile_buf[PROF_BUF_SIZE - 1] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC)); 173 float all = 0.0f; 174 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { 175 all += profile_buf[i]; 176 } 177 if (all < 1.0f) all = 1.0f; 178 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { 179 if (profile_buf[i] > 0.0f) { 180 AKLOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.", 181 i, (profile_buf[i] * 100.0f / all), 182 profile_buf[i] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC), 183 profile_counter[i]); 184 } 185 } 186} 187 188#else // FLAG_DO_PROFILE 189#define PROF_BUF_SIZE 0 190#define PROF_RESET 191#define PROF_COUNT(prof_buf_id) 192#define PROF_OPEN 193#define PROF_START(prof_buf_id) 194#define PROF_CLOSE 195#define PROF_END(prof_buf_id) 196#define PROF_CLOCK_OUT(prof_buf_id) 197#define PROF_CLOCKOUT(prof_buf_id) 198#define PROF_OUTALL 199 200#endif // FLAG_DO_PROFILE 201 202#ifdef FLAG_DBG 203#define DEBUG_DICT true 204#define DEBUG_DICT_FULL false 205#define DEBUG_EDIT_DISTANCE false 206#define DEBUG_SHOW_FOUND_WORD false 207#define DEBUG_NODE DEBUG_DICT_FULL 208#define DEBUG_TRACE DEBUG_DICT_FULL 209#define DEBUG_PROXIMITY_INFO false 210#define DEBUG_PROXIMITY_CHARS false 211#define DEBUG_CORRECTION false 212#define DEBUG_CORRECTION_FREQ false 213#define DEBUG_WORDS_PRIORITY_QUEUE false 214#define DEBUG_SAMPLING_POINTS false 215#define DEBUG_POINTS_PROBABILITY false 216#define DEBUG_DOUBLE_LETTER false 217#define DEBUG_CACHE false 218#define DEBUG_DUMP_ERROR false 219#define DEBUG_EVALUATE_MOST_PROBABLE_STRING false 220 221#ifdef FLAG_FULL_DBG 222#define DEBUG_GEO_FULL true 223#else 224#define DEBUG_GEO_FULL false 225#endif 226 227#else // FLAG_DBG 228 229#define DEBUG_DICT false 230#define DEBUG_DICT_FULL false 231#define DEBUG_EDIT_DISTANCE false 232#define DEBUG_SHOW_FOUND_WORD false 233#define DEBUG_NODE false 234#define DEBUG_TRACE false 235#define DEBUG_PROXIMITY_INFO false 236#define DEBUG_PROXIMITY_CHARS false 237#define DEBUG_CORRECTION false 238#define DEBUG_CORRECTION_FREQ false 239#define DEBUG_WORDS_PRIORITY_QUEUE false 240#define DEBUG_SAMPLING_POINTS false 241#define DEBUG_POINTS_PROBABILITY false 242#define DEBUG_DOUBLE_LETTER false 243#define DEBUG_CACHE false 244#define DEBUG_DUMP_ERROR false 245#define DEBUG_EVALUATE_MOST_PROBABLE_STRING false 246 247#define DEBUG_GEO_FULL false 248 249#endif // FLAG_DBG 250 251#ifndef S_INT_MAX 252#define S_INT_MAX 2147483647 // ((1 << 31) - 1) 253#endif 254#ifndef S_INT_MIN 255// The literal constant -2147483648 does not work in C prior C90, because 256// the compiler tries to fit the positive number into an int and then negate it. 257// GCC warns about this. 258#define S_INT_MIN (-2147483647 - 1) // -(1 << 31) 259#endif 260 261#define M_PI_F 3.14159265f 262#define MAX_PERCENTILE 100 263 264// Number of base-10 digits in the largest integer + 1 to leave room for a zero terminator. 265// As such, this is the maximum number of characters will be needed to represent an int as a 266// string, including the terminator; this is used as the size of a string buffer large enough to 267// hold any value that is intended to fit in an integer, e.g. in the code that reads the header 268// of the binary dictionary where a {key,value} string pair scheme is used. 269#define LARGEST_INT_DIGIT_COUNT 11 270 271// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap(). 272// We measured and compared performance of both, and found mmap() is fairly good in terms of 273// loading time, and acceptable even for several initial lookups which involve page faults. 274#define USE_MMAP_FOR_DICTIONARY 275 276#define NOT_VALID_WORD (-99) 277#define NOT_A_CODE_POINT (-1) 278#define NOT_A_DISTANCE (-1) 279#define NOT_A_COORDINATE (-1) 280#define MATCH_CHAR_WITHOUT_DISTANCE_INFO (-2) 281#define PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO (-3) 282#define ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO (-4) 283#define NOT_AN_INDEX (-1) 284#define NOT_A_PROBABILITY (-1) 285 286#define KEYCODE_SPACE ' ' 287#define KEYCODE_SINGLE_QUOTE '\'' 288#define KEYCODE_HYPHEN_MINUS '-' 289 290#define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true 291#define SUGGEST_MULTIPLE_WORDS true 292#define USE_SUGGEST_INTERFACE_FOR_TYPING true 293#define SUGGEST_INTERFACE_OUTPUT_SCALE 1000000.0f 294 295// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. 296#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80 297#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12 298#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58 299#define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50 300#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75 301#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75 302#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70 303#define FULL_MATCHED_WORDS_PROMOTION_RATE 120 304#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90 305#define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70 306#define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105 307#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148 308#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3 309#define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45 310#define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70 311#define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96 312#define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50 313#define TWO_WORDS_CORRECTION_DEMOTION_BASE 80 314#define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1 315#define ZERO_DISTANCE_PROMOTION_RATE 110.0f 316#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f 317#define HALF_SCORE_SQUARED_RADIUS 32.0f 318#define MAX_PROBABILITY 255 319#define MAX_BIGRAM_ENCODED_PROBABILITY 15 320 321// Assuming locale strings such as en_US, sr-Latn etc. 322#define MAX_LOCALE_STRING_LENGTH 10 323 324// Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used 325// for better performance. 326// Holds up to 1 candidate for each word 327#define SUB_QUEUE_MAX_WORDS 1 328#define SUB_QUEUE_MAX_COUNT 10 329#define SUB_QUEUE_MIN_WORD_LENGTH 4 330// TODO: Extend this limitation 331#define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5 332// TODO: Remove this limitation 333#define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12 334// TODO: Remove this limitation 335#define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45 336#define MULTIPLE_WORDS_DEMOTION_RATE 80 337#define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6 338 339#define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35f 340#define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185f 341/* heuristic... This should be changed if we change the unit of the probability. */ 342#define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_PROBABILITY * 58 / 100) 343 344#define MAX_DEPTH_MULTIPLIER 3 345#define FIRST_WORD_INDEX 0 346 347// Max value for length, distance and probability which are used in weighting 348// TODO: Remove 349#define MAX_VALUE_FOR_WEIGHTING 10000000 350 351// The max number of the keys in one keyboard layout 352#define MAX_KEY_COUNT_IN_A_KEYBOARD 64 353 354// TODO: Reduce this constant if possible; check the maximum number of digraphs in the same 355// word in the dictionary for languages with digraphs, like German and French 356#define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5 357 358#define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3 359 360// TODO: Remove 361#define MAX_POINTER_COUNT 1 362#define MAX_POINTER_COUNT_G 2 363 364// Size, in bytes, of the bloom filter index for bigrams 365// 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k, 366// where k is the number of hash functions, n the number of bigrams, and m the number of 367// bits we can test. 368// At the moment 100 is the maximum number of bigrams for a word with the current 369// dictionaries, so n = 100. 1024 buckets give us m = 1024. 370// With 1 hash function, our false positive rate is about 9.3%, which should be enough for 371// our uses since we are only using this to increase average performance. For the record, 372// k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%, 373// and m = 4096 gives 2.4%. 374#define BIGRAM_FILTER_BYTE_SIZE 128 375// Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest 376// prime under 128 * 8. 377#define BIGRAM_FILTER_MODULO 1021 378#if BIGRAM_FILTER_BYTE_SIZE * 8 < BIGRAM_FILTER_MODULO 379#error "BIGRAM_FILTER_MODULO is larger than BIGRAM_FILTER_BYTE_SIZE" 380#endif 381 382// Max number of bigram maps (previous word contexts) to be cached. Increasing this number could 383// improve bigram lookup speed for multi-word suggestions, but at the cost of more memory usage. 384// Also, there are diminishing returns since the most frequently used bigrams are typically near 385// the beginning of the input and are thus the first ones to be cached. Note that these bigrams 386// are reset for each new composing word. 387#define MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP 25 388// Most common previous word contexts currently have 100 bigrams 389#define DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP 100 390 391template<typename T> AK_FORCE_INLINE const T &min(const T &a, const T &b) { return a < b ? a : b; } 392template<typename T> AK_FORCE_INLINE const T &max(const T &a, const T &b) { return a > b ? a : b; } 393 394#define NELEMS(x) (sizeof(x) / sizeof((x)[0])) 395 396// DEBUG 397#define INPUTLENGTH_FOR_DEBUG (-1) 398#define MIN_OUTPUT_INDEX_FOR_DEBUG (-1) 399 400#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 401 TypeName(const TypeName&); \ 402 void operator=(const TypeName&) 403 404#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ 405 TypeName(); \ 406 DISALLOW_COPY_AND_ASSIGN(TypeName) 407 408// Used as a return value for character comparison 409typedef enum { 410 // Same char, possibly with different case or accent 411 MATCH_CHAR, 412 // It is a char located nearby on the keyboard 413 PROXIMITY_CHAR, 414 // Additional proximity char which can differ by language. 415 ADDITIONAL_PROXIMITY_CHAR, 416 // It is a substitution char 417 SUBSTITUTION_CHAR, 418 // It is an unrelated char 419 UNRELATED_CHAR, 420} ProximityType; 421 422typedef enum { 423 NOT_A_DOUBLE_LETTER, 424 A_DOUBLE_LETTER, 425 A_STRONG_DOUBLE_LETTER 426} DoubleLetterLevel; 427 428typedef enum { 429 // Correction for MATCH_CHAR 430 CT_MATCH, 431 // Correction for PROXIMITY_CHAR 432 CT_PROXIMITY, 433 // Correction for ADDITIONAL_PROXIMITY_CHAR 434 CT_ADDITIONAL_PROXIMITY, 435 // Correction for SUBSTITUTION_CHAR 436 CT_SUBSTITUTION, 437 // Skip one omitted letter 438 CT_OMISSION, 439 // Delete an unnecessarily inserted letter 440 CT_INSERTION, 441 // Swap the order of next two touch points 442 CT_TRANSPOSITION, 443 CT_COMPLETION, 444 CT_TERMINAL, 445 // Create new word with space omission 446 CT_NEW_WORD_SPACE_OMITTION, 447 // Create new word with space substitution 448 CT_NEW_WORD_SPACE_SUBSTITUTION, 449} CorrectionType; 450 451// ErrorType is mainly decided by CorrectionType but it is also depending on if 452// the correction has really been performed or not. 453typedef enum { 454 // Substitution, omission and transposition 455 ET_EDIT_CORRECTION, 456 // Proximity error 457 ET_PROXIMITY_CORRECTION, 458 // Completion 459 ET_COMPLETION, 460 // New word 461 // TODO: Remove. 462 // A new word error should be an edit correction error or a proximity correction error. 463 ET_NEW_WORD, 464 // Treat error as an intentional omission when the CorrectionType is omission and the node can 465 // be intentional omission. 466 ET_INTENTIONAL_OMISSION, 467 // Not treated as an error. Tracked for checking exact match 468 ET_NOT_AN_ERROR 469} ErrorType; 470#endif // LATINIME_DEFINES_H 471