17898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project/* 27898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Copyright (C) 2009 The Android Open Source Project 37898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 47898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Licensed under the Apache License, Version 2.0 (the "License"); 57898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * you may not use this file except in compliance with the License. 67898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * You may obtain a copy of the License at 77898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 87898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * http://www.apache.org/licenses/LICENSE-2.0 97898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Unless required by applicable law or agreed to in writing, software 117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * distributed under the License is distributed on an "AS IS" BASIS, 127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * See the License for the specific language governing permissions and 147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * limitations under the License. 157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project */ 167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/userdict.h" 187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/splparser.h" 197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/ngram.h" 207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <stdio.h> 217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <string.h> 227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <stdlib.h> 237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <cutils/log.h> 247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <unistd.h> 257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <fcntl.h> 267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <sys/stat.h> 277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <assert.h> 287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <ctype.h> 297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <sys/types.h> 307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <sys/time.h> 317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <time.h> 327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <pthread.h> 337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <math.h> 347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectnamespace ime_pinyin { 367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectstatic uint64 _ellapse_ = 0; 397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectstatic struct timeval _tv_start_, _tv_end_; 407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define DEBUG_PERF_BEGIN \ 417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project do { \ 427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project gettimeofday(&_tv_start_, NULL); \ 437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } while(0) 447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define DEBUG_PERF_END \ 457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project do { \ 467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project gettimeofday(&_tv_end_, NULL); \ 477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project _ellapse_ = (_tv_end_.tv_sec - _tv_start_.tv_sec) * 1000000 + \ 487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (_tv_end_.tv_usec - _tv_start_.tv_usec); \ 497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } while(0) 507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define LOGD_PERF(message) \ 51a40797a8da53e570d4046e4dfee601be4b1bdb63Steve Block ALOGD("PERFORMANCE[%s] %llu usec.", message, _ellapse_); 527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#else 537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define DEBUG_PERF_BEGIN 547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define DEBUG_PERF_END 557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define LOGD_PERF(message) 567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project// XXX File load and write are thread-safe by g_mutex_ 597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectstatic pthread_mutex_t g_mutex_ = PTHREAD_MUTEX_INITIALIZER; 607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectstatic struct timeval g_last_update_ = {0, 0}; 617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline uint32 UserDict::get_dict_file_size(UserDictInfo * info) { 637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (4 + info->lemma_size + (info->lemma_count << 3) 647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (info->lemma_count << 2) 667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (info->sync_count << 2) 697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + sizeof(*info)); 717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline LmaScoreType UserDict::translate_score(int raw_score) { 747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 1) ori_freq: original user frequency 757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 ori_freq = extract_score_freq(raw_score); 767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 2) lmt_off: lmt index (week offset for example) 777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 lmt_off = ((raw_score & 0xffff0000) >> 16); 787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (kUserDictLMTBitWidth < 16) { 797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 mask = ~(1 << kUserDictLMTBitWidth); 807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt_off &= mask; 817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 3) now_off: current time index (current week offset for example) 837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // assuming load_time_ is around current time 847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 now_off = load_time_.tv_sec; 857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project now_off = (now_off - kUserDictLMTSince) / kUserDictLMTGranularity; 867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project now_off = (now_off << (64 - kUserDictLMTBitWidth)); 877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project now_off = (now_off >> (64 - kUserDictLMTBitWidth)); 887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 4) factor: decide expand-factor 897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int delta = now_off - lmt_off; 907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (delta > 4) 917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project delta = 4; 927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int factor = 80 - (delta << 4); 937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project double tf = (double)(dict_info_.total_nfreq + total_other_nfreq_); 957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (LmaScoreType)(log((double)factor * (double)ori_freq / tf) 967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * NGram::kLogValueAmplifier); 977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline int UserDict::extract_score_freq(int raw_score) { 1007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Frequence stored in lowest 16 bits 1017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int freq = (raw_score & 0x0000ffff); 1027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return freq; 1037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline uint64 UserDict::extract_score_lmt(int raw_score) { 1067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 lmt = ((raw_score & 0xffff0000) >> 16); 1077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (kUserDictLMTBitWidth < 16) { 1087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 mask = ~(1 << kUserDictLMTBitWidth); 1097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt &= mask; 1107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt = lmt * kUserDictLMTGranularity + kUserDictLMTSince; 1127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lmt; 1137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline int UserDict::build_score(uint64 lmt, int freq) { 1167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt = (lmt - kUserDictLMTSince) / kUserDictLMTGranularity; 1177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt = (lmt << (64 - kUserDictLMTBitWidth)); 1187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt = (lmt >> (64 - kUserDictLMTBitWidth)); 1197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lmt16 = (uint16)lmt; 1207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int s = freq; 1217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project s &= 0x0000ffff; 1227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project s = (lmt16 << 16) | s; 1237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return s; 1247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline int64 UserDict::utf16le_atoll(uint16 *s, int len) { 1277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int64 ret = 0; 1287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (len <= 0) 1297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret; 1307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int flag = 1; 1327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * endp = s + len; 1337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (*s == '-') { 1347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project flag = -1; 1357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project s++; 1367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (*s == '+') { 1377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project s++; 1387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (*s >= '0' && *s <= '9' && s < endp) { 1417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret += ret * 10 + (*s) - '0'; 1427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project s++; 1437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret * flag; 1457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline int UserDict::utf16le_lltoa(int64 v, uint16 *s, int size) { 1487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!s || size <= 0) 1497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 1507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 *endp = s + size; 1517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int ret_len = 0; 1527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (v < 0) { 1537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(s++) = '-'; 1547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ++ret_len; 1557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project v *= -1; 1567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 *b = s; 1597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (s < endp && v != 0) { 1607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(s++) = '0' + (v % 10); 1617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project v = v / 10; 1627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ++ret_len; 1637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (v != 0) 1667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 1677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project --s; 1697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (b < s) { 1717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *b = *s; 1727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ++b, --s; 1737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret_len; 1767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline void UserDict::set_lemma_flag(uint32 offset, uint8 flag) { 1797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 1807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_[offset] |= flag; 1817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline char UserDict::get_lemma_flag(uint32 offset) { 1847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 1857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (char)(lemmas_[offset]); 1867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline char UserDict::get_lemma_nchar(uint32 offset) { 1897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 1907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (char)(lemmas_[offset + 1]); 1917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline uint16 * UserDict::get_lemma_spell_ids(uint32 offset) { 1947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 1957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (uint16 *)(lemmas_ + offset + 2); 1967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline uint16 * UserDict::get_lemma_word(uint32 offset) { 1997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 2007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 2017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return (uint16 *)(lemmas_ + offset + 2 + (nchar << 1)); 2027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline LemmaIdType UserDict::get_max_lemma_id() { 2057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // When a lemma is deleted, we don't not claim its id back for 2067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // simplicity and performance 2077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return start_id_ + dict_info_.lemma_count - 1; 2087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline bool UserDict::is_valid_lemma_id(LemmaIdType id) { 2117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (id >= start_id_ && id <= get_max_lemma_id()) 2127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline bool UserDict::is_valid_state() { 2177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ == USER_DICT_NONE) 2187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectUserDict::UserDict() 2237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project : start_id_(0), 2247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project version_(0), 2257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_(NULL), 2267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_(NULL), 2277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_(NULL), 2287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_(NULL), 2297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 2307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_(NULL), 2317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 2327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 2337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_(NULL), 2347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sync_count_size_(0), 2357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 2367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_(NULL), 2377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_count_left_(0), 2387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_(0), 2397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_file_(NULL), 2407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_(USER_DICT_NONE) { 2417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(&dict_info_, 0, sizeof(dict_info_)); 2427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(&load_time_, 0, sizeof(load_time_)); 2437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 2447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache_init(); 2457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 2467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectUserDict::~UserDict() { 2497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project close_dict(); 2507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::load_dict(const char *file_name, LemmaIdType start_id, 2537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType end_id) { 2547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 2557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 2567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 2577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_file_ = strdup(file_name); 2587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!dict_file_) 2597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project start_id_ = start_id; 2627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (false == validate(file_name) && false == reset(file_name)) { 2647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 2657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (false == load(file_name, start_id)) { 2677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 2687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SYNC; 2717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project gettimeofday(&load_time_, NULL); 2737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 2757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 2767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("load_dict"); 2777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 2787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project error: 2807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free((void*)dict_file_); 2817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project start_id_ = 0; 2827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::close_dict() { 2867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ == USER_DICT_NONE) 2877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ == USER_DICT_SYNC) 2897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto out; 2907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If dictionary is written back by others, 2927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // we can not simply write back here 2937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // To do a safe flush, we have to discard all newly added 2947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // lemmas and try to reload dict file. 2957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_lock(&g_mutex_); 2967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (load_time_.tv_sec > g_last_update_.tv_sec || 2977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (load_time_.tv_sec == g_last_update_.tv_sec && 2987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project load_time_.tv_usec > g_last_update_.tv_usec)) { 2997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back(); 3007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project gettimeofday(&g_last_update_, NULL); 3017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 3037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project out: 3057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free((void*)dict_file_); 3067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(lemmas_); 3077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(offsets_); 3087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(offsets_by_id_); 3097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(scores_); 3107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(ids_); 3117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 3127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(predicts_); 3137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 3147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project version_ = 0; 3167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_file_ = NULL; 3177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_ = NULL; 3187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 3197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_ = NULL; 3207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sync_count_size_ = 0; 3217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 3227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_ = NULL; 3237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_ = NULL; 3247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_ = NULL; 3257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_ = NULL; 3267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 3277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_ = NULL; 3287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 3297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(&dict_info_, 0, sizeof(dict_info_)); 3317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_count_left_ = 0; 3327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_ = 0; 3337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_NONE; 3347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 3367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t UserDict::number_of_lemmas() { 3397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_info_.lemma_count; 3407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::reset_milestones(uint16 from_step, MileStoneHandle from_handle) { 3437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 3447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectMileStoneHandle UserDict::extend_dict(MileStoneHandle from_handle, 3477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const DictExtPara *dep, 3487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, 3497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_max, size_t *lpi_num) { 3507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 3517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 3527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool need_extend = false; 3547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 3567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 3577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 3587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *lpi_num = _get_lpis(dep->splids, dep->splids_extended + 1, 3597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items, lpi_max, &need_extend); 3607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 3617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 3627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("extend_dict"); 3637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 3647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ((*lpi_num > 0 || need_extend) ? 1 : 0); 3657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::is_fuzzy_prefix_spell_id( 3687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * id1, uint16 len1, const UserDictSearchable *searchable) { 3697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (len1 < searchable->splids_len) 3707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 3717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 3737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 3747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (i = 0; i < searchable->splids_len; i++) { 3757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const char py1 = *spl_trie.get_spelling_str(id1[i]); 3767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 off = 8 * (i % 4); 3777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const char py2 = ((searchable->signature[i/4] & (0xff << off)) >> off); 3787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (py1 == py2) 3797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 3807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 3817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 1; 3837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::fuzzy_compare_spell_id( 3867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * id1, uint16 len1, const UserDictSearchable *searchable) { 3877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (len1 < searchable->splids_len) 3887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return -1; 3897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (len1 > searchable->splids_len) 3907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 1; 3917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 3937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 3947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (i = 0; i < len1; i++) { 3957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const char py1 = *spl_trie.get_spelling_str(id1[i]); 3967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 off = 8 * (i % 4); 3977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const char py2 = ((searchable->signature[i/4] & (0xff << off)) >> off); 3987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (py1 == py2) 3997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 4007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (py1 > py2) 4017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 1; 4027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return -1; 4037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 4057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::is_prefix_spell_id( 4087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * fullids, uint16 fulllen, 4097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const UserDictSearchable *searchable) { 4107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fulllen < searchable->splids_len) 4117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 4127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 4147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < searchable->splids_len; i++) { 4157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 start_id = searchable->splid_start[i]; 4167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 count = searchable->splid_count[i]; 4177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fullids[i] >= start_id && fullids[i] < start_id + count) 4187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 4197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else 4207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 4217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 4237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::equal_spell_id( 4267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * fullids, uint16 fulllen, 4277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const UserDictSearchable *searchable) { 4287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fulllen != searchable->splids_len) 4297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 4307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 4327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < fulllen; i++) { 4337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 start_id = searchable->splid_start[i]; 4347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 count = searchable->splid_count[i]; 4357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fullids[i] >= start_id && fullids[i] < start_id + count) 4367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 4377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else 4387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 4397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 4417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint32 UserDict::locate_first_in_offsets(const UserDictSearchable * searchable) { 4447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 begin = 0; 4457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 end = dict_info_.lemma_count - 1; 4467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 middle = -1; 4477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 first_prefix = middle; 4497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 last_matched = middle; 4507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (begin <= end) { 4527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle = (begin + end) >> 1; 4537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[middle]; 4547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 4557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * splids = get_lemma_spell_ids(offset); 4567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int cmp = fuzzy_compare_spell_id(splids, nchar, searchable); 4577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int pre = is_fuzzy_prefix_spell_id(splids, nchar, searchable); 4587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (pre) 4607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_prefix = middle; 4617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cmp < 0) { 4637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin = middle + 1; 4647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (cmp > 0) { 4657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 4667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 4677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 4687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_matched = middle; 4697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return first_prefix; 4737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::prepare_locate(UserDictSearchable *searchable, 4767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 *splid_str, 4777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 splid_str_len) { 4787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project searchable->splids_len = splid_str_len; 4797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(searchable->signature, 0, sizeof(searchable->signature)); 4807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 4827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 4837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < splid_str_len; i++) { 4847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_trie.is_half_id(splid_str[i])) { 4857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project searchable->splid_count[i] = 4867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_trie.half_to_full(splid_str[i], 4877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project &(searchable->splid_start[i])); 4887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 4897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project searchable->splid_count[i] = 1; 4907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project searchable->splid_start[i] = splid_str[i]; 4917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const unsigned char py = *spl_trie.get_spelling_str(splid_str[i]); 4937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project searchable->signature[i>>2] |= (py << (8 * (i % 4))); 4947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t UserDict::get_lpis(const uint16 *splid_str, uint16 splid_str_len, 4987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, size_t lpi_max) { 4997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return _get_lpis(splid_str, splid_str_len, lpi_items, lpi_max, NULL); 5007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 5017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t UserDict::_get_lpis(const uint16 *splid_str, 5037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 splid_str_len, LmaPsbItem *lpi_items, 5047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_max, bool * need_extend) { 5057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool tmp_extend; 5067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!need_extend) 5077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project need_extend = &tmp_extend; 5087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *need_extend = false; 5107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 5127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 5137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lpi_max <= 0) 5147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 5157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == pthread_mutex_trylock(&g_mutex_)) { 5177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (load_time_.tv_sec < g_last_update_.tv_sec || 5187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (load_time_.tv_sec == g_last_update_.tv_sec && 5197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project load_time_.tv_usec < g_last_update_.tv_usec)) { 5207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Others updated disk file, have to reload 5217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 5227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project flush_cache(); 5237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 5247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 5257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 5277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictSearchable searchable; 5307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project prepare_locate(&searchable, splid_str, splid_str_len); 5317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 max_off = dict_info_.lemma_count; 5337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 5347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 middle; 5357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 start, count; 5367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool cached = cache_hit(&searchable, &start, &count); 5377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cached) { 5387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle = start; 5397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project max_off = start + count; 5407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 5417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle = locate_first_in_offsets(&searchable); 5427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project start = middle; 5437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#else 5457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 middle = locate_first_in_offsets(&searchable); 5467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 5477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (middle == -1) { 5497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 5507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!cached) 5517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache_push(USER_DICT_MISS_CACHE, &searchable, 0, 0); 5527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 5537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 5547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_current = 0; 5577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool fuzzy_break = false; 5597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool prefix_break = false; 5607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while ((size_t)middle < max_off && !fuzzy_break && !prefix_break) { 5617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lpi_current >= lpi_max) 5627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 5637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[middle]; 5647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Ignore deleted lemmas 5657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offset & kUserDictOffsetFlagRemove) { 5667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle++; 5677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 5687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 5707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * splids = get_lemma_spell_ids(offset); 5717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 5727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!cached && 0 != fuzzy_compare_spell_id(splids, nchar, &searchable)) { 5737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#else 5747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 != fuzzy_compare_spell_id(splids, nchar, &searchable)) { 5757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 5767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fuzzy_break = true; 5777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (prefix_break == false) { 5807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_fuzzy_prefix_spell_id(splids, nchar, &searchable)) { 5817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (*need_extend == false && 5827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project is_prefix_spell_id(splids, nchar, &searchable)) { 5837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *need_extend = true; 5847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 5867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project prefix_break = true; 5877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (equal_spell_id(splids, nchar, &searchable) == true) { 5917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_current].psb = translate_score(scores_[middle]); 5927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_current].id = ids_[middle]; 5937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_current].lma_len = nchar; 5947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_current++; 5957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle++; 5977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 6007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!cached) { 6017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project count = middle - start; 6027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache_push(USER_DICT_CACHE, &searchable, start, count); 6037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 6057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lpi_current; 6077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 6087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectuint16 UserDict::get_lemma_str(LemmaIdType id_lemma, char16* str_buf, 6107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 str_max) { 6117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 6127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(id_lemma) == false) 6147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_by_id_[id_lemma - start_id_]; 6167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 6177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * str = get_lemma_word(offset); 6187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 m = nchar < str_max -1 ? nchar : str_max - 1; 6197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int i = 0; 6207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < m; i++) { 6217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project str_buf[i] = str[i]; 6227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project str_buf[i] = 0; 6247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return m; 6257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 6267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectuint16 UserDict::get_lemma_splids(LemmaIdType id_lemma, uint16 *splids, 6287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 splids_max, bool arg_valid) { 6297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(id_lemma) == false) 6307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_by_id_[id_lemma - start_id_]; 6327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 6337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * ids = get_lemma_spell_ids(offset); 6347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int i = 0; 6357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < nchar && i < splids_max; i++) 6367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splids[i] = ids[i]; 6377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return i; 6387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 6397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t UserDict::predict(const char16 last_hzs[], uint16 hzs_len, 6417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NPredictItem *npre_items, size_t npre_max, 6427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t b4_used) { 6437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 new_added = 0; 6447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 6457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 end = dict_info_.lemma_count - 1; 6467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int j = locate_first_in_predicts((const uint16*)last_hzs, hzs_len); 6477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j == -1) 6487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (j <= end) { 6517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = predicts_[j]; 6527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Ignore deleted lemmas 6537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offset & kUserDictOffsetFlagRemove) { 6547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j++; 6557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 6567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 6587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * words = get_lemma_word(offset); 6597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * splids = get_lemma_spell_ids(offset); 6607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (nchar <= hzs_len) { 6627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j++; 6637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 6647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (memcmp(words, last_hzs, hzs_len << 1) == 0) { 6677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (new_added >= npre_max) { 6687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return new_added; 6697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 cpy_len = 6717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (nchar < kMaxPredictSize ? (nchar << 1) : (kMaxPredictSize << 1)) 6727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project - (hzs_len << 1); 6737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[new_added].his_len = hzs_len; 6747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[new_added].psb = get_lemma_score(words, splids, nchar); 6757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memcpy(npre_items[new_added].pre_hzs, words + hzs_len, cpy_len); 6767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if ((cpy_len >> 1) < kMaxPredictSize) { 6777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[new_added].pre_hzs[cpy_len >> 1] = 0; 6787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project new_added++; 6807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 6817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 6827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j++; 6857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 6877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return new_added; 6887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 6897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint32 UserDict::locate_in_offsets(char16 lemma_str[], uint16 splid_str[], 6917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len) { 6927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 max_off = dict_info_.lemma_count; 6937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictSearchable searchable; 6957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project prepare_locate(&searchable, splid_str, lemma_len); 6967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 6977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off; 6987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 start, count; 6997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool cached = load_cache(&searchable, &start, &count); 7007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cached) { 7017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off = start; 7027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project max_off = start + count; 7037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 7047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off = locate_first_in_offsets(&searchable); 7057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project start = off; 7067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#else 7087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_first_in_offsets(&searchable); 7097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 7107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == -1) { 7127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return off; 7137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (off < max_off) { 7167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[off]; 7177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offset & kUserDictOffsetFlagRemove) { 7187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off++; 7197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 7207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * splids = get_lemma_spell_ids(offset); 7227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 7237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!cached && 0 != fuzzy_compare_spell_id(splids, lemma_len, &searchable)) 7247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#else 7267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 != fuzzy_compare_spell_id(splids, lemma_len, &searchable)) 7277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 7297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (equal_spell_id(splids, lemma_len, &searchable) == true) { 7307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * str = get_lemma_word(offset); 7317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 7327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (i = 0; i < lemma_len; i++) { 7337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (str[i] == lemma_str[i]) 7347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 7357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i < lemma_len) { 7387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off++; 7397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 7407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 7427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // No need to save_cache here, since current function is invoked by 7437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // put_lemma. It's rarely possible for a user input same lemma twice. 7447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // That means first time user type a new lemma, it is newly added into 7457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // user dictionary, then it's possible that user type the same lemma 7467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // again. 7477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Another reason save_cache can not be invoked here is this function 7487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // aborts when lemma is found, and it never knows the count. 7497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 7507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return off; 7517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off++; 7537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return -1; 7567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 7577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 7597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectuint32 UserDict::locate_where_to_insert_in_predicts( 7607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * words, int lemma_len) { 7617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 begin = 0; 7627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 end = dict_info_.lemma_count - 1; 7637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 middle = end; 7647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 last_matched = middle; 7667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (begin <= end) { 7687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle = (begin + end) >> 1; 7697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[middle]; 7707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 7717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * ws = get_lemma_word(offset); 7727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 minl = nchar < lemma_len ? nchar : lemma_len; 7747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 k = 0; 7757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int cmp = 0; 7767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; k < minl; k++) { 7787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ws[k] < words[k]) { 7797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = -1; 7807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (ws[k] > words[k]) { 7827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = 1; 7837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cmp == 0) { 7877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (nchar < lemma_len) 7887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = -1; 7897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else if (nchar > lemma_len) 7907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = 1; 7917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cmp < 0) { 7947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin = middle + 1; 7957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_matched = middle; 7967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (cmp > 0) { 7977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 7987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 7997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 8007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_matched = middle; 8017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return last_matched; 8057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint32 UserDict::locate_first_in_predicts(const uint16 * words, int lemma_len) { 8087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 begin = 0; 8097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 end = dict_info_.lemma_count - 1; 8107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 middle = -1; 8117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 last_matched = middle; 8137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (begin <= end) { 8157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project middle = (begin + end) >> 1; 8167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[middle]; 8177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 nchar = get_lemma_nchar(offset); 8187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const uint16 * ws = get_lemma_word(offset); 8197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 minl = nchar < lemma_len ? nchar : lemma_len; 8217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 k = 0; 8227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int cmp = 0; 8237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; k < minl; k++) { 8257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ws[k] < words[k]) { 8267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = -1; 8277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 8287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (ws[k] > words[k]) { 8297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = 1; 8307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 8317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cmp == 0) { 8347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (nchar >= lemma_len) 8357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_matched = middle; 8367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (nchar < lemma_len) 8377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = -1; 8387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else if (nchar > lemma_len) 8397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cmp = 1; 8407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cmp < 0) { 8437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin = middle + 1; 8447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (cmp > 0) { 8457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 8467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 8477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = middle - 1; 8487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return last_matched; 8527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 8557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::get_lemma_id(char16 lemma_str[], uint16 splids[], 8577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len) { 8587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(lemma_str, splids, lemma_len); 8597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == -1) { 8607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ids_[off]; 8647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLmaScoreType UserDict::get_lemma_score(LemmaIdType lemma_id) { 8677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 8687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(lemma_id) == false) 8707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return translate_score(_get_lemma_score(lemma_id)); 8737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLmaScoreType UserDict::get_lemma_score(char16 lemma_str[], uint16 splids[], 8767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len) { 8777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 8787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return translate_score(_get_lemma_score(lemma_str, splids, lemma_len)); 8807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::_get_lemma_score(LemmaIdType lemma_id) { 8837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 8847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(lemma_id) == false) 8867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_by_id_[lemma_id - start_id_]; 8897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 8917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * spl = get_lemma_spell_ids(offset); 8927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * wrd = get_lemma_word(offset); 8937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(wrd, spl, nchar); 8957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == -1) { 8967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return scores_[off]; 9007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::_get_lemma_score(char16 lemma_str[], uint16 splids[], 9037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len) { 9047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 9057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 9067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(lemma_str, splids, lemma_len); 9087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == -1) { 9097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 9107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return scores_[off]; 9137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 9167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::remove_lemma_from_sync_list(uint32 offset) { 9177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 9187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 9197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < dict_info_.sync_count; i++) { 9207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned int off = (syncs_[i] & kUserDictOffsetMask); 9217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == offset) 9227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 9237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i < dict_info_.sync_count) { 9257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_[i] = syncs_[dict_info_.sync_count - 1]; 9267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.sync_count--; 9277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 9307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 9327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::remove_lemma_from_predict_list(uint32 offset) { 9337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset &= kUserDictOffsetMask; 9347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i = 0; 9357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; i < dict_info_.lemma_count; i++) { 9367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned int off = (predicts_[i] & kUserDictOffsetMask); 9377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == offset) { 9387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[i] |= kUserDictOffsetFlagRemove; 9397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 9407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 9447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::remove_lemma_by_offset_index(int offset_index) { 9467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 9477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 9487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = offset_index; 9507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off == -1) { 9517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 9527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_[off]; 9557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 9567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[off] |= kUserDictOffsetFlagRemove; 9587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 9607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Remove corresponding sync item 9617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project remove_lemma_from_sync_list(offset); 9627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 9637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 9657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project remove_lemma_from_predict_list(offset); 9667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 9677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.free_count++; 9687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.free_size += (2 + (nchar << 2)); 9697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_OFFSET_DIRTY) 9717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_OFFSET_DIRTY; 9727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 9737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::remove_lemma(LemmaIdType lemma_id) { 9767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 9777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 9787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(lemma_id) == false) 9797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 9807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_by_id_[lemma_id - start_id_]; 9817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 9837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * spl = get_lemma_spell_ids(offset); 9847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * wrd = get_lemma_word(offset); 9857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(wrd, spl, nchar); 9877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return remove_lemma_by_offset_index(off); 9897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::flush_cache() { 9927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType start_id = start_id_; 9937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const char * file = strdup(dict_file_); 9947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!file) 9957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 9967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project close_dict(); 9977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project load_dict(file, start_id, kUserDictIdEnd); 9987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free((void*)file); 9997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 10007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache_init(); 10017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 10027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 10037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 10047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::reset(const char *file) { 10067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fopen(file, "w+"); 10077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!fp) { 10087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 version = kUserDictVersion; 10117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t wred = fwrite(&version, 1, 4, fp); 10127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictInfo info; 10137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(&info, 0, sizeof(info)); 10147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // By default, no limitation for lemma count and size 10157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // thereby, reclaim_ratio is never used 10167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project wred += fwrite(&info, 1, sizeof(info), fp); 10177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (wred != sizeof(info) + sizeof(version)) { 10187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 10197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unlink(file); 10207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 10237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 10247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 10257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::validate(const char *file) { 10277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // b is ignored in POSIX compatible os including Linux 10287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // while b is important flag for Windows to specify binary mode 10297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fopen(file, "rb"); 10307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!fp) { 10317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t size; 10357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t readed; 10367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 version; 10377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictInfo dict_info; 10387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // validate 10407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = fseek(fp, 0, SEEK_END); 10417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err) { 10427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size = ftell(fp); 10467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (size < 4 + sizeof(dict_info)) { 10477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project err = fseek(fp, 0, SEEK_SET); 10517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err) { 10527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = fread(&version, 1, sizeof(version), fp); 10567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < sizeof(version)) { 10577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (version != kUserDictVersion) { 10607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project err = fseek(fp, -1 * sizeof(dict_info), SEEK_END); 10647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err) { 10657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = fread(&dict_info, 1, sizeof(dict_info), fp); 10697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed != sizeof(dict_info)) { 10707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (size != get_dict_file_size(&dict_info)) { 10747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 10757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 10787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 10797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project error: 10817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 10827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 10847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::load(const char *file, LemmaIdType start_id) { 10867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 != pthread_mutex_trylock(&g_mutex_)) { 10877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // b is ignored in POSIX compatible os including Linux 10907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // while b is important flag for Windows to specify binary mode 10917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fopen(file, "rb"); 10927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!fp) { 10937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 10947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 10957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 10967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 10977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t readed, toread; 10987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictInfo dict_info; 10997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 *lemmas = NULL; 11007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *offsets = NULL; 11017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 11027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *syncs = NULL; 11037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *scores = NULL; 11057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *ids = NULL; 11067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *offsets_by_id = NULL; 11077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 11087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *predicts = NULL; 11097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t i; 11117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err; 11127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project err = fseek(fp, -1 * sizeof(dict_info), SEEK_END); 11147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err) goto error; 11157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = fread(&dict_info, 1, sizeof(dict_info), fp); 11177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed != sizeof(dict_info)) goto error; 11187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas = (uint8 *)malloc( 11207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info.lemma_size + 11217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (kUserDictPreAlloc * (2 + (kUserDictAverageNchar << 2)))); 11227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!lemmas) goto error; 11247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2); 11267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!offsets) goto error; 11277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 11297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2); 11307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!predicts) goto error; 11317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 11347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs = (uint32 *)malloc((dict_info.sync_count + kUserDictPreAlloc) << 2); 11357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!syncs) goto error; 11367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2); 11397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!scores) goto error; 11407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2); 11427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!ids) goto error; 11437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id = (uint32 *)malloc( 11457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (dict_info.lemma_count + kUserDictPreAlloc) << 2); 11467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!offsets_by_id) goto error; 11477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project err = fseek(fp, 4, SEEK_SET); 11497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err) goto error; 11507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = 0; 11527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (readed < dict_info.lemma_size && !ferror(fp) && !feof(fp)) { 11537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed += fread(lemmas + readed, 1, dict_info.lemma_size - readed, fp); 11547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < dict_info.lemma_size) 11567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 11577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project toread = (dict_info.lemma_count << 2); 11597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = 0; 11607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (readed < toread && !ferror(fp) && !feof(fp)) { 11617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed += fread((((uint8*)offsets) + readed), 1, toread - readed, fp); 11627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < toread) 11647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 11657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 11677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project toread = (dict_info.lemma_count << 2); 11687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = 0; 11697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (readed < toread && !ferror(fp) && !feof(fp)) { 11707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed += fread((((uint8*)predicts) + readed), 1, toread - readed, fp); 11717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < toread) 11737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 11747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = 0; 11777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (readed < toread && !ferror(fp) && !feof(fp)) { 11787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed += fread((((uint8*)scores) + readed), 1, toread - readed, fp); 11797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < toread) 11817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 11827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 11847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project toread = (dict_info.sync_count << 2); 11857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed = 0; 11867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (readed < toread && !ferror(fp) && !feof(fp)) { 11877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project readed += fread((((uint8*)syncs) + readed), 1, toread - readed, fp); 11887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (readed < toread) 11907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto error; 11917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 11927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (i = 0; i < dict_info.lemma_count; i++) { 11947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids[i] = start_id + i; 11957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id[i] = offsets[i]; 11967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 11977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 11987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_ = lemmas; 11997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_ = offsets; 12007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 12017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_ = syncs; 12027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sync_count_size_ = dict_info.sync_count + kUserDictPreAlloc; 12037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_ = offsets_by_id; 12057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_ = scores; 12067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_ = ids; 12077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 12087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_ = predicts; 12097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_count_left_ = kUserDictPreAlloc; 12117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_ = kUserDictPreAlloc * (2 + (kUserDictAverageNchar << 2)); 12127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memcpy(&dict_info_, &dict_info, sizeof(dict_info)); 12137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SYNC; 12147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 12167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 12187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 12197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project error: 12217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lemmas) free(lemmas); 12227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offsets) free(offsets); 12237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 12247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (syncs) free(syncs); 12257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (scores) free(scores); 12277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ids) free(ids); 12287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offsets_by_id) free(offsets_by_id); 12297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 12307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (predicts) free(predicts); 12317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 12337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 12347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 12357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 12367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back() { 12387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX write back is only allowed from close_dict due to thread-safe sake 12397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ == USER_DICT_NONE || state_ == USER_DICT_SYNC) 12407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 12417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int fd = open(dict_file_, O_WRONLY); 12427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fd == -1) 12437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 12447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project switch (state_) { 12457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_DEFRAGMENTED: 12467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back_all(fd); 12477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_LEMMA_DIRTY: 12497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back_lemma(fd); 12507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_OFFSET_DIRTY: 12527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back_offset(fd); 12537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_SCORE_DIRTY: 12557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back_score(fd); 12567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 12587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_SYNC_DIRTY: 12597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write_back_sync(fd); 12607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project default: 12637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 12647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 12657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // It seems truncate is not need on Linux, Windows except Mac 12667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // I am doing it here anyway for safety. 12677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project off_t cur = lseek(fd, 0, SEEK_CUR); 12687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ftruncate(fd, cur); 12697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project close(fd); 12707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SYNC; 12717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 12727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 12747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back_sync(int fd) { 12757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = lseek(fd, 4 + dict_info_.lemma_size 12767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (dict_info_.lemma_count << 3) 12777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 12787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (dict_info_.lemma_count << 2) 12797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project , SEEK_SET); 12817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 12827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 12837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, syncs_, dict_info_.sync_count << 2); 12847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, &dict_info_, sizeof(dict_info_)); 12857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 12867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 12887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back_offset(int fd) { 12897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = lseek(fd, 4 + dict_info_.lemma_size, SEEK_SET); 12907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 12917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 12927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, offsets_, dict_info_.lemma_count << 2); 12937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 12947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, predicts_, dict_info_.lemma_count << 2); 12957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 12967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, scores_, dict_info_.lemma_count << 2); 12977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 12987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, syncs_, dict_info_.sync_count << 2); 12997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, &dict_info_, sizeof(dict_info_)); 13017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 13027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back_score(int fd) { 13047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = lseek(fd, 4 + dict_info_.lemma_size 13057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (dict_info_.lemma_count << 2) 13067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 13077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + (dict_info_.lemma_count << 2) 13087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project , SEEK_SET); 13107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 13117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 13127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, scores_, dict_info_.lemma_count << 2); 13137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 13147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, syncs_, dict_info_.sync_count << 2); 13157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, &dict_info_, sizeof(dict_info_)); 13177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 13187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back_lemma(int fd) { 13207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = lseek(fd, 4, SEEK_SET); 13217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 13227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 13237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // New lemmas are always appended, no need to write whole lemma block 13247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t need_write = kUserDictPreAlloc * 13257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (2 + (kUserDictAverageNchar << 2)) - lemma_size_left_; 13267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project err = lseek(fd, dict_info_.lemma_size - need_write, SEEK_CUR); 13277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 13287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 13297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, lemmas_ + dict_info_.lemma_size - need_write, need_write); 13307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, offsets_, dict_info_.lemma_count << 2); 13327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 13337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, predicts_, dict_info_.lemma_count << 2); 13347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, scores_, dict_info_.lemma_count << 2); 13367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 13377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, syncs_, dict_info_.sync_count << 2); 13387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, &dict_info_, sizeof(dict_info_)); 13407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 13417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::write_back_all(int fd) { 13437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX lemma_size is handled differently in writeall 13447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // and writelemma. I update lemma_size and lemma_count in different 13457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // places for these two cases. Should fix it to make it consistent. 13467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int err = lseek(fd, 4, SEEK_SET); 13477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (err == -1) 13487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 13497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, lemmas_, dict_info_.lemma_size); 13507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, offsets_, dict_info_.lemma_count << 2); 13517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 13527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, predicts_, dict_info_.lemma_count << 2); 13537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, scores_, dict_info_.lemma_count << 2); 13557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 13567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, syncs_, dict_info_.sync_count << 2); 13577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 13587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project write(fd, &dict_info_, sizeof(dict_info_)); 13597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 13607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 13627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::load_cache(UserDictSearchable *searchable, 13637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *offset, uint32 *length) { 13647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictCache *cache = &caches_[searchable->splids_len - 1]; 13657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->head == cache->tail) 13667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 13677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 j, sig_len = kMaxLemmaSize / 4; 13697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 i = cache->head; 13707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (1) { 13717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j = 0; 13727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; j < sig_len; j++) { 13737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->signatures[i][j] != searchable->signature[j]) 13747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 13757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 13767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j < sig_len) { 13777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project i++; 13787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i >= kUserDictCacheSize) 13797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project i -= kUserDictCacheSize; 13807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i == cache->tail) 13817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 13827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 13837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 13847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *offset = cache->offsets[i]; 13857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *length = cache->lengths[i]; 13867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 13877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 13887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 13897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 13907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::save_cache(UserDictSearchable *searchable, 13927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset, uint32 length) { 13937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictCache *cache = &caches_[searchable->splids_len - 1]; 13947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 next = cache->tail; 13957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 13967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->offsets[next] = offset; 13977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->lengths[next] = length; 13987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 sig_len = kMaxLemmaSize / 4; 13997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 j = 0; 14007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; j < sig_len; j++) { 14017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->signatures[next][j] = searchable->signature[j]; 14027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (++next >= kUserDictCacheSize) { 14057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project next -= kUserDictCacheSize; 14067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (next == cache->head) { 14087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->head++; 14097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->head >= kUserDictCacheSize) { 14107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->head -= kUserDictCacheSize; 14117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->tail = next; 14147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::reset_cache() { 14177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(caches_, 0, sizeof(caches_)); 14187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::load_miss_cache(UserDictSearchable *searchable) { 14217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictMissCache *cache = &miss_caches_[searchable->splids_len - 1]; 14227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->head == cache->tail) 14237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 14247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 j, sig_len = kMaxLemmaSize / 4; 14267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 i = cache->head; 14277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (1) { 14287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j = 0; 14297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; j < sig_len; j++) { 14307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->signatures[i][j] != searchable->signature[j]) 14317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 14327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j < sig_len) { 14347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project i++; 14357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i >= kUserDictMissCacheSize) 14367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project i -= kUserDictMissCacheSize; 14377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i == cache->tail) 14387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 14397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 14407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 14427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 14447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::save_miss_cache(UserDictSearchable *searchable) { 14477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictMissCache *cache = &miss_caches_[searchable->splids_len - 1]; 14487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 next = cache->tail; 14497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 sig_len = kMaxLemmaSize / 4; 14517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 j = 0; 14527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (; j < sig_len; j++) { 14537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->signatures[next][j] = searchable->signature[j]; 14547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (++next >= kUserDictMissCacheSize) { 14577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project next -= kUserDictMissCacheSize; 14587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (next == cache->head) { 14607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->head++; 14617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (cache->head >= kUserDictMissCacheSize) { 14627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->head -= kUserDictMissCacheSize; 14637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache->tail = next; 14667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::reset_miss_cache() { 14697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(miss_caches_, 0, sizeof(miss_caches_)); 14707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::cache_init() { 14737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reset_cache(); 14747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reset_miss_cache(); 14757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::cache_hit(UserDictSearchable *searchable, 14787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 *offset, uint32 *length) { 14797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool hit = load_miss_cache(searchable); 14807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (hit) { 14817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *offset = 0; 14827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *length = 0; 14837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 14847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project hit = load_cache(searchable, offset, length); 14867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (hit) { 14877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 14887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 14897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 14907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 14917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 14927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::cache_push(UserDictCacheType type, 14937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictSearchable *searchable, 14947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset, uint32 length) { 14957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project switch (type) { 14967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_MISS_CACHE: 14977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project save_miss_cache(searchable); 14987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 14997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case USER_DICT_CACHE: 15007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project save_cache(searchable, offset, length); 15017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project default: 15037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 15067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 15077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 15087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 15097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::defragment(void) { 15107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 15117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 15127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 15137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 15147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 15157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Fixup offsets_, set REMOVE flag to lemma's flag if needed 15167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t first_freed = 0; 15177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t first_inuse = 0; 15187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (first_freed < dict_info_.lemma_count) { 15197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Find first freed offset 15207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while ((offsets_[first_freed] & kUserDictOffsetFlagRemove) == 0 && 15217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed < dict_info_.lemma_count) { 15227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed++; 15237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (first_freed < dict_info_.lemma_count) { 15257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Save REMOVE flag to lemma flag 15267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int off = offsets_[first_freed]; 15277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project set_lemma_flag(off, kUserDictLemmaFlagRemove); 15287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 15297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Find first inuse offse after first_freed 15327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_inuse = first_freed + 1; 15337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while ((offsets_[first_inuse] & kUserDictOffsetFlagRemove) && 15347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (first_inuse < dict_info_.lemma_count)) { 15357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Save REMOVE flag to lemma flag 15367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int off = offsets_[first_inuse]; 15377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project set_lemma_flag(off, kUserDictLemmaFlagRemove); 15387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_inuse++; 15397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (first_inuse >= dict_info_.lemma_count) { 15417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Swap offsets_ 15447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int tmp = offsets_[first_inuse]; 15457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[first_inuse] = offsets_[first_freed]; 15467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[first_freed] = tmp; 15477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Move scores_, no need to swap 15487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project tmp = scores_[first_inuse]; 15497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[first_inuse] = scores_[first_freed]; 15507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[first_freed] = tmp; 15517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Swap ids_ 15527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType tmpid = ids_[first_inuse]; 15537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_[first_inuse] = ids_[first_freed]; 15547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_[first_freed] = tmpid; 15557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Go on 15567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed++; 15577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 15597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Fixup predicts_ 15607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed = 0; 15617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_inuse = 0; 15627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (first_freed < dict_info_.lemma_count) { 15637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Find first freed offset 15647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while ((predicts_[first_freed] & kUserDictOffsetFlagRemove) == 0 && 15657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed < dict_info_.lemma_count) { 15667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed++; 15677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (first_freed >= dict_info_.lemma_count) 15697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Find first inuse offse after first_freed 15717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_inuse = first_freed + 1; 15727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while ((predicts_[first_inuse] & kUserDictOffsetFlagRemove) 15737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && (first_inuse < dict_info_.lemma_count)) { 15747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_inuse++; 15757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (first_inuse >= dict_info_.lemma_count) { 15777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 15787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Swap offsets_ 15807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int tmp = predicts_[first_inuse]; 15817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[first_inuse] = predicts_[first_freed]; 15827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[first_freed] = tmp; 15837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Go on 15847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project first_freed++; 15857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 15867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 15877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_count = first_freed; 15887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Fixup lemmas_ 15897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t begin = 0; 15907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t end = 0; 15917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t dst = 0; 15927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int total_size = dict_info_.lemma_size + lemma_size_left_; 15937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int total_count = dict_info_.lemma_count + lemma_count_left_; 15947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t real_size = total_size - lemma_size_left_; 15957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (dst < real_size) { 15967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char flag = get_lemma_flag(dst); 15977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char nchr = get_lemma_nchar(dst); 15987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if ((flag & kUserDictLemmaFlagRemove) == 0) { 15997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dst += nchr * 4 + 2; 16007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 16017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 16037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (dst >= real_size) 16057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 16067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = dst; 16087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (end < real_size) { 16097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin = end + get_lemma_nchar(end) * 4 + 2; 16107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project repeat: 16117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // not used any more 16127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (begin >= real_size) 16137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 16147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char flag = get_lemma_flag(begin); 16157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char nchr = get_lemma_nchar(begin); 16167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (flag & kUserDictLemmaFlagRemove) { 16177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin += nchr * 4 + 2; 16187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto repeat; 16197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = begin + nchr * 4 + 2; 16217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (end < real_size) { 16227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char eflag = get_lemma_flag(end); 16237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project unsigned char enchr = get_lemma_nchar(end); 16247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if ((eflag & kUserDictLemmaFlagRemove) == 0) { 16257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end += enchr * 4 + 2; 16267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 16277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 16297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(lemmas_ + dst, lemmas_ + begin, end - begin); 16317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t j = 0; j < dict_info_.lemma_count; j++) { 16327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offsets_[j] >= begin && offsets_[j] < end) { 16337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[j] -= (begin - dst); 16347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_[ids_[j] - start_id_] = offsets_[j]; 16357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 16377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (predicts_[j] >= begin && predicts_[j] < end) { 16387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[j] -= (begin - dst); 16397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 16417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 16437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t j = 0; j < dict_info_.sync_count; j++) { 16447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (syncs_[j] >= begin && syncs_[j] < end) { 16457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_[j] -= (begin - dst); 16467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 16497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dst += (end - begin); 16507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.free_count = 0; 16537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.free_size = 0; 16547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_size = dst; 16557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_ = total_size - dict_info_.lemma_size; 16567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_count_left_ = total_count - dict_info_.lemma_count; 16577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX Without following code, 16597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // offsets_by_id_ is not reordered. 16607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // That's to say, all removed lemmas' ids are not collected back. 16617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // There may not be room for addition of new lemmas due to 16627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // offsests_by_id_ reason, although lemma_size_left_ is fixed. 16637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // By default, we do want defrag as fast as possible, because 16647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // during defrag procedure, other peers can not write new lemmas 16657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // to user dictionary file. 16667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX If write-back is invoked immediately after 16677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // this defragment, no need to fix up following in-mem data. 16687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint32 i = 0; i < dict_info_.lemma_count; i++) { 16697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_[i] = start_id_ + i; 16707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_[i] = offsets_[i]; 16717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 16727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_DEFRAGMENTED; 16747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 16767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 16777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("defragment"); 16787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 16797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 16807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 16827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::clear_sync_lemmas(unsigned int start, unsigned int end) { 16837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 16847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 16857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (end > dict_info_.sync_count) 16867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project end = dict_info_.sync_count; 16877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(syncs_ + start, syncs_ + end, (dict_info_.sync_count - end) << 2); 16887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.sync_count -= (end - start); 16897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_SYNC_DIRTY) 16907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SYNC_DIRTY; 16917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 16927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::get_sync_count() { 16947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 16957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 16967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_info_.sync_count; 16977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 16987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 16997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::put_lemma_no_sync(char16 lemma_str[], uint16 splids[], 17007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len, uint16 count, uint64 lmt) { 17017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int again = 0; 17027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project begin: 17037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id; 17047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 * syncs_bak = syncs_; 17057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_ = NULL; 17067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project id = _put_lemma(lemma_str, splids, lemma_len, count, lmt); 17077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_ = syncs_bak; 17087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (id == 0 && again == 0) { 17097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if ((dict_info_.limit_lemma_count > 0 && 17107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_count >= dict_info_.limit_lemma_count) 17117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project || (dict_info_.limit_lemma_size > 0 && 17127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_size + (2 + (lemma_len << 2)) 17137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project > dict_info_.limit_lemma_size)) { 17147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX Always reclaim and defrag in sync code path 17157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // sync thread is background thread and ok with heavy work 17167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reclaim(); 17177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project defragment(); 17187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project flush_cache(); 17197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project again = 1; 17207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project goto begin; 17217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return id; 17247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 17257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::put_lemmas_no_sync_from_utf16le_string(char16 * lemmas, int len) { 17277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int newly_added = 0; 17287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingParser * spl_parser = new SpellingParser(); 17307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!spl_parser) { 17317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 17327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 17347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 17357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 17367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 *ptr = lemmas; 17377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Extract pinyin,words,frequence,last_mod_time 17397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * p = ptr, * py16 = ptr; 17407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * hz16 = NULL; 17417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int py16_len = 0; 1742c78dcdec607b4f50738a5b84fa170403e23c75adXiaotao Duan uint16 splid[kMaxLemmaSize]; 17437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int splid_len = 0; 17447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int hz16_len = 0; 17457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * fr16 = NULL; 17467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int fr16_len = 0; 17477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (p - ptr < len) { 17497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Pinyin 17507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project py16 = p; 17517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_len = 0; 17527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (*p != 0x2c && (p - ptr) < len) { 17537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (*p == 0x20) 17547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_len++; 17557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project p++; 17567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_len++; 17587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (p - ptr == len) 17597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 17607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project py16_len = p - py16; 1761c78dcdec607b4f50738a5b84fa170403e23c75adXiaotao Duan if (kMaxLemmaSize < splid_len) { 1762c78dcdec607b4f50738a5b84fa170403e23c75adXiaotao Duan break; 17637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool is_pre; 17657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int splidl = spl_parser->splstr16_to_idxs_f( 1766c78dcdec607b4f50738a5b84fa170403e23c75adXiaotao Duan py16, py16_len, splid, NULL, kMaxLemmaSize, is_pre); 17677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (splidl != splid_len) 17687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 17697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Phrase 17707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project hz16 = ++p; 17717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (*p != 0x2c && (p - ptr) < len) { 17727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project p++; 17737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project hz16_len = p - hz16; 17757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (hz16_len != splid_len) 17767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 17777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Frequency 17787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16 = ++p; 17797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16_len = 0; 17807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (*p != 0x2c && (p - ptr) < len) { 17817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project p++; 17827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16_len = p - fr16; 17847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 intf = (uint32)utf16le_atoll(fr16, fr16_len); 17857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Last modified time 17867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16 = ++p; 17877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16_len = 0; 17887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (*p != 0x3b && (p - ptr) < len) { 17897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project p++; 17907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fr16_len = p - fr16; 17927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 last_mod = utf16le_atoll(fr16, fr16_len); 17937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project put_lemma_no_sync(hz16, splid, splid_len, intf, last_mod); 17957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project newly_added++; 17967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 17977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project p++; 17987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 17997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 18017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 18027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("put_lemmas_no_sync_from_utf16le_string"); 18037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 18047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return newly_added; 18057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 18067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectint UserDict::get_sync_lemmas_in_utf16le_string_from_beginning( 18087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * str, int size, int * count) { 18097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int len = 0; 18107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *count = 0; 18117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int left_len = size; 18137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 18157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return len; 18167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie * spl_trie = &SpellingTrie::get_instance(); 18187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!spl_trie) { 18197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 18207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 i; 18237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (i = 0; i < dict_info_.sync_count; i++) { 18247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int offset = syncs_[i]; 18257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 18267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 *spl = get_lemma_spell_ids(offset); 18277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 *wrd = get_lemma_word(offset); 18287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int score = _get_lemma_score(wrd, spl, nchar); 18297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static char score_temp[32], *pscore_temp = score_temp; 18317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static char16 temp[256], *ptemp = temp; 18327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pscore_temp = score_temp; 18347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ptemp = temp; 18357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 j; 18377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Add pinyin 18387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (j = 0; j < nchar; j++) { 18397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int ret_len = spl_trie->get_spelling_str16( 18407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl[j], ptemp, temp + sizeof(temp) - ptemp); 18417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ret_len <= 0) 18427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 18437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ptemp += ret_len; 18447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = ' '; 18467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j = 0; 18487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 18497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j < nchar) { 18527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ptemp--; 18557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = ','; 18577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Add phrase 18617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (j = 0; j < nchar; j++) { 18627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = wrd[j]; 18647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 18667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j < nchar) { 18697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = ','; 18737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Add frequency 18777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 intf = extract_score_freq(score); 18787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int ret_len = utf16le_lltoa(intf, ptemp, temp + sizeof(temp) - ptemp); 18797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ret_len <= 0) 18807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ptemp += ret_len; 18827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = ','; 18847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Add last modified time 18887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 last_mod = extract_score_lmt(score); 18897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_len = utf16le_lltoa(last_mod, ptemp, temp + sizeof(temp) - ptemp); 18907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ret_len <= 0) 18917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ptemp += ret_len; 18937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ptemp < temp + sizeof(temp) - 1) { 18947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *(ptemp++) = ';'; 18957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 18967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 18977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 18987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 18997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Write to string 19007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int need_len = ptemp - temp; 19017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (need_len > left_len) 19027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 19037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memcpy(str + len, temp, need_len * 2); 19047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project left_len -= need_len; 19057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project len += need_len; 19077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (*count)++; 19087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (len > 0) { 19117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_SYNC_DIRTY) 19127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SYNC_DIRTY; 19137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return len; 19157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 19167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 19187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool UserDict::state(UserDictStat * stat) { 19207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 19217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 19227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!stat) 19237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 19247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->version = version_; 19257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->file_name = dict_file_; 19267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->load_time.tv_sec = load_time_.tv_sec; 19277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->load_time.tv_usec = load_time_.tv_usec; 19287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_lock(&g_mutex_); 19297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->last_update.tv_sec = g_last_update_.tv_sec; 19307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->last_update.tv_usec = g_last_update_.tv_usec; 19317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project pthread_mutex_unlock(&g_mutex_); 19327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->disk_size = get_dict_file_size(&dict_info_); 19337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->lemma_count = dict_info_.lemma_count; 19347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->lemma_size = dict_info_.lemma_size; 19357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->delete_count = dict_info_.free_count; 19367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->delete_size = dict_info_.free_size; 19377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 19387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->sync_count = dict_info_.sync_count; 19397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 19407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->limit_lemma_count = dict_info_.limit_lemma_count; 19417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->limit_lemma_size = dict_info_.limit_lemma_size; 19427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project stat->reclaim_ratio = dict_info_.reclaim_ratio; 19437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 19447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 19457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::set_limit(uint32 max_lemma_count, 19477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 max_lemma_size, uint32 reclaim_ratio) { 19487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.limit_lemma_count = max_lemma_count; 19497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.limit_lemma_size = max_lemma_size; 19507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (reclaim_ratio > 100) 19517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reclaim_ratio = 100; 19527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.reclaim_ratio = reclaim_ratio; 19537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 19547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::reclaim() { 19567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 19577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 19587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project switch (dict_info_.reclaim_ratio) { 19607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case 0: 19617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 19627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project case 100: 19637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // TODO: CLEAR to be implemented 19647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(false); 19657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 19667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project default: 19677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 19687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX Reclaim is only based on count, not size 19717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 count = dict_info_.lemma_count; 19727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int rc = count * dict_info_.reclaim_ratio / 100; 19737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictScoreOffsetPair * score_offset_pairs = NULL; 19757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project score_offset_pairs = (UserDictScoreOffsetPair *)malloc( 19767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sizeof(UserDictScoreOffsetPair) * rc); 19777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (score_offset_pairs == NULL) { 19787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return; 19797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (int i = 0; i < rc; i++) { 19827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int s = scores_[i]; 19837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project score_offset_pairs[i].score = s; 19847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project score_offset_pairs[i].offset_index = i; 19857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (int i = (rc + 1) / 2; i >= 0; i--) 19887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project shift_down(score_offset_pairs, i, rc); 19897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint32 i = rc; i < dict_info_.lemma_count; i++) { 19917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int s = scores_[i]; 19927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (s < score_offset_pairs[0].score) { 19937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project score_offset_pairs[0].score = s; 19947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project score_offset_pairs[0].offset_index = i; 19957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project shift_down(score_offset_pairs, 0, rc); 19967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 19987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 19997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (int i = 0; i < rc; i++) { 20007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int off = score_offset_pairs[i].offset_index; 20017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project remove_lemma_by_offset_index(off); 20027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (rc > 0) { 20047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_OFFSET_DIRTY) 20057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_OFFSET_DIRTY; 20067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(score_offset_pairs); 20097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 20107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline void UserDict::swap(UserDictScoreOffsetPair * sop, int i, int j) { 20127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int s = sop[i].score; 20137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int p = sop[i].offset_index; 20147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[i].score = sop[j].score; 20157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[i].offset_index = sop[j].offset_index; 20167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[j].score = s; 20177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[j].offset_index = p; 20187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 20197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::shift_down(UserDictScoreOffsetPair * sop, int i, int n) { 20217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int par = i; 20227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (par < n) { 20237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int left = par * 2 + 1; 20247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int right = left + 1; 20257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (left >= n && right >= n) 20267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 20277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (right >= n) { 20287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (sop[left].score > sop[par].score) { 20297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project swap(sop, left, par); 20307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project par = left; 20317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 20327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (sop[left].score > sop[right].score && 20347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[left].score > sop[par].score) { 20357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project swap(sop, left, par); 20367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project par = left; 20377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 20387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (sop[right].score > sop[left].score && 20397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sop[right].score > sop[par].score) { 20407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project swap(sop, right, par); 20417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project par = right; 20427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 20437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 20457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 20477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::put_lemma(char16 lemma_str[], uint16 splids[], 20497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len, uint16 count) { 20507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return _put_lemma(lemma_str, splids, lemma_len, count, time(NULL)); 20517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 20527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::_put_lemma(char16 lemma_str[], uint16 splids[], 20547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len, uint16 count, uint64 lmt) { 20557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 20567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 20577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 20587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 20597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 20607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(lemma_str, splids, lemma_len); 20617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off != -1) { 20627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int delta_score = count - scores_[off]; 20637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.total_nfreq += delta_score; 20647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[off] = build_score(lmt, count); 20657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_SCORE_DIRTY) 20667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SCORE_DIRTY; 20677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 20687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 20697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("_put_lemma(update)"); 20707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 20717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ids_[off]; 20727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 20737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if ((dict_info_.limit_lemma_count > 0 && 20747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_count >= dict_info_.limit_lemma_count) 20757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project || (dict_info_.limit_lemma_size > 0 && 20767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_size + (2 + (lemma_len << 2)) 20777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project > dict_info_.limit_lemma_size)) { 20787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX Don't defragment here, it's too time-consuming. 20797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 20807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int flushed = 0; 20827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lemma_count_left_ == 0 || 20837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_ < (size_t)(2 + (lemma_len << 2))) { 20847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 20857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // XXX When there is no space for new lemma, we flush to disk 20867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // flush_cache() may be called by upper user 20877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // and better place shoule be found instead of here 20887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project flush_cache(); 20897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project flushed = 1; 20907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Or simply return and do nothing 20917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // return 0; 20927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 20937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 20947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 20957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF(flushed ? "_put_lemma(flush+add)" : "_put_lemma(add)"); 20967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 20977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id = append_a_lemma(lemma_str, splids, lemma_len, count, lmt); 20987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 20997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (syncs_ && id != 0) { 21007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project queue_lemma_for_sync(id); 21017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return id; 21047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 21067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 21077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 21097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::queue_lemma_for_sync(LemmaIdType id) { 21107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (dict_info_.sync_count < sync_count_size_) { 21117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_[dict_info_.sync_count++] = offsets_by_id_[id - start_id_]; 21127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 21137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 * syncs = (uint32*)realloc( 21147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_, (sync_count_size_ + kUserDictPreAlloc) << 2); 21157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (syncs) { 21167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project sync_count_size_ += kUserDictPreAlloc; 21177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_ = syncs; 21187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project syncs_[dict_info_.sync_count++] = offsets_by_id_[id - start_id_]; 21197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 21227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::update_lemma(LemmaIdType lemma_id, int16 delta_count, 21257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool selected) { 21267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 21277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_BEGIN; 21287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_state() == false) 21307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 21317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (is_valid_lemma_id(lemma_id) == false) 21327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 21337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 offset = offsets_by_id_[lemma_id - start_id_]; 21347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint8 lemma_len = get_lemma_nchar(offset); 21357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 * lemma_str = get_lemma_word(offset); 21367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * splids = get_lemma_spell_ids(offset); 21377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int32 off = locate_in_offsets(lemma_str, splids, lemma_len); 21397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (off != -1) { 21407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int score = scores_[off]; 21417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project int count = extract_score_freq(score); 21427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint64 lmt = extract_score_lmt(score); 21437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (count + delta_count > kUserDictMaxFrequency || 21447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project count + delta_count < count) { 21457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project delta_count = kUserDictMaxFrequency - count; 21467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project count += delta_count; 21487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.total_nfreq += delta_count; 21497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (selected) { 21507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lmt = time(NULL); 21517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[off] = build_score(lmt, count); 21537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_SCORE_DIRTY) 21547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_SCORE_DIRTY; 21557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___DEBUG_PERF___ 21567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DEBUG_PERF_END; 21577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LOGD_PERF("update_lemma"); 21587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___SYNC_ENABLED___ 21607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project queue_lemma_for_sync(ids_[off]); 21617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ids_[off]; 21637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 21657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 21667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t UserDict::get_total_lemma_count() { 21687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_info_.total_nfreq; 21697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 21707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid UserDict::set_total_lemma_count_of_others(size_t count) { 21727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project total_other_nfreq_ = count; 21737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 21747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType UserDict::append_a_lemma(char16 lemma_str[], uint16 splids[], 21767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lemma_len, uint16 count, uint64 lmt) { 21777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id = get_max_lemma_id() + 1; 21787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t offset = dict_info_.lemma_size; 21797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (offset > kUserDictOffsetMask) 21807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 21817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_[offset] = 0; 21837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemmas_[offset + 1] = (uint8)lemma_len; 21847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t i = 0; i < lemma_len; i++) { 21857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *((uint16*)&lemmas_[offset + 2 + (i << 1)]) = splids[i]; 21867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *((char16*)&lemmas_[offset + 2 + (lemma_len << 1) + (i << 1)]) 21877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project = lemma_str[i]; 21887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 21897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 off = dict_info_.lemma_count; 21907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[off] = offset; 21917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[off] = build_score(lmt, count); 21927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_[off] = id; 21937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 21947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[off] = offset; 21957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 21967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_by_id_[id - start_id_] = offset; 21987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 21997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_count++; 22007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.lemma_size += (2 + (lemma_len << 2)); 22017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_count_left_--; 22027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lemma_size_left_ -= (2 + (lemma_len << 2)); 22037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Sort 22057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project UserDictSearchable searchable; 22077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project prepare_locate(&searchable, splids, lemma_len); 22087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t i = 0; 22107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (i < off) { 22117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offset = offsets_[i]; 22127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 nchar = get_lemma_nchar(offset); 22137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * spl = get_lemma_spell_ids(offset); 22147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 <= fuzzy_compare_spell_id(spl, nchar, &searchable)) 22167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 22177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project i++; 22187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 22197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (i != off) { 22207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 temp = offsets_[off]; 22217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(offsets_ + i + 1, offsets_ + i, (off - i) << 2); 22227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project offsets_[i] = temp; 22237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project temp = scores_[off]; 22257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(scores_ + i + 1, scores_ + i, (off - i) << 2); 22267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project scores_[i] = temp; 22277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project temp = ids_[off]; 22297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(ids_ + i + 1, ids_ + i, (off - i) << 2); 22307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ids_[i] = temp; 22317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 22327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___PREDICT_ENABLED___ 22347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 j = 0; 22357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 * words_new = get_lemma_word(predicts_[off]); 22367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project j = locate_where_to_insert_in_predicts(words_new, lemma_len); 22377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (j != off) { 22387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint32 temp = predicts_[off]; 22397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memmove(predicts_ + j + 1, predicts_ + j, (off - j) << 2); 22407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project predicts_[j] = temp; 22417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 22427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 22437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (state_ < USER_DICT_LEMMA_DIRTY) 22457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project state_ = USER_DICT_LEMMA_DIRTY; 22467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___CACHE_ENABLED___ 22487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cache_init(); 22497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif 22507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 22517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_info_.total_nfreq += count; 22527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return id; 22537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 22547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2255