17898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project/* 27898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Copyright (C) 2009 The Android Open Source Project 37898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 47898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Licensed under the Apache License, Version 2.0 (the "License"); 57898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * you may not use this file except in compliance with the License. 67898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * You may obtain a copy of the License at 77898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 87898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * http://www.apache.org/licenses/LICENSE-2.0 97898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * 107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * Unless required by applicable law or agreed to in writing, software 117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * distributed under the License is distributed on an "AS IS" BASIS, 127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * See the License for the specific language governing permissions and 147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project * limitations under the License. 157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project */ 167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <assert.h> 187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <stdio.h> 197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include <string.h> 207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/dicttrie.h" 217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/dictbuilder.h" 227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/lpicache.h" 237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/mystdlib.h" 247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#include "../include/ngram.h" 257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectnamespace ime_pinyin { 277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectDictTrie::DictTrie() { 297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_trie_ = SpellingTrie::get_cpinstance(); 307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project root_ = NULL; 327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_ = NULL; 337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_node_num_le0_ = 0; 347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project nodes_ge1_ = NULL; 357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_node_num_ge1_ = 0; 367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_idx_buf_ = NULL; 377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_idx_buf_len_ = 0; 387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project total_lma_num_ = 0; 397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project top_lmas_num_ = 0; 407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_list_ = NULL; 417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_ = NULL; 437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_ = NULL; 447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reset_milestones(0, kFirstValidMileStoneHandle); 457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectDictTrie::~DictTrie() { 487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid DictTrie::free_resource(bool free_dict_list) { 527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL != root_) 537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(root_); 547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project root_ = NULL; 557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL != splid_le0_index_) 577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(splid_le0_index_); 587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_ = NULL; 597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL != nodes_ge1_) 617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(nodes_ge1_); 627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project nodes_ge1_ = NULL; 637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL != nodes_ge1_) 657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free(nodes_ge1_); 667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project nodes_ge1_ = NULL; 677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (free_dict_list) { 697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL != dict_list_) { 707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project delete dict_list_; 717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_list_ = NULL; 737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (parsing_marks_) 767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project delete [] parsing_marks_; 777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_ = NULL; 787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (mile_stones_) 807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project delete [] mile_stones_; 817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_ = NULL; 827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reset_milestones(0, kFirstValidMileStoneHandle); 847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline size_t DictTrie::get_son_offset(const LmaNodeGE1 *node) { 877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ((size_t)node->son_1st_off_l + ((size_t)node->son_1st_off_h << 16)); 887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline size_t DictTrie::get_homo_idx_buf_offset(const LmaNodeGE1 *node) { 917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ((size_t)node->homo_idx_buf_off_l + 927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ((size_t)node->homo_idx_buf_off_h << 16)); 937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectinline LemmaIdType DictTrie::get_lemma_id(size_t id_offset) { 967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id = 0; 977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 pos = kLemmaIdSize - 1; pos > 0; pos--) 987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project id = (id << 8) + lma_idx_buf_[id_offset * kLemmaIdSize + pos]; 997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project id = (id << 8) + lma_idx_buf_[id_offset * kLemmaIdSize]; 1007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return id; 1017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#ifdef ___BUILD_MODEL___ 1047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::build_dict(const char* fn_raw, const char* fn_validhzs) { 1057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project DictBuilder* dict_builder = new DictBuilder(); 1067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 1087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_builder->build_dict(fn_raw, fn_validhzs, this); 1107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::save_dict(FILE *fp) { 1137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == fp) 1147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(&lma_node_num_le0_, sizeof(size_t), 1, fp) != 1) 1177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(&lma_node_num_ge1_, sizeof(size_t), 1, fp) != 1) 1207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(&lma_idx_buf_len_, sizeof(size_t), 1, fp) != 1) 1237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(&top_lmas_num_, sizeof(size_t), 1, fp) != 1) 1267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(root_, sizeof(LmaNodeLE0), lma_node_num_le0_, fp) 1297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project != lma_node_num_le0_) 1307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(nodes_ge1_, sizeof(LmaNodeGE1), lma_node_num_ge1_, fp) 1337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project != lma_node_num_ge1_) 1347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fwrite(lma_idx_buf_, sizeof(unsigned char), lma_idx_buf_len_, fp) != 1377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_idx_buf_len_) 1387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 1417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::save_dict(const char *filename) { 1447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == filename) 1457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == root_ || NULL == dict_list_) 1487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 1517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram &ngram = NGram::get_instance(); 1527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fopen(filename, "wb"); 1547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == fp) 1557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!spl_trie.save_spl_trie(fp) || !dict_list_->save_list(fp) || 1587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project !save_dict(fp) || !ngram.save_ngram(fp)) { 1597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 1607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 1627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 1647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 1657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 1667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#endif // ___BUILD_MODEL___ 1677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::load_dict(FILE *fp) { 1697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == fp) 1707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(&lma_node_num_le0_, sizeof(size_t), 1, fp) != 1) 1737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(&lma_node_num_ge1_, sizeof(size_t), 1, fp) != 1) 1767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(&lma_idx_buf_len_, sizeof(size_t), 1, fp) != 1) 1797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(&top_lmas_num_, sizeof(size_t), 1, fp) != 1 || 1827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project top_lmas_num_ >= lma_idx_buf_len_) 1837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 1847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(false); 1867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project root_ = static_cast<LmaNodeLE0*> 1887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (malloc(lma_node_num_le0_ * sizeof(LmaNodeLE0))); 1897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project nodes_ge1_ = static_cast<LmaNodeGE1*> 1907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (malloc(lma_node_num_ge1_ * sizeof(LmaNodeGE1))); 1917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_idx_buf_ = (unsigned char*)malloc(lma_idx_buf_len_); 1927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project total_lma_num_ = lma_idx_buf_len_ / kLemmaIdSize; 1937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t buf_size = SpellingTrie::get_instance().get_spelling_num() + 1; 1957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(lma_node_num_le0_ <= buf_size); 1967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_ = static_cast<uint16*>(malloc(buf_size * sizeof(uint16))); 1977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 1987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Init the space for parsing. 1997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_ = new ParsingMark[kMaxParsingMark]; 2007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_ = new MileStone[kMaxMileStone]; 2017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reset_milestones(0, kFirstValidMileStoneHandle); 2027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == root_ || NULL == nodes_ge1_ || NULL == lma_idx_buf_ || 2047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NULL == splid_le0_index_ || NULL == parsing_marks_ || 2057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NULL == mile_stones_) { 2067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(false); 2077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(root_, sizeof(LmaNodeLE0), lma_node_num_le0_, fp) 2117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project != lma_node_num_le0_) 2127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(nodes_ge1_, sizeof(LmaNodeGE1), lma_node_num_ge1_, fp) 2157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project != lma_node_num_ge1_) 2167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (fread(lma_idx_buf_, sizeof(unsigned char), lma_idx_buf_len_, fp) != 2197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_idx_buf_len_) 2207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // The quick index for the first level sons 2237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 last_splid = kFullSplIdStart; 2247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t last_pos = 0; 2257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t i = 1; i < lma_node_num_le0_; i++) { 2267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 splid = last_splid; splid < root_[i].spl_idx; splid++) 2277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_[splid - kFullSplIdStart] = last_pos; 2287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_[root_[i].spl_idx - kFullSplIdStart] = 2307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static_cast<uint16>(i); 2317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_splid = root_[i].spl_idx; 2327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project last_pos = i; 2337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 splid = last_splid + 1; 2367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid < buf_size + kFullSplIdStart; splid++) { 2377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(static_cast<size_t>(splid - kFullSplIdStart) < buf_size); 2387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_[splid - kFullSplIdStart] = last_pos + 1; 2397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::load_dict(const char *filename, LemmaIdType start_id, 2457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType end_id) { 2467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == filename || end_id <= start_id) 2477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fopen(filename, "rb"); 2507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == fp) 2517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 2547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_list_ = new DictList(); 2567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == dict_list_) { 2577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 2587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 2627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram &ngram = NGram::get_instance(); 2637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!spl_trie.load_spl_trie(fp) || !dict_list_->load_list(fp) || 2657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project !load_dict(fp) || !ngram.load_ngram(fp) || 2667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project total_lma_num_ > end_id - start_id + 1) { 2677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 2687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 2697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 2737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 2747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 2757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::load_dict_fd(int sys_fd, long start_offset, 2777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project long length, LemmaIdType start_id, 2787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType end_id) { 2797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (start_offset < 0 || length <= 0 || end_id <= start_id) 2807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project FILE *fp = fdopen(sys_fd, "rb"); 2837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == fp) 2847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (-1 == fseek(fp, start_offset, SEEK_SET)) { 2877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 2887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 2927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dict_list_ = new DictList(); 2947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == dict_list_) { 2957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 2967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 2977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 2987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project SpellingTrie &spl_trie = SpellingTrie::get_instance(); 3007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram &ngram = NGram::get_instance(); 3017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!spl_trie.load_spl_trie(fp) || !dict_list_->load_list(fp) || 3037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project !load_dict(fp) || !ngram.load_ngram(fp) || 3047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ftell(fp) < start_offset + length || 3057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project total_lma_num_ > end_id - start_id + 1) { 3067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project free_resource(true); 3077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 3087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 3097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project fclose(fp); 3127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 3137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t DictTrie::fill_lpi_buffer(LmaPsbItem lpi_items[], size_t lpi_max, 3167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node) { 3177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_num = 0; 3187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram& ngram = NGram::get_instance(); 3197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo = 0; homo < (size_t)node->num_of_homo; homo++) { 3207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].id = get_lemma_id(node->homo_idx_buf_off + 3217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project homo); 3227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].lma_len = 1; 3237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].psb = 3247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static_cast<LmaScoreType>(ngram.get_uni_psb(lpi_items[lpi_num].id)); 3257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_num++; 3267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lpi_num >= lpi_max) 3277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 3287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lpi_num; 3317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t DictTrie::fill_lpi_buffer(LmaPsbItem lpi_items[], size_t lpi_max, 3347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t homo_buf_off, LmaNodeGE1 *node, 3357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lma_len) { 3367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_num = 0; 3377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram& ngram = NGram::get_instance(); 3387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo = 0; homo < (size_t)node->num_of_homo; homo++) { 3397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].id = get_lemma_id(homo_buf_off + homo); 3407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].lma_len = lma_len; 3417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_items[lpi_num].psb = 3427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static_cast<LmaScoreType>(ngram.get_uni_psb(lpi_items[lpi_num].id)); 3437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_num++; 3447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lpi_num >= lpi_max) 3457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 3467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lpi_num; 3497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid DictTrie::reset_milestones(uint16 from_step, MileStoneHandle from_handle) { 3527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == from_step) { 3537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_ = 0; 3547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_pos_ = kFirstValidMileStoneHandle; 3557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 3567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (from_handle > 0 && from_handle < mile_stones_pos_) { 3577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_pos_ = from_handle; 3587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStone *mile_stone = mile_stones_ + from_handle; 3607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_ = mile_stone->mark_start; 3617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectMileStoneHandle DictTrie::extend_dict(MileStoneHandle from_handle, 3667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const DictExtPara *dep, 3677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, size_t lpi_max, 3687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t *lpi_num) { 3697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == dep) 3707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 3717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // from LmaNodeLE0 (root) to LmaNodeLE0 3737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == from_handle) { 3747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(0 == dep->splids_extended); 3757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return extend_dict0(from_handle, dep, lpi_items, lpi_max, lpi_num); 3767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 3777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // from LmaNodeLE0 to LmaNodeGE1 3797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (1 == dep->splids_extended) 3807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return extend_dict1(from_handle, dep, lpi_items, lpi_max, lpi_num); 3817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // From LmaNodeGE1 to LmaNodeGE1 3837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return extend_dict2(from_handle, dep, lpi_items, lpi_max, lpi_num); 3847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 3857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectMileStoneHandle DictTrie::extend_dict0(MileStoneHandle from_handle, 3877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const DictExtPara *dep, 3887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, 3897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_max, size_t *lpi_num) { 3907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(NULL != dep && 0 == from_handle); 3917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *lpi_num = 0; 3927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStoneHandle ret_handle = 0; 3937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 splid = dep->splids[dep->splids_extended]; 3957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_start = dep->id_start; 3967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_num = dep->id_num; 3977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 3987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LpiCache& lpi_cache = LpiCache::get_instance(); 3997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool cached = lpi_cache.is_cached(splid); 4007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 2. Begin exgtending 4027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 2.1 Get the LmaPsbItem list 4037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node = root_; 4047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t son_start = splid_le0_index_[id_start - kFullSplIdStart]; 4057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t son_end = splid_le0_index_[id_start + id_num - kFullSplIdStart]; 4067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = son_start; son_pos < son_end; son_pos++) { 4077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(1 == node->son_1st_off); 4087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *son = root_ + son_pos; 4097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(son->spl_idx >= id_start && son->spl_idx < id_start + id_num); 4107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (!cached && *lpi_num < lpi_max) { 4127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project bool need_lpi = true; 4137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_trie_->is_half_id_yunmu(splid) && son_pos != son_start) 4147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project need_lpi = false; 4157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (need_lpi) 4177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *lpi_num += fill_lpi_buffer(lpi_items + (*lpi_num), 4187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_max - *lpi_num, son); 4197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If necessary, fill in a new mile stone. 4227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx == id_start) { 4237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (mile_stones_pos_ < kMaxMileStone && 4247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_ < kMaxParsingMark) { 4257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_offset = son_pos; 4267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_num = id_num; 4277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_start = parsing_marks_pos_; 4287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_num = 1; 4297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_handle = mile_stones_pos_; 4307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_++; 4317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_pos_++; 4327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx >= id_start + id_num -1) 4367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 4377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // printf("----- parsing marks: %d, mile stone: %d \n", parsing_marks_pos_, 4407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // mile_stones_pos_); 4417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret_handle; 4427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 4437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectMileStoneHandle DictTrie::extend_dict1(MileStoneHandle from_handle, 4457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const DictExtPara *dep, 4467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, 4477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_max, size_t *lpi_num) { 4487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(NULL != dep && from_handle > 0 && from_handle < mile_stones_pos_); 4497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStoneHandle ret_handle = 0; 4517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 1. If this is a half Id, get its corresponding full starting Id and 4537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // number of full Id. 4547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t ret_val = 0; 4557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_start = dep->id_start; 4577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_num = dep->id_num; 4587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 2. Begin extending. 4607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStone *mile_stone = mile_stones_ + from_handle; 4617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 h_pos = 0; h_pos < mile_stone->mark_num; h_pos++) { 4637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ParsingMark p_mark = parsing_marks_[mile_stone->mark_start + h_pos]; 4647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 ext_num = p_mark.node_num; 4657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 ext_pos = 0; ext_pos < ext_num; ext_pos++) { 4667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node = root_ + p_mark.node_offset + ext_pos; 4677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t found_start = 0; 4687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t found_num = 0; 4697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = 0; son_pos < (size_t)node->num_of_son; son_pos++) { 4707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node->son_1st_off <= lma_node_num_ge1_); 4717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *son = nodes_ge1_ + node->son_1st_off + son_pos; 4727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx >= id_start 4737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && son->spl_idx < id_start + id_num) { 4747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (*lpi_num < lpi_max) { 4757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t homo_buf_off = get_homo_idx_buf_offset(son); 4767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *lpi_num += fill_lpi_buffer(lpi_items + (*lpi_num), 4777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_max - *lpi_num, homo_buf_off, son, 4787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 2); 4797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 4817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If necessary, fill in the new DTMI 4827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == found_num) { 4837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project found_start = son_pos; 4847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project found_num++; 4867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 4877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx >= id_start + id_num - 1 || son_pos == 4887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (size_t)node->num_of_son - 1) { 4897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (found_num > 0) { 4907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (mile_stones_pos_ < kMaxMileStone && 4917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_ < kMaxParsingMark) { 4927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_offset = 4937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node->son_1st_off + found_start; 4947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_num = found_num; 4957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == ret_val) 4967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_start = 4977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_; 4987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_++; 4997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_val++; 5027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 5047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for son_pos 5057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for ext_pos 5067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for h_pos 5077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ret_val > 0) { 5107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_num = ret_val; 5117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_handle = mile_stones_pos_; 5127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_pos_++; 5137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_val = 1; 5147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // printf("----- parsing marks: %d, mile stone: %d \n", parsing_marks_pos_, 5177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // mile_stones_pos_); 5187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret_handle; 5197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 5207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectMileStoneHandle DictTrie::extend_dict2(MileStoneHandle from_handle, 5227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project const DictExtPara *dep, 5237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem *lpi_items, 5247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lpi_max, size_t *lpi_num) { 5257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(NULL != dep && from_handle > 0 && from_handle < mile_stones_pos_); 5267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStoneHandle ret_handle = 0; 5287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 1. If this is a half Id, get its corresponding full starting Id and 5307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // number of full Id. 5317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t ret_val = 0; 5327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_start = dep->id_start; 5347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_num = dep->id_num; 5357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // 2. Begin extending. 5377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project MileStone *mile_stone = mile_stones_ + from_handle; 5387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 h_pos = 0; h_pos < mile_stone->mark_num; h_pos++) { 5407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ParsingMark p_mark = parsing_marks_[mile_stone->mark_start + h_pos]; 5417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 ext_num = p_mark.node_num; 5427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 ext_pos = 0; ext_pos < ext_num; ext_pos++) { 5437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node = nodes_ge1_ + p_mark.node_offset + ext_pos; 5447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t found_start = 0; 5457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t found_num = 0; 5467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = 0; son_pos < (size_t)node->num_of_son; son_pos++) { 5487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node->son_1st_off_l > 0 || node->son_1st_off_h > 0); 5497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *son = nodes_ge1_ + get_son_offset(node) + son_pos; 5507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx >= id_start 5517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && son->spl_idx < id_start + id_num) { 5527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (*lpi_num < lpi_max) { 5537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t homo_buf_off = get_homo_idx_buf_offset(son); 5547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project *lpi_num += fill_lpi_buffer(lpi_items + (*lpi_num), 5557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lpi_max - *lpi_num, homo_buf_off, son, 5567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project dep->splids_extended + 1); 5577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If necessary, fill in the new DTMI 5607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == found_num) { 5617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project found_start = son_pos; 5627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project found_num++; 5647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son->spl_idx >= id_start + id_num - 1 || son_pos == 5667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project (size_t)node->num_of_son - 1) { 5677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (found_num > 0) { 5687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (mile_stones_pos_ < kMaxMileStone && 5697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_ < kMaxParsingMark) { 5707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_offset = 5717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project get_son_offset(node) + found_start; 5727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_[parsing_marks_pos_].node_num = found_num; 5737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == ret_val) 5747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_start = 5757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_; 5767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project parsing_marks_pos_++; 5777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_val++; 5807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 5827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for son_pos 5847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for ext_pos 5857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // for h_pos 5867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (ret_val > 0) { 5887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_[mile_stones_pos_].mark_num = ret_val; 5897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ret_handle = mile_stones_pos_; 5907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mile_stones_pos_++; 5917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 5927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // printf("----- parsing marks: %d, mile stone: %d \n", parsing_marks_pos_, 5947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // mile_stones_pos_); 5957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return ret_handle; 5967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 5977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 5987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectbool DictTrie::try_extend(const uint16 *splids, uint16 splid_num, 5997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id_lemma) { 6007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == splid_num || NULL == splids) 6017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 6027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project void *node = root_ + splid_le0_index_[splids[0] - kFullSplIdStart]; 6047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 pos = 1; pos < splid_num; pos++) { 6067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (1 == pos) { 6077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node_le0 = reinterpret_cast<LmaNodeLE0*>(node); 6087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node_son; 6097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 son_pos; 6107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (son_pos = 0; son_pos < static_cast<uint16>(node_le0->num_of_son); 6117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project son_pos++) { 6127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node_le0->son_1st_off <= lma_node_num_ge1_); 6137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_son = nodes_ge1_ + node_le0->son_1st_off 6147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + son_pos; 6157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx == splids[pos]) 6167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 6177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son_pos < node_le0->num_of_son) 6197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node = reinterpret_cast<void*>(node_son); 6207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else 6217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 6227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 6237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node_ge1 = reinterpret_cast<LmaNodeGE1*>(node); 6247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node_son; 6257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 son_pos; 6267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (son_pos = 0; son_pos < static_cast<uint16>(node_ge1->num_of_son); 6277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project son_pos++) { 6287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node_ge1->son_1st_off_l > 0 || node_ge1->son_1st_off_h > 0); 6297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_son = nodes_ge1_ + get_son_offset(node_ge1) + son_pos; 6307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx == splids[pos]) 6317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 6327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (son_pos < node_ge1->num_of_son) 6347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node = reinterpret_cast<void*>(node_son); 6357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project else 6367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 6377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (1 == splid_num) { 6417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0* node_le0 = reinterpret_cast<LmaNodeLE0*>(node); 6427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t num_of_homo = (size_t)node_le0->num_of_homo; 6437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo_pos = 0; homo_pos < num_of_homo; homo_pos++) { 6447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType id_this = get_lemma_id(node_le0->homo_idx_buf_off + homo_pos); 6457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 str[2]; 6467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project get_lemma_str(id_this, str, 2); 6477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (id_this == id_lemma) 6487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 6497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 6517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1* node_ge1 = reinterpret_cast<LmaNodeGE1*>(node); 6527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t num_of_homo = (size_t)node_ge1->num_of_homo; 6537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo_pos = 0; homo_pos < num_of_homo; homo_pos++) { 6547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t node_homo_off = get_homo_idx_buf_offset(node_ge1); 6557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (get_lemma_id(node_homo_off + homo_pos) == id_lemma) 6567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return true; 6577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return false; 6617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 6627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t DictTrie::get_lpis(const uint16* splid_str, uint16 splid_str_len, 6647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaPsbItem* lma_buf, size_t max_lma_buf) { 6657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (splid_str_len > kMaxLemmaSize) 6667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project#define MAX_EXTENDBUF_LEN 200 6697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t* node_buf1[MAX_EXTENDBUF_LEN]; // use size_t for data alignment 6717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t* node_buf2[MAX_EXTENDBUF_LEN]; 6727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0** node_fr_le0 = 6737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reinterpret_cast<LmaNodeLE0**>(node_buf1); // Nodes from. 6747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0** node_to_le0 = 6757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project reinterpret_cast<LmaNodeLE0**>(node_buf2); // Nodes to. 6767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1** node_fr_ge1 = NULL; 6777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1** node_to_ge1 = NULL; 6787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t node_fr_num = 1; 6797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t node_to_num = 0; 6807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_le0[0] = root_; 6817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == node_fr_le0[0]) 6827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 6837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t spl_pos = 0; 6857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (spl_pos < splid_str_len) { 6877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_num = 1; 6887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 id_start = splid_str[spl_pos]; 6897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If it is a half id 6907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_trie_->is_half_id(splid_str[spl_pos])) { 6917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project id_num = spl_trie_->half_to_full(splid_str[spl_pos], &id_start); 6927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(id_num > 0); 6937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 6947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 6957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Extend the nodes 6967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == spl_pos) { // From LmaNodeLE0 (root) to LmaNodeLE0 nodes 6977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t node_fr_pos = 0; node_fr_pos < node_fr_num; node_fr_pos++) { 6987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node = node_fr_le0[node_fr_pos]; 6997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node == root_ && 1 == node_fr_num); 7007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t son_start = splid_le0_index_[id_start - kFullSplIdStart]; 7017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t son_end = 7027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splid_le0_index_[id_start + id_num - kFullSplIdStart]; 7037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = son_start; son_pos < son_end; son_pos++) { 7047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(1 == node->son_1st_off); 7057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node_son = root_ + son_pos; 7067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node_son->spl_idx >= id_start 7077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && node_son->spl_idx < id_start + id_num); 7087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_to_num < MAX_EXTENDBUF_LEN) { 7097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_le0[node_to_num] = node_son; 7107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_num++; 7117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // id_start + id_num - 1 is the last one, which has just been 7137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // recorded. 7147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx >= id_start + id_num - 1) 7157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_pos++; 7207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_pos >= splid_str_len || node_to_num == 0) 7217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Prepare the nodes for next extending 7237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // next time, from LmaNodeLE0 to LmaNodeGE1 7247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0** node_tmp = node_fr_le0; 7257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_le0 = node_to_le0; 7267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_le0 = NULL; 7277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_ge1 = reinterpret_cast<LmaNodeGE1**>(node_tmp); 7287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else if (1 == spl_pos) { // From LmaNodeLE0 to LmaNodeGE1 nodes 7297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t node_fr_pos = 0; node_fr_pos < node_fr_num; node_fr_pos++) { 7307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0 *node = node_fr_le0[node_fr_pos]; 7317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = 0; son_pos < (size_t)node->num_of_son; 7327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project son_pos++) { 7337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node->son_1st_off <= lma_node_num_ge1_); 7347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node_son = nodes_ge1_ + node->son_1st_off 7357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + son_pos; 7367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx >= id_start 7377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && node_son->spl_idx < id_start + id_num) { 7387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_to_num < MAX_EXTENDBUF_LEN) { 7397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_ge1[node_to_num] = node_son; 7407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_num++; 7417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // id_start + id_num - 1 is the last one, which has just been 7447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // recorded. 7457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx >= id_start + id_num - 1) 7467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_pos++; 7517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_pos >= splid_str_len || node_to_num == 0) 7527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Prepare the nodes for next extending 7547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // next time, from LmaNodeGE1 to LmaNodeGE1 7557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_ge1 = node_to_ge1; 7567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_ge1 = reinterpret_cast<LmaNodeGE1**>(node_fr_le0); 7577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_le0 = NULL; 7587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_le0 = NULL; 7597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { // From LmaNodeGE1 to LmaNodeGE1 nodes 7607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t node_fr_pos = 0; node_fr_pos < node_fr_num; node_fr_pos++) { 7617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node = node_fr_ge1[node_fr_pos]; 7627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t son_pos = 0; son_pos < (size_t)node->num_of_son; 7637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project son_pos++) { 7647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(node->son_1st_off_l > 0 || node->son_1st_off_h > 0); 7657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 *node_son = nodes_ge1_ 7667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project + get_son_offset(node) + son_pos; 7677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx >= id_start 7687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project && node_son->spl_idx < id_start + id_num) { 7697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_to_num < MAX_EXTENDBUF_LEN) { 7707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_ge1[node_to_num] = node_son; 7717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_num++; 7727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // id_start + id_num - 1 is the last one, which has just been 7757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // recorded. 7767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (node_son->spl_idx >= id_start + id_num - 1) 7777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_pos++; 7827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_pos >= splid_str_len || node_to_num == 0) 7837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 7847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // Prepare the nodes for next extending 7857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // next time, from LmaNodeGE1 to LmaNodeGE1 7867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1 **node_tmp = node_fr_ge1; 7877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_ge1 = node_to_ge1; 7887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_ge1 = node_tmp; 7897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 7907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // The number of node for next extending 7927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_fr_num = node_to_num; 7937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_num = 0; 7947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } // while 7957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (0 == node_to_num) 7977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 7987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 7997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram &ngram = NGram::get_instance(); 8007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t lma_num = 0; 8017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // If the length is 1, and the splid is a one-char Yunmu like 'a', 'o', 'e', 8037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project // only those candidates for the full matched one-char id will be returned. 8047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (1 == splid_str_len && spl_trie_->is_half_id_yunmu(splid_str[0])) 8057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project node_to_num = node_to_num > 0 ? 1 : 0; 8067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t node_pos = 0; node_pos < node_to_num; node_pos++) { 8087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t num_of_homo = 0; 8097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (spl_pos <= 1) { // Get from LmaNodeLE0 nodes 8107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeLE0* node_le0 = node_to_le0[node_pos]; 8117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project num_of_homo = (size_t)node_le0->num_of_homo; 8127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo_pos = 0; homo_pos < num_of_homo; homo_pos++) { 8137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t ch_pos = lma_num + homo_pos; 8147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].id = 8157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project get_lemma_id(node_le0->homo_idx_buf_off + homo_pos); 8167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].lma_len = 1; 8177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].psb = 8187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static_cast<LmaScoreType>(ngram.get_uni_psb(lma_buf[ch_pos].id)); 8197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lma_num + homo_pos >= max_lma_buf - 1) 8217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 8227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { // Get from LmaNodeGE1 nodes 8247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LmaNodeGE1* node_ge1 = node_to_ge1[node_pos]; 8257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project num_of_homo = (size_t)node_ge1->num_of_homo; 8267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (size_t homo_pos = 0; homo_pos < num_of_homo; homo_pos++) { 8277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t ch_pos = lma_num + homo_pos; 8287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t node_homo_off = get_homo_idx_buf_offset(node_ge1); 8297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].id = get_lemma_id(node_homo_off + homo_pos); 8307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].lma_len = splid_str_len; 8317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_buf[ch_pos].psb = 8327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project static_cast<LmaScoreType>(ngram.get_uni_psb(lma_buf[ch_pos].id)); 8337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lma_num + homo_pos >= max_lma_buf - 1) 8357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 8367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_num += num_of_homo; 8407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (lma_num >= max_lma_buf) { 8417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project lma_num = max_lma_buf; 8427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project break; 8437898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8447898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8457898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lma_num; 8467898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8477898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8487898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectuint16 DictTrie::get_lemma_str(LemmaIdType id_lemma, char16 *str_buf, 8497898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 str_max) { 8507898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_list_->get_lemma_str(id_lemma, str_buf, str_max); 8517898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8527898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8537898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectuint16 DictTrie::get_lemma_splids(LemmaIdType id_lemma, uint16 *splids, 8547898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 splids_max, bool arg_valid) { 8557898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project char16 lma_str[kMaxLemmaSize + 1]; 8567898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 lma_len = get_lemma_str(id_lemma, lma_str, kMaxLemmaSize + 1); 8577898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert((!arg_valid && splids_max >= lma_len) || lma_len == splids_max); 8587898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8597898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 spl_mtrx[kMaxLemmaSize * 5]; 8607898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 spl_start[kMaxLemmaSize + 1]; 8617898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_start[0] = 0; 8627898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 try_num = 1; 8637898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8647898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 pos = 0; pos < lma_len; pos++) { 8657898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 cand_splids_this = 0; 8667898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (arg_valid && spl_trie_->is_full_id(splids[pos])) { 8677898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_mtrx[spl_start[pos]] = splids[pos]; 8687898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cand_splids_this = 1; 8697898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } else { 8707898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project cand_splids_this = dict_list_->get_splids_for_hanzi(lma_str[pos], 8717898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project arg_valid ? splids[pos] : 0, spl_mtrx + spl_start[pos], 8727898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project kMaxLemmaSize * 5 - spl_start[pos]); 8737898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project assert(cand_splids_this > 0); 8747898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8757898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project spl_start[pos + 1] = spl_start[pos] + cand_splids_this; 8767898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project try_num *= cand_splids_this; 8777898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8787898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8797898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 try_pos = 0; try_pos < try_num; try_pos++) { 8807898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 mod = 1; 8817898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project for (uint16 pos = 0; pos < lma_len; pos++) { 8827898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project uint16 radix = spl_start[pos + 1] - spl_start[pos]; 8837898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project splids[pos] = spl_mtrx[ spl_start[pos] + try_pos / mod % radix]; 8847898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project mod *= radix; 8857898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8867898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8877898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (try_extend(splids, lma_len, id_lemma)) 8887898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return lma_len; 8897898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 8907898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8917898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 8927898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8937898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8947898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid DictTrie::set_total_lemma_count_of_others(size_t count) { 8957898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram& ngram = NGram::get_instance(); 8967898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project ngram.set_total_freq_none_sys(count); 8977898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 8987898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 8997898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid DictTrie::convert_to_hanzis(char16 *str, uint16 str_len) { 9007898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_list_->convert_to_hanzis(str, str_len); 9017898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9027898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9037898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectvoid DictTrie::convert_to_scis_ids(char16 *str, uint16 str_len) { 9047898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_list_->convert_to_scis_ids(str, str_len); 9057898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9067898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9077898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source ProjectLemmaIdType DictTrie::get_lemma_id(const char16 lemma_str[], uint16 lemma_len) { 9087898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (NULL == lemma_str || lemma_len > kMaxLemmaSize) 9097898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return 0; 9107898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9117898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_list_->get_lemma_id(lemma_str, lemma_len); 9127898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9137898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9147898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t DictTrie::predict_top_lmas(size_t his_len, NPredictItem *npre_items, 9157898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t npre_max, size_t b4_used) { 9167898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NGram &ngram = NGram::get_instance(); 9177898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9187898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t item_num = 0; 9197898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t top_lmas_id_offset = lma_idx_buf_len_ / kLemmaIdSize - top_lmas_num_; 9207898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t top_lmas_pos = 0; 9217898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project while (item_num < npre_max && top_lmas_pos < top_lmas_num_) { 9227898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project memset(npre_items + item_num, 0, sizeof(NPredictItem)); 9237898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project LemmaIdType top_lma_id = get_lemma_id(top_lmas_id_offset + top_lmas_pos); 9247898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project top_lmas_pos += 1; 9257898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project if (dict_list_->get_lemma_str(top_lma_id, 9267898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[item_num].pre_hzs, 9277898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project kMaxLemmaSize - 1) == 0) { 9287898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project continue; 9297898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9307898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[item_num].psb = ngram.get_uni_psb(top_lma_id); 9317898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project npre_items[item_num].his_len = his_len; 9327898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project item_num++; 9337898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project } 9347898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return item_num; 9357898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9367898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project 9377898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Projectsize_t DictTrie::predict(const char16 *last_hzs, uint16 hzs_len, 9387898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project NPredictItem *npre_items, size_t npre_max, 9397898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project size_t b4_used) { 9407898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project return dict_list_->predict(last_hzs, hzs_len, npre_items, npre_max, b4_used); 9417898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} 9427898d76cc005bbe1c5893a9f57439561e0771ccThe Android Open Source Project} // namespace ime_pinyin 943