dynamic_patricia_trie_policy.cpp revision 699531099630edd8416e309c914187c285af4c44
126de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi/* 226de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * Copyright (C) 2013, The Android Open Source Project 326de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * 426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * Licensed under the Apache License, Version 2.0 (the "License"); 526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * you may not use this file except in compliance with the License. 626de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * You may obtain a copy of the License at 726de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * 826de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * http://www.apache.org/licenses/LICENSE-2.0 926de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * 1026de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * Unless required by applicable law or agreed to in writing, software 1126de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * distributed under the License is distributed on an "AS IS" BASIS, 1226de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1326de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * See the License for the specific language governing permissions and 1426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi * limitations under the License. 1526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi */ 1626de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 179601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h" 1826de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 1931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi#include <cstdio> 2031097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi#include <cstring> 21fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi#include <ctime> 2231097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi 2326de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi#include "defines.h" 2426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi#include "suggest/core/dicnode/dic_node.h" 2526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi#include "suggest/core/dicnode/dic_node_vector.h" 262b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h" 274d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h" 282b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" 29f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" 302b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" 31fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" 3265d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/probability_utils.h" 3326de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 3426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynaginamespace latinime { 3526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 36699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi// Note that these are corresponding definitions in Java side in BinaryDictionaryTests and 37699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi// BinaryDictionaryDecayingTests. 3831097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagiconst char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 3931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagiconst char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 40699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagiconst char *const DynamicPatriciaTriePolicy::SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY = 41699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi "SET_NEEDS_TO_DECAY_FOR_TESTING"; 42fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagiconst int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024; 43fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagiconst int DynamicPatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = 44fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - 1024; 45699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagiconst int DynamicPatriciaTriePolicy::DECAY_INTERVAL_FOR_DECAYING_DICTS = 2 * 60 * 60; 4631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi 4726de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagivoid DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, 487fd9667d76cdc6febe32545865648ea90dc28904Keisuke Kuroyanagi DicNodeVector *const childDicNodes) const { 492b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi if (!dicNode->hasChildren()) { 502b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return; 512b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi } 523e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, 533e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 5477ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPos()); 554d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); 564d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi while (!readingHelper.isEnd()) { 578c69ddb53e05cf2740137a09dc139aed7a9831a5Keisuke Kuroyanagi childDicNodes->pushLeavingChild(dicNode, nodeReader->getHeadPos(), 584d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi nodeReader->getChildrenPos(), nodeReader->getProbability(), 594d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi nodeReader->isTerminal() && !nodeReader->isDeleted(), 604d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi nodeReader->hasChildren(), nodeReader->isBlacklisted() || nodeReader->isNotAWord(), 614d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi nodeReader->getCodePointCount(), readingHelper.getMergedNodeCodePoints()); 624d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.readNextSiblingNode(); 634d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi } 6426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 6526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 6626de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagiint DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( 6777ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, 6826de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi int *const outUnigramProbability) const { 699601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi // This method traverses parent nodes from the terminal by following parent pointers; thus, 709601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi // node code points are stored in the buffer in the reverse order. 719601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi int reverseCodePoints[maxCodePointCount]; 723e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, 733e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 744d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // First, read the terminal node and get its probability. 7577ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi readingHelper.initWithPtNodePos(ptNodePos); 764d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (!readingHelper.isValidTerminalNode()) { 7777ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi // Node at the ptNodePos is not a valid terminal node. 784d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi *outUnigramProbability = NOT_A_PROBABILITY; 794d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi return 0; 809601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi } 814d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Store terminal node probability. 824d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi *outUnigramProbability = readingHelper.getNodeReader()->getProbability(); 834d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Then, following parent node link to the dictionary root and fetch node code points. 844d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi while (!readingHelper.isEnd()) { 854d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (readingHelper.getTotalCodePointCount() > maxCodePointCount) { 8677ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi // The ptNodePos is not a valid terminal node position in the dictionary. 879601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi *outUnigramProbability = NOT_A_PROBABILITY; 889601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi return 0; 899601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi } 909601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi // Store node code points to buffer in the reverse order. 914d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.fetchMergedNodeCodePointsInReverseOrder( 924d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.getPrevTotalCodePointCount(), reverseCodePoints); 934d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Follow parent node toward the root node. 944d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.readParentNode(); 954d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi } 964d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (readingHelper.isError()) { 974d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // The node position or the dictionary is invalid. 984d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi *outUnigramProbability = NOT_A_PROBABILITY; 994d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi return 0; 1009601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi } 1019601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi // Reverse the stored code points to output them. 1024d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi const int codePointCount = readingHelper.getTotalCodePointCount(); 1039601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi for (int i = 0; i < codePointCount; ++i) { 1049601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi outCodePoints[i] = reverseCodePoints[codePointCount - i - 1]; 1059601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi } 1069601df5aac492b12ea0912ba6da8ab3d11e1f5d7Keisuke Kuroynagi return codePointCount; 10726de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 10826de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 109e1ebef6124241ef51d5ed17884e6299a330d496bKeisuke Kuroyanagiint DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, 11026de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi const int length, const bool forceLowerCaseSearch) const { 111744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi int searchCodePoints[length]; 112744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi for (int i = 0; i < length; ++i) { 113744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi searchCodePoints[i] = forceLowerCaseSearch ? CharUtils::toLowerCase(inWord[i]) : inWord[i]; 114744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 1153e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, 1163e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 11777ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi readingHelper.initWithPtNodeArrayPos(getRootPosition()); 1184d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); 1194d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi while (!readingHelper.isEnd()) { 1204d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi const int matchedCodePointCount = readingHelper.getPrevTotalCodePointCount(); 1214d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (readingHelper.getTotalCodePointCount() > length 1224d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi || !readingHelper.isMatchedCodePoint(0 /* index */, 1234d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi searchCodePoints[matchedCodePointCount])) { 1244d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Current node has too many code points or its first code point is different from 1254d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // target code point. Skip this node and read the next sibling node. 1264d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.readNextSiblingNode(); 1274d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi continue; 1284d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi } 1294d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Check following merged node code points. 1304d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi const int nodeCodePointCount = nodeReader->getCodePointCount(); 1314d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi for (int j = 1; j < nodeCodePointCount; ++j) { 1324d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (!readingHelper.isMatchedCodePoint( 1334d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi j, searchCodePoints[matchedCodePointCount + j])) { 1344d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Different code point is found. The given word is not included in the dictionary. 135cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi return NOT_A_DICT_POS; 136744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 1374d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi } 1384d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // All characters are matched. 1394d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (length == readingHelper.getTotalCodePointCount()) { 1404d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Terminal position is found. 1418c69ddb53e05cf2740137a09dc139aed7a9831a5Keisuke Kuroyanagi return nodeReader->getHeadPos(); 1424d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi } 1434d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi if (!nodeReader->hasChildren()) { 144cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi return NOT_A_DICT_POS; 145744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 1464d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi // Advance to the children nodes. 1474d814bfcb76c6a7637aed0046079251dfdc08095Keisuke Kuroyanagi readingHelper.readChildNode(); 148744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 149744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi // If we already traversed the tree further than the word is long, there means 150744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi // there was no match (or we would have found it). 151cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi return NOT_A_DICT_POS; 15226de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 15326de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 15465d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagiint DynamicPatriciaTriePolicy::getProbability(const int unigramProbability, 15565d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi const int bigramProbability) const { 156fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mHeaderPolicy.isDecayingDict()) { 157fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return DecayingUtils::getProbability(unigramProbability, bigramProbability); 15865d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi } else { 159fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (unigramProbability == NOT_A_PROBABILITY) { 160fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return NOT_A_PROBABILITY; 161fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else if (bigramProbability == NOT_A_PROBABILITY) { 162fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return ProbabilityUtils::backoff(unigramProbability); 163fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else { 164fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, 165fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi bigramProbability); 166fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 16765d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi } 16865d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi} 16965d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi 17077ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagiint DynamicPatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int ptNodePos) const { 17177ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi if (ptNodePos == NOT_A_DICT_POS) { 172744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi return NOT_A_PROBABILITY; 173744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 1743e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, 1756c4d09e9e12d02aa87b27def6529220c93ff4588Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 17677ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi nodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos); 1772b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi if (nodeReader.isDeleted() || nodeReader.isBlacklisted() || nodeReader.isNotAWord()) { 1782b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return NOT_A_PROBABILITY; 1792b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi } 18065d19946bebd1cc6299e2789cc0fc097d1898e80Keisuke Kuroyanagi return getProbability(nodeReader.getProbability(), NOT_A_PROBABILITY); 18126de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 18226de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 18377ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagiint DynamicPatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) const { 18477ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi if (ptNodePos == NOT_A_DICT_POS) { 185744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi return NOT_A_DICT_POS; 186744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 1873e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, 1886c4d09e9e12d02aa87b27def6529220c93ff4588Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 18977ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi nodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos); 1902b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi if (nodeReader.isDeleted()) { 1912b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return NOT_A_DICT_POS; 1922b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi } 1932b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return nodeReader.getShortcutPos(); 19426de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 19526de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 19677ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagiint DynamicPatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) const { 19777ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi if (ptNodePos == NOT_A_DICT_POS) { 198744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi return NOT_A_DICT_POS; 199744084defb769c4631bf0d4f9579036fd81a8af1Keisuke Kuroynagi } 2003e76487c6c95ccec49622b9d7e0b45efff97f937Keisuke Kuroyanagi DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, 2016c4d09e9e12d02aa87b27def6529220c93ff4588Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 20277ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi nodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos); 2032b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi if (nodeReader.isDeleted()) { 2042b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return NOT_A_DICT_POS; 2052b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi } 2062b1dd6e2532ee041248c3f7c48f28d789713b18bKeisuke Kuroynagi return nodeReader.getBigramsPos(); 20726de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} 20826de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi 20966facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagibool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length, 21066facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi const int probability) { 2110624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 2120624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); 2130624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi return false; 2140624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi } 215fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mBufferWithExtendableBuffer.getTailPosition() 216fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { 217fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi AKLOGE("The dictionary is too large to dynamically update."); 218fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return false; 219fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 220f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, 221f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi getBigramsStructurePolicy(), getShortcutsStructurePolicy()); 22277ef75cbe6722d1eb45115c1ad82f963444d71cdKeisuke Kuroyanagi readingHelper.initWithPtNodeArrayPos(getRootPosition()); 223f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, 224fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); 22531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi bool addedNewUnigram = false; 22631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (writingHelper.addUnigramWord(&readingHelper, word, length, probability, 22731097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi &addedNewUnigram)) { 22831097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (addedNewUnigram) { 22931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi mUnigramCount++; 23031097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 23131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return true; 23231097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } else { 23331097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return false; 23431097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 23566facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi} 23666facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi 23766facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagibool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, 23866facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi const int *const word1, const int length1, const int probability) { 2390624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 2400624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); 2410624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi return false; 2420624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi } 243fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mBufferWithExtendableBuffer.getTailPosition() 244fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { 245fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi AKLOGE("The dictionary is too large to dynamically update."); 246fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return false; 247fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 248f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi const int word0Pos = getTerminalNodePositionOfWord(word0, length0, 249f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi false /* forceLowerCaseSearch */); 250cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi if (word0Pos == NOT_A_DICT_POS) { 251f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi return false; 252f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi } 253f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi const int word1Pos = getTerminalNodePositionOfWord(word1, length1, 254f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi false /* forceLowerCaseSearch */); 255cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi if (word1Pos == NOT_A_DICT_POS) { 256f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi return false; 257f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi } 258f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, 259fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); 26031097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi bool addedNewBigram = false; 26131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (writingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) { 26231097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (addedNewBigram) { 26331097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi mBigramCount++; 26431097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 26531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return true; 26631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } else { 26731097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return false; 26831097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 26966facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi} 27066facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi 27166facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagibool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0, 27266facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi const int *const word1, const int length1) { 2730624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 2740624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary."); 2750624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi return false; 2760624cc6cf3971aa3c189185208571a5f3d0c459cKeisuke Kuroyanagi } 277fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mBufferWithExtendableBuffer.getTailPosition() 278fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { 279fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi AKLOGE("The dictionary is too large to dynamically update."); 280fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return false; 281fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 282f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi const int word0Pos = getTerminalNodePositionOfWord(word0, length0, 283f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi false /* forceLowerCaseSearch */); 284cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi if (word0Pos == NOT_A_DICT_POS) { 285f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi return false; 286f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi } 287f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi const int word1Pos = getTerminalNodePositionOfWord(word1, length1, 288f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi false /* forceLowerCaseSearch */); 289cb816e5e16f086d98c8d05a0a5805c1cdfaf1c02Keisuke Kuroyanagi if (word1Pos == NOT_A_DICT_POS) { 290f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi return false; 291f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi } 292f1cd7a081c1365da389e14bd190d7e15fa402eb8Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, 293fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); 29431097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (writingHelper.removeBigramWords(word0Pos, word1Pos)) { 29531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi mBigramCount--; 29631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return true; 29731097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } else { 29831097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return false; 29931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 30066facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi} 30166facd37ddf8fc23ed2508a114c446147aaca724Keisuke Kuroyanagi 302d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagivoid DynamicPatriciaTriePolicy::flush(const char *const filePath) { 303d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 304d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi AKLOGI("Warning: flush() is called for non-updatable dictionary."); 305d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi return; 306d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 30715605437548f7187c33bc8f260f80fae4303b460Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, 308699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi &mBigramListPolicy, &mShortcutListPolicy, false /* needsToDecay */); 30931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount); 310d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi} 311d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 312d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagivoid DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { 313d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 314d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); 315d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi return; 316d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 317699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi const bool runGCwithDecay = needsToDecay(); 318699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi DynamicBigramListPolicy bigramListPolicyForGC(&mBufferWithExtendableBuffer, 319699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi &mShortcutListPolicy, runGCwithDecay); 3202cfe7f9e3b8a09aa00b18efcb82a1b3d5fed43f0Keisuke Kuroyanagi DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, 321699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi &bigramListPolicyForGC, &mShortcutListPolicy, runGCwithDecay); 3222cfe7f9e3b8a09aa00b18efcb82a1b3d5fed43f0Keisuke Kuroyanagi writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy); 323699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi if (runGCwithDecay) { 324699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi mNeedsToDecayForTesting = false; 325699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi } 326d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi} 327d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 328c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagibool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { 329d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi if (!mBuffer->isUpdatable()) { 330d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); 331d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi return false; 332d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 333fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mBufferWithExtendableBuffer.isNearSizeLimit()) { 334fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Additional buffer size is near the limit. 335fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 336fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else if (mHeaderPolicy.getExtendedRegionSize() 337fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi + mBufferWithExtendableBuffer.getUsedAdditionalBufferSize() 338fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi > MAX_DICT_EXTENDED_REGION_SIZE) { 339fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Total extended region size exceeds the limit. 340fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 341fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else if (mBufferWithExtendableBuffer.getTailPosition() 342fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS 343fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi && mBufferWithExtendableBuffer.getUsedAdditionalBufferSize() > 0) { 344fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Needs to reduce dictionary size. 345fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 346fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else if (mHeaderPolicy.isDecayingDict()) { 347fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi if (mUnigramCount >= DecayingUtils::MAX_UNIGRAM_COUNT) { 348fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Unigram count exceeds the limit. 349fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 350fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } else if (mBigramCount >= DecayingUtils::MAX_BIGRAM_COUNT) { 351fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Bigram count exceeds the limit. 352fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 353699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi } else if (mindsBlockByGC && needsToDecay()) { 354fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi // Time to update probabilities for decaying. 355fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return true; 356fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 357fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi } 358fd02b2d6ee55d4aee7faab89a7a2b72764eafc47Keisuke Kuroyanagi return false; 359d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi} 360d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 36131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagivoid DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult, 362699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi const int maxResultLength) { 36331097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) { 36431097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi snprintf(outResult, maxResultLength, "%d", mUnigramCount); 36531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) { 36631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi snprintf(outResult, maxResultLength, "%d", mBigramCount); 367699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, maxResultLength) == 0) { 368699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi mNeedsToDecayForTesting = true; 36931097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 37031097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi} 37131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi 372699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagibool DynamicPatriciaTriePolicy::needsToDecay() const { 373699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi return mHeaderPolicy.isDecayingDict() && (mNeedsToDecayForTesting 374699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi || mHeaderPolicy.getLastDecayedTime() + DECAY_INTERVAL_FOR_DECAYING_DICTS < time(0)); 375699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi} 376699531099630edd8416e309c914187c285af4c44Keisuke Kuroyanagi 37726de7079b6bbaac1636445ef730c0229bc1add98Keisuke Kuroynagi} // namespace latinime 378