16bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* 26bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Copyright (C) 2013, The Android Open Source Project 36bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 46bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Licensed under the Apache License, Version 2.0 (the "License"); 56bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * you may not use this file except in compliance with the License. 66bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * You may obtain a copy of the License at 76bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 86bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * http://www.apache.org/licenses/LICENSE-2.0 96bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Unless required by applicable law or agreed to in writing, software 116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * distributed under the License is distributed on an "AS IS" BASIS, 126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * See the License for the specific language governing permissions and 146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * limitations under the License. 156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */ 166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* 186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * !!!!! DO NOT EDIT THIS FILE !!!!! 196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * This file was generated from 2188bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi * dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp 226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */ 236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2488bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h" 256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2688bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/header/header_policy.h" 2788bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/property/unigram_property.h" 2888bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/pt_common/dynamic_pt_reading_utils.h" 2988bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/pt_common/dynamic_pt_writing_utils.h" 3088bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/pt_common/patricia_trie_reading_utils.h" 3188bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h" 3288bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/content/probability_entry.h" 3388bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h" 3488bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" 3588bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/structure/backward/v402/ver4_dict_buffers.h" 3688bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/utils/buffer_with_extendable_buffer.h" 3788bc312ad34321fb3e81be2dc939a889d065f4a7Keisuke Kuroyanagi#include "dictionary/utils/forgetting_curve_utils.h" 386bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace latinime { 406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace backward { 4107e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanaginamespace v402 { 426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagiconst int Ver4PatriciaTrieNodeWriter::CHILDREN_POSITION_FIELD_SIZE = 3; 446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted( 466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams) { 476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int pos = toBeUpdatedPtNodeParams->getHeadPos(); 486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool usesAdditionalBuffer = mTrieBuffer->isInAdditionalBuffer(pos); 496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const uint8_t *const dictBuf = mTrieBuffer->getBuffer(usesAdditionalBuffer); 506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (usesAdditionalBuffer) { 516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi pos -= mTrieBuffer->getOriginalBufferSize(); 526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Read original flags 546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags originalFlags = 556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); 566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags updatedFlags = 576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, 586bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi true /* isDeleted */, false /* willBecomeNonTerminal */); 596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); 606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update flags. 616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, 626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &writingPos)) { 636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (toBeUpdatedPtNodeParams->isTerminal()) { 666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // The PtNode is a terminal. Delete entry from the terminal position lookup table. 676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( 686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */); 696bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } else { 706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( 756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams, 766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int movedPos, const int bigramLinkedNodePos) { 776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int pos = toBeUpdatedPtNodeParams->getHeadPos(); 786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool usesAdditionalBuffer = mTrieBuffer->isInAdditionalBuffer(pos); 796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const uint8_t *const dictBuf = mTrieBuffer->getBuffer(usesAdditionalBuffer); 806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (usesAdditionalBuffer) { 816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi pos -= mTrieBuffer->getOriginalBufferSize(); 826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Read original flags 846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags originalFlags = 856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); 866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags updatedFlags = 876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, 886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi false /* isDeleted */, false /* willBecomeNonTerminal */); 896bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); 906bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update flags. 916bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, 926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &writingPos)) { 936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update moved position, which is stored in the parent offset field. 966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( 976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { 986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1006bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (toBeUpdatedPtNodeParams->hasChildren()) { 1016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update children's parent position. 1026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mReadingHelper.initWithPtNodeArrayPos(toBeUpdatedPtNodeParams->getChildrenPos()); 1036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi while (!mReadingHelper.isEnd()) { 1046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams()); 1056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int parentOffsetFieldPos = childPtNodeParams.getHeadPos() 1066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; 1076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( 1086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(), 1096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &parentOffsetFieldPos)) { 1106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Parent offset cannot be written because of a bug or a broken dictionary; thus, 1116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // we give up to update dictionary. 1126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mReadingHelper.readNextSiblingNode(childPtNodeParams); 1156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 1186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 1196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( 1216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams) { 1226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int pos = toBeUpdatedPtNodeParams->getHeadPos(); 1236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool usesAdditionalBuffer = mTrieBuffer->isInAdditionalBuffer(pos); 1246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const uint8_t *const dictBuf = mTrieBuffer->getBuffer(usesAdditionalBuffer); 1256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (usesAdditionalBuffer) { 1266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi pos -= mTrieBuffer->getOriginalBufferSize(); 1276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Read original flags 1296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags originalFlags = 1306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); 1316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PatriciaTrieReadingUtils::NodeFlags updatedFlags = 1326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, 1336bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi false /* isDeleted */, true /* willBecomeNonTerminal */); 1346bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( 1356bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) { 1366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Cannot update terminal position lookup table. terminal id: %d", 1376bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId()); 1386bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update flags. 1416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); 1426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, 1436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &writingPos); 1446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 1456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updatePtNodeUnigramProperty( 1476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams, 1486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const UnigramProperty *const unigramProperty) { 1496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update probability and historical information. 1506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // TODO: Update other information in the unigram property. 1516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!toBeUpdatedPtNodeParams->isTerminal()) { 1526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry originalProbabilityEntry = 1556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mBuffers->getProbabilityDictContent()->getProbabilityEntry( 1566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId()); 1576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry probabilityEntry = createUpdatedEntryFrom(&originalProbabilityEntry, 1586bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi unigramProperty); 1596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( 1606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry); 1616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 1626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeAfterGC( 1646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams, bool *const outNeedsToKeepPtNode) { 1656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!toBeUpdatedPtNodeParams->isTerminal()) { 1666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("updatePtNodeProbabilityAndGetNeedsToSaveForGC is called for non-terminal PtNode."); 1676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1696bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry originalProbabilityEntry = 1706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mBuffers->getProbabilityDictContent()->getProbabilityEntry( 1716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId()); 1726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (originalProbabilityEntry.hasHistoricalInfo()) { 1736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( 1746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy); 1756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry probabilityEntry = 1766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo); 1776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( 1786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry)) { 1796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Cannot write updated probability entry. terminalId: %d", 1806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId()); 1816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool isValid = ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy); 1846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!isValid) { 1856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!markPtNodeAsWillBecomeNonTerminal(toBeUpdatedPtNodeParams)) { 1866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Cannot mark PtNode as willBecomeNonTerminal."); 1876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1896bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1906bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *outNeedsToKeepPtNode = isValid; 1916bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } else { 1926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // No need to update probability. 1936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *outNeedsToKeepPtNode = true; 1946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 1966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 1976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updateChildrenPosition( 1996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) { 2006bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos(); 2016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, 2026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi newChildrenPosition, &childrenPosFieldPos); 2036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updateTerminalId(const PtNodeParams *const toBeUpdatedPtNodeParams, 2066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int newTerminalId) { 2076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return mTrieBuffer->writeUint(newTerminalId, Ver4DictConstants::TERMINAL_ID_FIELD_SIZE, 2086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalIdFieldPos()); 2096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::writePtNodeAndAdvancePosition( 2126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos) { 2136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return writePtNodeAndGetTerminalIdAndAdvancePosition(ptNodeParams, 0 /* outTerminalId */, 2146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeWritingPos); 2156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition( 2196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const ptNodeParams, const UnigramProperty *const unigramProperty, 2206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int *const ptNodeWritingPos) { 2216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; 2226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!writePtNodeAndGetTerminalIdAndAdvancePosition(ptNodeParams, &terminalId, 2236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeWritingPos)) { 2246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 2256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write probability. 2276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ProbabilityEntry newProbabilityEntry; 2286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry probabilityEntryToWrite = createUpdatedEntryFrom( 2296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &newProbabilityEntry, unigramProperty); 2306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(terminalId, 2316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &probabilityEntryToWrite); 2326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2336bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2349069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId, 23579bb37d499ed6fcabe981153d5ff0b5b69509933Keisuke Kuroyanagi const NgramProperty *const ngramProperty, bool *const outAddedNewEntry) { 23679bb37d499ed6fcabe981153d5ff0b5b69509933Keisuke Kuroyanagi if (!mBigramPolicy->addNewEntry(prevWordIds[0], wordId, ngramProperty, outAddedNewEntry)) { 2379a23f0fba25137760a60e9bfaf6bf20a5889648cKeisuke Kuroyanagi AKLOGE("Cannot add new bigram entry. prevWordId: %d, wordId: %d", 2389a23f0fba25137760a60e9bfaf6bf20a5889648cKeisuke Kuroyanagi prevWordIds[0], wordId); 2396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 2406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2419069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi const int ptNodePos = 2429069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi mBuffers->getTerminalPositionLookupTable()->getTerminalPtNodePosition(prevWordIds[0]); 2439069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi const PtNodeParams sourcePtNodeParams = 2449069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos); 2459069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi if (!sourcePtNodeParams.hasBigrams()) { 2466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update has bigrams flag. 2479069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi return updatePtNodeFlags(sourcePtNodeParams.getHeadPos(), 24805172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu sourcePtNodeParams.isPossiblyOffensive(), sourcePtNodeParams.isNotAWord(), 2499069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi sourcePtNodeParams.isTerminal(), sourcePtNodeParams.hasShortcutTargets(), 2506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi true /* hasBigrams */, 2519069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi sourcePtNodeParams.getCodePointCount() > 1 /* hasMultipleChars */); 2526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 2546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2569069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWordIds, 2579069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi const int wordId) { 2589069d30043d5182dfd38465ad9bbc11ad73fab7cKeisuke Kuroyanagi return mBigramPolicy->removeEntry(prevWordIds[0], wordId); 2596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries( 2626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) { 2636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries( 2646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi sourcePtNodeParams->getTerminalId(), outBigramEntryCount); 2656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 2666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updateAllPositionFields( 2686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const toBeUpdatedPtNodeParams, 2696bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const DictPositionRelocationMap *const dictPositionRelocationMap, 2706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int *const outBigramEntryCount) { 2716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int parentPos = toBeUpdatedPtNodeParams->getParentPos(); 2726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (parentPos != NOT_A_DICT_POS) { 2736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PtNodeWriter::PtNodePositionRelocationMap::const_iterator it = 2746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi dictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos); 2756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (it != dictPositionRelocationMap->mPtNodePositionRelocationMap.end()) { 2766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi parentPos = it->second; 2776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int writingPos = toBeUpdatedPtNodeParams->getHeadPos() 2806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; 2816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write updated parent offset. 2826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, 2836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { 2846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 2856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Updates children position. 2886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int childrenPos = toBeUpdatedPtNodeParams->getChildrenPos(); 2896bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (childrenPos != NOT_A_DICT_POS) { 2906bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PtNodeWriter::PtNodeArrayPositionRelocationMap::const_iterator it = 2916bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos); 2926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (it != dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) { 2936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi childrenPos = it->second; 2946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!updateChildrenPosition(toBeUpdatedPtNodeParams, childrenPos)) { 2976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 2986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 2996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 3006bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Counts bigram entries. 3016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (outBigramEntryCount) { 3026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *outBigramEntryCount = mBigramPolicy->getBigramEntryConut( 3036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi toBeUpdatedPtNodeParams->getTerminalId()); 3046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 3066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 3076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 3086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::addShortcutTarget(const PtNodeParams *const ptNodeParams, 3096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int *const targetCodePoints, const int targetCodePointCount, 3106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int shortcutProbability) { 3116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mShortcutPolicy->addNewShortcut(ptNodeParams->getTerminalId(), 3126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi targetCodePoints, targetCodePointCount, shortcutProbability)) { 31344efbe64b1ba44f8edf13a52eca630fce3d7c809Adrian Velicu AKLOGE("Cannot add new shortcut entry. terminalId: %d", ptNodeParams->getTerminalId()); 3146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!ptNodeParams->hasShortcutTargets()) { 3176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update has shortcut targets flag. 3186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return updatePtNodeFlags(ptNodeParams->getHeadPos(), 31905172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu ptNodeParams->isPossiblyOffensive(), ptNodeParams->isNotAWord(), 3206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->isTerminal(), true /* hasShortcutTargets */, 3216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->hasBigrams(), 3226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getCodePointCount() > 1 /* hasMultipleChars */); 3236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 3256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 3266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 3276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updatePtNodeHasBigramsAndShortcutTargetsFlags( 3286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const ptNodeParams) { 3296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool hasBigrams = mBuffers->getBigramDictContent()->getBigramListHeadPos( 3306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getTerminalId()) != NOT_A_DICT_POS; 3316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool hasShortcutTargets = mBuffers->getShortcutDictContent()->getShortcutListHeadPos( 3326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getTerminalId()) != NOT_A_DICT_POS; 33305172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu return updatePtNodeFlags(ptNodeParams->getHeadPos(), ptNodeParams->isPossiblyOffensive(), 3346bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->isNotAWord(), ptNodeParams->isTerminal(), hasShortcutTargets, 3356bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi hasBigrams, ptNodeParams->getCodePointCount() > 1 /* hasMultipleChars */); 3366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 3376bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 3386bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( 3396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const PtNodeParams *const ptNodeParams, int *const outTerminalId, 3406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int *const ptNodeWritingPos) { 3416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int nodePos = *ptNodeWritingPos; 3426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the 3436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // PtNode writing. 3446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, 3456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 0 /* nodeFlags */, ptNodeWritingPos)) { 3466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Calculate a parent offset and write the offset. 3496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, 3506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) { 3516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write code points 3546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, 3556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) { 3566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3586bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; 3596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!ptNodeParams->willBecomeNonTerminal()) { 3606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (ptNodeParams->getTerminalId() != Ver4DictConstants::NOT_A_TERMINAL_ID) { 3616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi terminalId = ptNodeParams->getTerminalId(); 3626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } else if (ptNodeParams->isTerminal()) { 3636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write terminal information using a new terminal id. 3646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Get a new unused terminal id. 3656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi terminalId = mBuffers->getTerminalPositionLookupTable()->getNextTerminalId(); 3666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int isTerminal = terminalId != Ver4DictConstants::NOT_A_TERMINAL_ID; 3696bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (isTerminal) { 3706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Update the lookup table. 3716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( 3726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi terminalId, nodePos)) { 3736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write terminal Id. 3766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mTrieBuffer->writeUintAndAdvancePosition(terminalId, 3776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::TERMINAL_ID_FIELD_SIZE, ptNodeWritingPos)) { 3786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (outTerminalId) { 3816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *outTerminalId = terminalId; 3826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 3846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write children position 3856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, 3866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getChildrenPos(), ptNodeWritingPos)) { 3876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 3886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 38905172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu return updatePtNodeFlags(nodePos, ptNodeParams->isPossiblyOffensive(), 39005172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu ptNodeParams->isNotAWord(), isTerminal, ptNodeParams->hasShortcutTargets(), 39105172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu ptNodeParams->hasBigrams(), 3926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ptNodeParams->getCodePointCount() > 1 /* hasMultipleChars */); 3936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 3946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 3956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagiconst ProbabilityEntry Ver4PatriciaTrieNodeWriter::createUpdatedEntryFrom( 3966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const ProbabilityEntry *const originalProbabilityEntry, 3976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const UnigramProperty *const unigramProperty) const { 3986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // TODO: Consolidate historical info and probability. 3996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (mHeaderPolicy->hasHistoricalInfoOfWords()) { 400287e155e44b4e937f2a62d010805702bc813c43bKeisuke Kuroyanagi const HistoricalInfo &historicalInfoForUpdate = unigramProperty->getHistoricalInfo(); 4016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const HistoricalInfo updatedHistoricalInfo = 4026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ForgettingCurveUtils::createUpdatedHistoricalInfo( 4036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi originalProbabilityEntry->getHistoricalInfo(), 4046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi unigramProperty->getProbability(), &historicalInfoForUpdate, mHeaderPolicy); 4056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return originalProbabilityEntry->createEntryWithUpdatedHistoricalInfo( 4066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi &updatedHistoricalInfo); 4076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } else { 4086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return originalProbabilityEntry->createEntryWithUpdatedProbability( 4096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi unigramProperty->getProbability()); 4106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 4116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 4126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 4136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, 4146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool isBlacklisted, const bool isNotAWord, const bool isTerminal, 4156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool hasShortcutTargets, const bool hasBigrams, const bool hasMultipleChars) { 4166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Create node flags and write them. 4176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PatriciaTrieReadingUtils::NodeFlags nodeFlags = 4186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal, 4196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi hasShortcutTargets, hasBigrams, hasMultipleChars, 4206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi CHILDREN_POSITION_FIELD_SIZE); 4216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { 4226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos); 4236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 4246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 4256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 4266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 4276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 4288890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagibool Ver4PatriciaTrieNodeWriter::suppressUnigramEntry(const PtNodeParams *const ptNodeParams) { 4298890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi if (!mHeaderPolicy->hasHistoricalInfoOfWords()) { 4308890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi // Require historical info to suppress unigram entry. 4318890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi return false; 4328890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi } 4338890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi const HistoricalInfo suppressedHistorycalInfo(0 /* timestamp */, 0 /* level */, 0 /* count */); 4348890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi const ProbabilityEntry probabilityEntryToWrite = 4358890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi ProbabilityEntry().createEntryWithUpdatedHistoricalInfo(&suppressedHistorycalInfo); 4368890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( 4378890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi ptNodeParams->getTerminalId(), &probabilityEntryToWrite); 4388890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi} 4398890b015505febfa294445ab02bb8d957efac277Keisuke Kuroyanagi 44007e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi} // namespace v402 4416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace backward 4426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace latinime 443