16bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* 26bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Copyright (C) 2013 The Android Open Source Project 36bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 46bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Licensed under the Apache License, Version 2.0 (the "License"); 56bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * you may not use this file except in compliance with the License. 66bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * You may obtain a copy of the License at 76bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 86bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * http://www.apache.org/licenses/LICENSE-2.0 96bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Unless required by applicable law or agreed to in writing, software 116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * distributed under the License is distributed on an "AS IS" BASIS, 126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * See the License for the specific language governing permissions and 146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * limitations under the License. 156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */ 166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* 186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * !!!!! DO NOT EDIT THIS FILE !!!!! 196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * 206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * This file was generated from 216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp 226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */ 236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 2407e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" 256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <cerrno> 276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <cstring> 286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <sys/stat.h> 296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <sys/types.h> 306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" 326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/file_utils.h" 33c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi#include "utils/byte_array_view.h" 346bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 356bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace latinime { 366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace backward { 3707e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanaginamespace v402 { 386bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( 406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer, 416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const FormatUtils::FORMAT_VERSION formatVersion) { 426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!headerBuffer) { 436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ASSERT(false); 446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("The header buffer must be valid to open ver4 dict buffers."); 456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return Ver4DictBuffersPtr(nullptr); 466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // TODO: take only dictDirPath, and open both header and trie files in the constructor below 486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const bool isUpdatable = headerBuffer->isUpdatable(); 496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable, 506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi formatVersion)); 516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, 546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const BufferWithExtendableBuffer *const headerBuffer) const { 556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Create temporary directory. 566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int tmpDirPathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictDirPath, 576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); 586bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi char tmpDirPath[tmpDirPathBufSize]; 596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi FileUtils::getFilePathWithSuffix(dictDirPath, 606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize, 616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi tmpDirPath); 626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (FileUtils::existsDir(tmpDirPath)) { 636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!FileUtils::removeDirAndFiles(tmpDirPath)) { 646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Existing directory %s cannot be removed.", tmpDirPath); 656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ASSERT(false); 666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 6907e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi umask(S_IWGRP | S_IWOTH); 706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (mkdir(tmpDirPath, S_IRWXU) == -1) { 716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); 726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Get dictionary base path. 756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */; 766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi char dictName[dictNameBufSize]; 776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName); 786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const int dictPathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName); 796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi char dictPath[dictPathBufSize]; 806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi FileUtils::getFilePath(tmpDirPath, dictName, dictPathBufSize, dictPath); 816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write header file. 836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, 846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) { 856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath, 866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::HEADER_FILE_EXTENSION); 876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 896bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write trie file. 906bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, 916bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { 926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath, 936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::TRIE_FILE_EXTENSION); 946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Write dictionary contents. 976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mTerminalPositionLookupTable.flushToFile(dictPath)) { 986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); 996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1006bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mProbabilityDictContent.flushToFile(dictPath)) { 1026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Probability dict content cannot be written. %s", tmpDirPath); 1036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mBigramDictContent.flushToFile(dictPath)) { 1066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath); 1076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!mShortcutDictContent.flushToFile(dictPath)) { 1106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath); 1116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Remove existing dictionary. 1146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (!FileUtils::removeDirAndFiles(dictDirPath)) { 1156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("Existing directory %s cannot be removed.", dictDirPath); 1166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ASSERT(false); 1176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi // Rename temporary directory. 1206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi if (rename(tmpDirPath, dictDirPath) != 0) { 1216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi AKLOGE("%s cannot be renamed to %s", tmpDirPath, dictDirPath); 1226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi ASSERT(false); 1236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return false; 1246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi } 1256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi return true; 1266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} 1276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke KuroyanagiVer4DictBuffers::Ver4DictBuffers(const char *const dictPath, 1296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable, 1306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi const FormatUtils::FORMAT_VERSION formatVersion) 1316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi : mHeaderBuffer(std::move(headerBuffer)), 1326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mDictBuffer(MmappedBuffer::openBuffer(dictPath, 1336bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), 134c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion), 135c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(), 1366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), 137c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi mExpandableTrieBuffer( 138c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() : 139c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi ReadWriteByteArrayView(), 1406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), 1416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mTerminalPositionLookupTable(dictPath, isUpdatable), 1426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), 1436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), 1446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mShortcutDictContent(dictPath, isUpdatable), 1456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mIsUpdatable(isUpdatable) {} 1466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 1476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke KuroyanagiVer4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize) 1486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi : mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(headerPolicy), 1496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), 1506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(), 1516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()), 1526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), 1536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi mIsUpdatable(true) {} 1546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi 15507e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi} // namespace v402 1566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace backward 1576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace latinime 158