16bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/*
26bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Copyright (C) 2013 The Android Open Source Project
36bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *
46bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Licensed under the Apache License, Version 2.0 (the "License");
56bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * you may not use this file except in compliance with the License.
66bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * You may obtain a copy of the License at
76bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *
86bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *      http://www.apache.org/licenses/LICENSE-2.0
96bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *
106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * Unless required by applicable law or agreed to in writing, software
116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * distributed under the License is distributed on an "AS IS" BASIS,
126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * See the License for the specific language governing permissions and
146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * limitations under the License.
156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */
166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/*
186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * !!!!! DO NOT EDIT THIS FILE !!!!!
196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *
206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi * This file was generated from
216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi *   suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi */
236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
2407e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <cerrno>
276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <cstring>
286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <sys/stat.h>
296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include <sys/types.h>
306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/file_utils.h"
33c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi#include "utils/byte_array_view.h"
346bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
356bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace latinime {
366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanaginamespace backward {
3707e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanaginamespace v402 {
386bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
396bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        const FormatUtils::FORMAT_VERSION formatVersion) {
426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!headerBuffer) {
436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        ASSERT(false);
446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("The header buffer must be valid to open ver4 dict buffers.");
456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return Ver4DictBuffersPtr(nullptr);
466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // TODO: take only dictDirPath, and open both header and trie files in the constructor below
486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    const bool isUpdatable = headerBuffer->isUpdatable();
496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable,
506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            formatVersion));
516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi}
526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagibool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        const BufferWithExtendableBuffer *const headerBuffer) const {
556bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Create temporary directory.
566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    const int tmpDirPathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictDirPath,
576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
586bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    char tmpDirPath[tmpDirPathBufSize];
596bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    FileUtils::getFilePathWithSuffix(dictDirPath,
606bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize,
616bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            tmpDirPath);
626bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (FileUtils::existsDir(tmpDirPath)) {
636bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        if (!FileUtils::removeDirAndFiles(tmpDirPath)) {
646bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            AKLOGE("Existing directory %s cannot be removed.", tmpDirPath);
656bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            ASSERT(false);
666bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            return false;
676bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        }
686bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
6907e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi    umask(S_IWGRP | S_IWOTH);
706bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (mkdir(tmpDirPath, S_IRWXU) == -1) {
716bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
726bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
736bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
746bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Get dictionary base path.
756bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */;
766bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    char dictName[dictNameBufSize];
776bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName);
786bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    const int dictPathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName);
796bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    char dictPath[dictPathBufSize];
806bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    FileUtils::getFilePath(tmpDirPath, dictName, dictPathBufSize, dictPath);
816bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
826bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Write header file.
836bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
846bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) {
856bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath,
866bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi                Ver4DictConstants::HEADER_FILE_EXTENSION);
876bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
886bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
896bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Write trie file.
906bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
916bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi            Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
926bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
936bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi                Ver4DictConstants::TRIE_FILE_EXTENSION);
946bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
956bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
966bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Write dictionary contents.
976bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!mTerminalPositionLookupTable.flushToFile(dictPath)) {
986bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
996bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1006bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1016bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!mProbabilityDictContent.flushToFile(dictPath)) {
1026bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
1036bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1046bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1056bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!mBigramDictContent.flushToFile(dictPath)) {
1066bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
1076bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1086bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1096bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!mShortcutDictContent.flushToFile(dictPath)) {
1106bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
1116bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1126bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1136bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Remove existing dictionary.
1146bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (!FileUtils::removeDirAndFiles(dictDirPath)) {
1156bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("Existing directory %s cannot be removed.", dictDirPath);
1166bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        ASSERT(false);
1176bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1186bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1196bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    // Rename temporary directory.
1206bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    if (rename(tmpDirPath, dictDirPath) != 0) {
1216bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        AKLOGE("%s cannot be renamed to %s", tmpDirPath, dictDirPath);
1226bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        ASSERT(false);
1236bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        return false;
1246bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    }
1256bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi    return true;
1266bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi}
1276bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
1286bf268132d60061fd26bd8cba63a12b56b22056eKeisuke KuroyanagiVer4DictBuffers::Ver4DictBuffers(const char *const dictPath,
1296bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
1306bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        const FormatUtils::FORMAT_VERSION formatVersion)
1316bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        : mHeaderBuffer(std::move(headerBuffer)),
1326bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mDictBuffer(MmappedBuffer::openBuffer(dictPath,
1336bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi                  Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
134c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi          mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
135c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi          mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
1366bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi                  BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
137c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi          mExpandableTrieBuffer(
138c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi                  mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() :
139c0c674cdc0721a374e140ad5ee1409c0498b3262Keisuke Kuroyanagi                          ReadWriteByteArrayView(),
1406bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi                  BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
1416bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mTerminalPositionLookupTable(dictPath, isUpdatable),
1426bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
1436bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
1446bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mShortcutDictContent(dictPath, isUpdatable),
1456bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mIsUpdatable(isUpdatable) {}
1466bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
1476bf268132d60061fd26bd8cba63a12b56b22056eKeisuke KuroyanagiVer4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize)
1486bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi        : mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(headerPolicy),
1496bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
1506bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(),
1516bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()),
1526bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
1536bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi          mIsUpdatable(true) {}
1546bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi
15507e14126318f7661f76fdce421d723d64e7ea8deKeisuke Kuroyanagi} // namespace v402
1566bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace backward
1576bf268132d60061fd26bd8cba63a12b56b22056eKeisuke Kuroyanagi} // namespace latinime
158