12fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa/* 22fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * Copyright (C) 2013, The Android Open Source Project 32fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * 42fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * Licensed under the Apache License, Version 2.0 (the "License"); 52fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * you may not use this file except in compliance with the License. 62fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * You may obtain a copy of the License at 72fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * 82fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * http://www.apache.org/licenses/LICENSE-2.0 92fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * 102fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * Unless required by applicable law or agreed to in writing, software 112fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * distributed under the License is distributed on an "AS IS" BASIS, 122fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 132fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * See the License for the specific language governing permissions and 142fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa * limitations under the License. 152fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa */ 162fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 172fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#ifndef LATINIME_VER4_DICT_BUFFER_H 182fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#define LATINIME_VER4_DICT_BUFFER_H 192fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 202ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi#include <cstdio> 214ce480d5ce2d47f607448ce439aaf2cefba1bdd8Keisuke Kuroyanagi#include <memory> 224ce480d5ce2d47f607448ce439aaf2cefba1bdd8Keisuke Kuroyanagi 232fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "defines.h" 242fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/header/header_policy.h" 252fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" 26dc3856d7589aa3cf3dcfdee8360fa48a85983273Keisuke Kuroyanagi#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h" 272fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h" 282fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" 292fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" 302fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" 312fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" 322fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 332fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasanamespace latinime { 342fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 352fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasaclass Ver4DictBuffers { 362fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa public: 374ce480d5ce2d47f607448ce439aaf2cefba1bdd8Keisuke Kuroyanagi typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr; 382fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 392fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, 402ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi MmappedBuffer::MmappedBufferPtr &&headerBuffer, 41455dc84cf2c6526329b535f30000ea45b7d4d4d7Keisuke Kuroyanagi const FormatUtils::FORMAT_VERSION formatVersion); 422fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 432fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( 447b43d987fa9d225c4305f4681dc8e27f244279a8Keisuke Kuroyanagi const HeaderPolicy *const headerPolicy, const int maxTrieSize) { 457b43d987fa9d225c4305f4681dc8e27f244279a8Keisuke Kuroyanagi return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy, maxTrieSize)); 462fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 472fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 482fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE bool isValid() const { 492ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid(); 502fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 512fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 522fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE bool isNearSizeLimit() const { 532fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return mExpandableTrieBuffer.isNearSizeLimit() 542fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa || mTerminalPositionLookupTable.isNearSizeLimit() 55851e0458fe460526b1f953e39a1e406a21ab4647Keisuke Kuroyanagi || mLanguageModelDictContent.isNearSizeLimit() 562fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa || mBigramDictContent.isNearSizeLimit() 572fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa || mShortcutDictContent.isNearSizeLimit(); 582fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 592fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 602fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE const HeaderPolicy *getHeaderPolicy() const { 612fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mHeaderPolicy; 622fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 632fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 642fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() { 652fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mExpandableHeaderBuffer; 662fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 672fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 682fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableTrieBuffer() { 692fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mExpandableTrieBuffer; 702fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 712fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 722fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE const BufferWithExtendableBuffer *getTrieBuffer() const { 732fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mExpandableTrieBuffer; 742fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 752fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 762fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE TerminalPositionLookupTable *getMutableTerminalPositionLookupTable() { 772fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mTerminalPositionLookupTable; 782fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 792fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 802fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE const TerminalPositionLookupTable *getTerminalPositionLookupTable() const { 812fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mTerminalPositionLookupTable; 822fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 832fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 8408894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi AK_FORCE_INLINE LanguageModelDictContent *getMutableLanguageModelDictContent() { 8508894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi return &mLanguageModelDictContent; 8608894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi } 8708894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi 8808894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi AK_FORCE_INLINE const LanguageModelDictContent *getLanguageModelDictContent() const { 8908894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi return &mLanguageModelDictContent; 9008894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi } 9108894842662eff666a713a7f4deb79204a322f8cKeisuke Kuroyanagi 922fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE BigramDictContent *getMutableBigramDictContent() { 932fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mBigramDictContent; 942fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 952fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 962fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE const BigramDictContent *getBigramDictContent() const { 972fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mBigramDictContent; 982fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 992fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1002fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE ShortcutDictContent *getMutableShortcutDictContent() { 1012fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mShortcutDictContent; 1022fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 1032fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1042fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE const ShortcutDictContent *getShortcutDictContent() const { 1052fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return &mShortcutDictContent; 1062fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 1072fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1082fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa AK_FORCE_INLINE bool isUpdatable() const { 1092fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return mIsUpdatable; 1102fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 1112fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1122fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa bool flush(const char *const dictDirPath) const { 1132fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return flushHeaderAndDictBuffers(dictDirPath, &mExpandableHeaderBuffer); 1142fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 1152fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1162fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa bool flushHeaderAndDictBuffers(const char *const dictDirPath, 1172fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa const BufferWithExtendableBuffer *const headerBuffer) const; 1182fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1192fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa private: 1202fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); 1212fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1222ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, 1232ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi MmappedBuffer::MmappedBufferPtr &&bodyBuffer, 124b22f95ec8a8325e65b95e7b0d57156854f7984f6Keisuke Kuroyanagi const FormatUtils::FORMAT_VERSION formatVersion, 1252ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi const std::vector<uint8_t *> &contentBuffers, 1262ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi const std::vector<int> &contentBufferSizes); 1272fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1287b43d987fa9d225c4305f4681dc8e27f244279a8Keisuke Kuroyanagi Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize); 1292fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 1302ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi bool flushDictBuffers(FILE *const file) const; 1312ac934296c0571ea252f3fb5a23fba29eb89c666Keisuke Kuroyanagi 1322fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa const MmappedBuffer::MmappedBufferPtr mHeaderBuffer; 1332fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa const MmappedBuffer::MmappedBufferPtr mDictBuffer; 1342fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa const HeaderPolicy mHeaderPolicy; 1352fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa BufferWithExtendableBuffer mExpandableHeaderBuffer; 1362fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa BufferWithExtendableBuffer mExpandableTrieBuffer; 1372fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa TerminalPositionLookupTable mTerminalPositionLookupTable; 138dc3856d7589aa3cf3dcfdee8360fa48a85983273Keisuke Kuroyanagi LanguageModelDictContent mLanguageModelDictContent; 1392fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa BigramDictContent mBigramDictContent; 1402fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa ShortcutDictContent mShortcutDictContent; 1412fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa const int mIsUpdatable; 1422fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa}; 1432fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa} // namespace latinime 1442fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa#endif /* LATINIME_VER4_DICT_BUFFER_H */ 145