language_model_dict_content.cpp revision 03dc44f543795040a092723085fac1209103b7bd 
1/*
2 * Copyright (C) 2014, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
18
19namespace latinime {
20
21bool LanguageModelDictContent::save(FILE *const file) const {
22    return mTrieMap.save(file);
23}
24
25bool LanguageModelDictContent::runGC(
26        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
27        const LanguageModelDictContent *const originalContent,
28        int *const outNgramCount) {
29    return runGCInner(terminalIdMap, originalContent->mTrieMap.getEntriesInRootLevel(),
30            0 /* nextLevelBitmapEntryIndex */, outNgramCount);
31}
32
33ProbabilityEntry LanguageModelDictContent::getNgramProbabilityEntry(
34        const WordIdArrayView prevWordIds, const int wordId) const {
35    const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds);
36    if (bitmapEntryIndex == TrieMap::INVALID_INDEX) {
37        return ProbabilityEntry();
38    }
39    const TrieMap::Result result = mTrieMap.get(wordId, bitmapEntryIndex);
40    if (!result.mIsValid) {
41        // Not found.
42        return ProbabilityEntry();
43    }
44    return ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo);
45}
46
47bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView prevWordIds,
48        const int terminalId, const ProbabilityEntry *const probabilityEntry) {
49    const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds);
50    if (bitmapEntryIndex == TrieMap::INVALID_INDEX) {
51        return false;
52    }
53    return mTrieMap.put(terminalId, probabilityEntry->encode(mHasHistoricalInfo), bitmapEntryIndex);
54}
55
56bool LanguageModelDictContent::runGCInner(
57        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
58        const TrieMap::TrieMapRange trieMapRange,
59        const int nextLevelBitmapEntryIndex, int *const outNgramCount) {
60    for (auto &entry : trieMapRange) {
61        const auto it = terminalIdMap->find(entry.key());
62        if (it == terminalIdMap->end() || it->second == Ver4DictConstants::NOT_A_TERMINAL_ID) {
63            // The word has been removed.
64            continue;
65        }
66        if (!mTrieMap.put(it->second, entry.value(), nextLevelBitmapEntryIndex)) {
67            return false;
68        }
69        if (outNgramCount) {
70            *outNgramCount += 1;
71        }
72        if (entry.hasNextLevelMap()) {
73            if (!runGCInner(terminalIdMap, entry.getEntriesInNextLevel(),
74                    mTrieMap.getNextLevelBitmapEntryIndex(it->second, nextLevelBitmapEntryIndex),
75                    outNgramCount)) {
76                return false;
77            }
78        }
79    }
80    return true;
81}
82
83int LanguageModelDictContent::getBitmapEntryIndex(const WordIdArrayView prevWordIds) const {
84    int bitmapEntryIndex = mTrieMap.getRootBitmapEntryIndex();
85    for (const int wordId : prevWordIds) {
86        const TrieMap::Result result = mTrieMap.get(wordId, bitmapEntryIndex);
87        if (!result.mIsValid) {
88            return TrieMap::INVALID_INDEX;
89        }
90        bitmapEntryIndex = result.mNextLevelBitmapEntryIndex;
91    }
92    return bitmapEntryIndex;
93}
94
95} // namespace latinime
96