language_model_dict_content.cpp revision 851e0458fe460526b1f953e39a1e406a21ab4647 
1/*
2 * Copyright (C) 2014, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
18
19namespace latinime {
20
21bool LanguageModelDictContent::save(FILE *const file) const {
22    return mTrieMap.save(file);
23}
24
25bool LanguageModelDictContent::runGC(
26        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
27        const LanguageModelDictContent *const originalContent,
28        int *const outNgramCount) {
29    return runGCInner(terminalIdMap, originalContent->mTrieMap.getEntriesInRootLevel(),
30            0 /* nextLevelBitmapEntryIndex */, outNgramCount);
31}
32
33ProbabilityEntry LanguageModelDictContent::getNgramProbabilityEntry(
34        const WordIdArrayView prevWordIds, const int wordId) const {
35    if (!prevWordIds.empty()) {
36        // TODO: Read n-gram entry.
37        return ProbabilityEntry();
38    }
39    const TrieMap::Result result = mTrieMap.getRoot(wordId);
40    if (!result.mIsValid) {
41        // Not found.
42        return ProbabilityEntry();
43    }
44    return ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo);
45}
46
47bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView prevWordIds,
48        const int terminalId, const ProbabilityEntry *const probabilityEntry) {
49    if (!prevWordIds.empty()) {
50        // TODO: Add n-gram entry.
51        return false;
52    }
53    return mTrieMap.putRoot(terminalId, probabilityEntry->encode(mHasHistoricalInfo));
54}
55
56
57bool LanguageModelDictContent::runGCInner(
58        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
59        const TrieMap::TrieMapRange trieMapRange,
60        const int nextLevelBitmapEntryIndex, int *const outNgramCount) {
61    for (auto &entry : trieMapRange) {
62        const auto it = terminalIdMap->find(entry.key());
63        if (it == terminalIdMap->end() || it->second == Ver4DictConstants::NOT_A_TERMINAL_ID) {
64            // The word has been removed.
65            continue;
66        }
67        if (!mTrieMap.put(it->second, entry.value(), nextLevelBitmapEntryIndex)) {
68            return false;
69        }
70        if (outNgramCount) {
71            *outNgramCount += 1;
72        }
73        if (entry.hasNextLevelMap()) {
74            if (!runGCInner(terminalIdMap, entry.getEntriesInNextLevel(),
75                    mTrieMap.getNextLevelBitmapEntryIndex(it->second, nextLevelBitmapEntryIndex),
76                    outNgramCount)) {
77                return false;
78            }
79        }
80    }
81    return true;
82}
83
84} // namespace latinime
85