BinaryDictionary.java revision 981717da4c414caee57ba98596f9bc634a97f74f 
1/*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.inputmethod.latin;
18
19import android.text.TextUtils;
20import android.util.SparseArray;
21
22import com.android.inputmethod.keyboard.ProximityInfo;
23import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
24import com.android.inputmethod.latin.utils.AdditionalFeaturesSettingUtils;
25import com.android.inputmethod.latin.utils.CollectionUtils;
26import com.android.inputmethod.latin.utils.JniUtils;
27import com.android.inputmethod.latin.utils.StringUtils;
28
29import java.util.ArrayList;
30import java.util.Arrays;
31import java.util.Locale;
32
33/**
34 * Implements a static, compacted, binary dictionary of standard words.
35 */
36public final class BinaryDictionary extends Dictionary {
37    private static final String TAG = BinaryDictionary.class.getSimpleName();
38
39    // Must be equal to MAX_WORD_LENGTH in native/jni/src/defines.h
40    private static final int MAX_WORD_LENGTH = Constants.Dictionary.MAX_WORD_LENGTH;
41    // Must be equal to MAX_RESULTS in native/jni/src/defines.h
42    private static final int MAX_RESULTS = 18;
43
44    private long mNativeDict;
45    private final Locale mLocale;
46    private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH];
47    private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS];
48    private final int[] mSpaceIndices = new int[MAX_RESULTS];
49    private final int[] mOutputScores = new int[MAX_RESULTS];
50    private final int[] mOutputTypes = new int[MAX_RESULTS];
51
52    private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions();
53
54    private final SparseArray<DicTraverseSession> mDicTraverseSessions =
55            CollectionUtils.newSparseArray();
56
57    // TODO: There should be a way to remove used DicTraverseSession objects from
58    // {@code mDicTraverseSessions}.
59    private DicTraverseSession getTraverseSession(final int traverseSessionId) {
60        synchronized(mDicTraverseSessions) {
61            DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId);
62            if (traverseSession == null) {
63                traverseSession = mDicTraverseSessions.get(traverseSessionId);
64                if (traverseSession == null) {
65                    traverseSession = new DicTraverseSession(mLocale, mNativeDict);
66                    mDicTraverseSessions.put(traverseSessionId, traverseSession);
67                }
68            }
69            return traverseSession;
70        }
71    }
72
73    /**
74     * Constructor for the binary dictionary. This is supposed to be called from the
75     * dictionary factory.
76     * @param filename the name of the file to read through native code.
77     * @param offset the offset of the dictionary data within the file.
78     * @param length the length of the binary data.
79     * @param useFullEditDistance whether to use the full edit distance in suggestions
80     * @param dictType the dictionary type, as a human-readable string
81     * @param isUpdatable whether to open the dictionary file in writable mode.
82     */
83    public BinaryDictionary(final String filename, final long offset, final long length,
84            final boolean useFullEditDistance, final Locale locale, final String dictType,
85            final boolean isUpdatable) {
86        super(dictType);
87        mLocale = locale;
88        mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance);
89        loadDictionary(filename, offset, length, isUpdatable);
90    }
91
92    static {
93        JniUtils.loadNativeLibrary();
94    }
95
96    private static native long openNative(String sourceDir, long dictOffset, long dictSize,
97            boolean isUpdatable);
98    private static native void closeNative(long dict);
99    private static native int getProbabilityNative(long dict, int[] word);
100    private static native boolean isValidBigramNative(long dict, int[] word1, int[] word2);
101    private static native int getSuggestionsNative(long dict, long proximityInfo,
102            long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times,
103            int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint,
104            int[] suggestOptions, int[] prevWordCodePointArray,
105            int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes);
106    private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
107    private static native int editDistanceNative(int[] before, int[] after);
108
109    // TODO: Move native dict into session
110    private final void loadDictionary(final String path, final long startOffset,
111            final long length, final boolean isUpdatable) {
112        mNativeDict = openNative(path, startOffset, length, isUpdatable);
113    }
114
115    @Override
116    public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
117            final String prevWord, final ProximityInfo proximityInfo,
118            final boolean blockOffensiveWords) {
119        return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords,
120                0 /* sessionId */);
121    }
122
123    @Override
124    public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer,
125            final String prevWord, final ProximityInfo proximityInfo,
126            final boolean blockOffensiveWords, final int sessionId) {
127        if (!isValidDictionary()) return null;
128
129        Arrays.fill(mInputCodePoints, Constants.NOT_A_CODE);
130        // TODO: toLowerCase in the native code
131        final int[] prevWordCodePointArray = (null == prevWord)
132                ? null : StringUtils.toCodePointArray(prevWord);
133        final int composerSize = composer.size();
134
135        final boolean isGesture = composer.isBatchMode();
136        if (composerSize <= 1 || !isGesture) {
137            if (composerSize > MAX_WORD_LENGTH - 1) return null;
138            for (int i = 0; i < composerSize; i++) {
139                mInputCodePoints[i] = composer.getCodeAt(i);
140            }
141        }
142
143        final InputPointers ips = composer.getInputPointers();
144        final int inputSize = isGesture ? ips.getPointerSize() : composerSize;
145        mNativeSuggestOptions.setIsGesture(isGesture);
146        mNativeSuggestOptions.setAdditionalFeaturesOptions(
147                AdditionalFeaturesSettingUtils.getAdditionalNativeSuggestOptions());
148        // proximityInfo and/or prevWordForBigrams may not be null.
149        final int count = getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(),
150                getTraverseSession(sessionId).getSession(), ips.getXCoordinates(),
151                ips.getYCoordinates(), ips.getTimes(), ips.getPointerIds(), mInputCodePoints,
152                inputSize, 0 /* commitPoint */, mNativeSuggestOptions.getOptions(),
153                prevWordCodePointArray, mOutputCodePoints, mOutputScores, mSpaceIndices,
154                mOutputTypes);
155        final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
156        for (int j = 0; j < count; ++j) {
157            final int start = j * MAX_WORD_LENGTH;
158            int len = 0;
159            while (len < MAX_WORD_LENGTH && mOutputCodePoints[start + len] != 0) {
160                ++len;
161            }
162            if (len > 0) {
163                final int flags = mOutputTypes[j] & SuggestedWordInfo.KIND_MASK_FLAGS;
164                if (blockOffensiveWords
165                        && 0 != (flags & SuggestedWordInfo.KIND_FLAG_POSSIBLY_OFFENSIVE)
166                        && 0 == (flags & SuggestedWordInfo.KIND_FLAG_EXACT_MATCH)) {
167                    // If we block potentially offensive words, and if the word is possibly
168                    // offensive, then we don't output it unless it's also an exact match.
169                    continue;
170                }
171                final int kind = mOutputTypes[j] & SuggestedWordInfo.KIND_MASK_KIND;
172                final int score = SuggestedWordInfo.KIND_WHITELIST == kind
173                        ? SuggestedWordInfo.MAX_SCORE : mOutputScores[j];
174                // TODO: check that all users of the `kind' parameter are ready to accept
175                // flags too and pass mOutputTypes[j] instead of kind
176                suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len),
177                        score, kind, mDictType));
178            }
179        }
180        return suggestions;
181    }
182
183    public boolean isValidDictionary() {
184        return mNativeDict != 0;
185    }
186
187    public static float calcNormalizedScore(final String before, final String after,
188            final int score) {
189        return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),
190                StringUtils.toCodePointArray(after), score);
191    }
192
193    public static int editDistance(final String before, final String after) {
194        if (before == null || after == null) {
195            throw new IllegalArgumentException();
196        }
197        return editDistanceNative(StringUtils.toCodePointArray(before),
198                StringUtils.toCodePointArray(after));
199    }
200
201    @Override
202    public boolean isValidWord(final String word) {
203        return getFrequency(word) >= 0;
204    }
205
206    @Override
207    public int getFrequency(final String word) {
208        if (word == null) return -1;
209        int[] codePoints = StringUtils.toCodePointArray(word);
210        return getProbabilityNative(mNativeDict, codePoints);
211    }
212
213    // TODO: Add a batch process version (isValidBigramMultiple?) to avoid excessive numbers of jni
214    // calls when checking for changes in an entire dictionary.
215    public boolean isValidBigram(final String word1, final String word2) {
216        if (TextUtils.isEmpty(word1) || TextUtils.isEmpty(word2)) return false;
217        final int[] codePoints1 = StringUtils.toCodePointArray(word1);
218        final int[] codePoints2 = StringUtils.toCodePointArray(word2);
219        return isValidBigramNative(mNativeDict, codePoints1, codePoints2);
220    }
221
222    @Override
223    public void close() {
224        synchronized (mDicTraverseSessions) {
225            final int sessionsSize = mDicTraverseSessions.size();
226            for (int index = 0; index < sessionsSize; ++index) {
227                final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index);
228                if (traverseSession != null) {
229                    traverseSession.close();
230                }
231            }
232        }
233        closeInternal();
234    }
235
236    private synchronized void closeInternal() {
237        if (mNativeDict != 0) {
238            closeNative(mNativeDict);
239            mNativeDict = 0;
240        }
241    }
242
243    @Override
244    protected void finalize() throws Throwable {
245        try {
246            closeInternal();
247        } finally {
248            super.finalize();
249        }
250    }
251}
252