BinaryDictionary.java revision 576c96af95d7f1df869224ada78933d968e9a9c3
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.text.TextUtils; 20import android.util.Log; 21import android.util.SparseArray; 22 23import com.android.inputmethod.annotations.UsedForTesting; 24import com.android.inputmethod.keyboard.ProximityInfo; 25import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 26import com.android.inputmethod.latin.makedict.DictionaryHeader; 27import com.android.inputmethod.latin.makedict.FormatSpec; 28import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; 29import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 30import com.android.inputmethod.latin.makedict.WordProperty; 31import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 32import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; 33import com.android.inputmethod.latin.utils.FileUtils; 34import com.android.inputmethod.latin.utils.JniUtils; 35import com.android.inputmethod.latin.utils.LanguageModelParam; 36import com.android.inputmethod.latin.utils.StringUtils; 37 38import java.io.File; 39import java.util.ArrayList; 40import java.util.Arrays; 41import java.util.HashMap; 42import java.util.Locale; 43import java.util.Map; 44 45/** 46 * Implements a static, compacted, binary dictionary of standard words. 47 */ 48// TODO: All methods which should be locked need to have a suffix "Locked". 49public final class BinaryDictionary extends Dictionary { 50 private static final String TAG = BinaryDictionary.class.getSimpleName(); 51 52 // The cutoff returned by native for auto-commit confidence. 53 // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h 54 private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; 55 56 @UsedForTesting 57 public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 58 @UsedForTesting 59 public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 60 @UsedForTesting 61 public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT"; 62 @UsedForTesting 63 public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; 64 65 public static final int NOT_A_VALID_TIMESTAMP = -1; 66 67 // Format to get unigram flags from native side via getWordPropertyNative(). 68 private static final int FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT = 5; 69 private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; 70 private static final int FORMAT_WORD_PROPERTY_IS_BLACKLISTED_INDEX = 1; 71 private static final int FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX = 2; 72 private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; 73 private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; 74 75 // Format to get probability and historical info from native side via getWordPropertyNative(). 76 public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4; 77 public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0; 78 public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1; 79 public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2; 80 public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3; 81 82 public static final String DICT_FILE_NAME_SUFFIX_FOR_MIGRATION = ".migrate"; 83 public static final String DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION = ".migrating"; 84 85 private long mNativeDict; 86 private final long mDictSize; 87 private final String mDictFilePath; 88 private final boolean mUseFullEditDistance; 89 private final boolean mIsUpdatable; 90 private boolean mHasUpdated; 91 92 private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); 93 94 // TODO: There should be a way to remove used DicTraverseSession objects from 95 // {@code mDicTraverseSessions}. 96 private DicTraverseSession getTraverseSession(final int traverseSessionId) { 97 synchronized(mDicTraverseSessions) { 98 DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); 99 if (traverseSession == null) { 100 traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); 101 mDicTraverseSessions.put(traverseSessionId, traverseSession); 102 } 103 return traverseSession; 104 } 105 } 106 107 /** 108 * Constructs binary dictionary using existing dictionary file. 109 * @param filename the name of the file to read through native code. 110 * @param offset the offset of the dictionary data within the file. 111 * @param length the length of the binary data. 112 * @param useFullEditDistance whether to use the full edit distance in suggestions 113 * @param dictType the dictionary type, as a human-readable string 114 * @param isUpdatable whether to open the dictionary file in writable mode. 115 */ 116 public BinaryDictionary(final String filename, final long offset, final long length, 117 final boolean useFullEditDistance, final Locale locale, final String dictType, 118 final boolean isUpdatable) { 119 super(dictType, locale); 120 mDictSize = length; 121 mDictFilePath = filename; 122 mIsUpdatable = isUpdatable; 123 mHasUpdated = false; 124 mUseFullEditDistance = useFullEditDistance; 125 loadDictionary(filename, offset, length, isUpdatable); 126 } 127 128 /** 129 * Constructs binary dictionary on memory. 130 * @param filename the name of the file used to flush. 131 * @param useFullEditDistance whether to use the full edit distance in suggestions 132 * @param dictType the dictionary type, as a human-readable string 133 * @param formatVersion the format version of the dictionary 134 * @param attributeMap the attributes of the dictionary 135 */ 136 public BinaryDictionary(final String filename, final boolean useFullEditDistance, 137 final Locale locale, final String dictType, final long formatVersion, 138 final Map<String, String> attributeMap) { 139 super(dictType, locale); 140 mDictSize = 0; 141 mDictFilePath = filename; 142 // On memory dictionary is always updatable. 143 mIsUpdatable = true; 144 mHasUpdated = false; 145 mUseFullEditDistance = useFullEditDistance; 146 final String[] keyArray = new String[attributeMap.size()]; 147 final String[] valueArray = new String[attributeMap.size()]; 148 int index = 0; 149 for (final String key : attributeMap.keySet()) { 150 keyArray[index] = key; 151 valueArray[index] = attributeMap.get(key); 152 index++; 153 } 154 mNativeDict = createOnMemoryNative(formatVersion, locale.toString(), keyArray, valueArray); 155 } 156 157 158 static { 159 JniUtils.loadNativeLibrary(); 160 } 161 162 private static native long openNative(String sourceDir, long dictOffset, long dictSize, 163 boolean isUpdatable); 164 private static native long createOnMemoryNative(long formatVersion, 165 String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray); 166 private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, 167 int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, 168 ArrayList<int[]> outAttributeValues); 169 private static native boolean flushNative(long dict, String filePath); 170 private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); 171 private static native boolean flushWithGCNative(long dict, String filePath); 172 private static native void closeNative(long dict); 173 private static native int getFormatVersionNative(long dict); 174 private static native int getProbabilityNative(long dict, int[] word); 175 private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); 176 private static native int getNgramProbabilityNative(long dict, int[][] prevWordCodePointArrays, 177 boolean[] isBeginningOfSentenceArray, int[] word); 178 private static native void getWordPropertyNative(long dict, int[] word, 179 boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, 180 int[] outProbabilityInfo, ArrayList<int[]> outBigramTargets, 181 ArrayList<int[]> outBigramProbabilityInfo, ArrayList<int[]> outShortcutTargets, 182 ArrayList<Integer> outShortcutProbabilities); 183 private static native int getNextWordNative(long dict, int token, int[] outCodePoints, 184 boolean[] outIsBeginningOfSentence); 185 private static native void getSuggestionsNative(long dict, long proximityInfo, 186 long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, 187 int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, 188 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 189 int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores, 190 int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence, 191 float[] inOutLanguageWeight); 192 private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, 193 int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, 194 boolean isNotAWord, boolean isBlacklisted, int timestamp); 195 private static native boolean removeUnigramEntryNative(long dict, int[] word); 196 private static native boolean addNgramEntryNative(long dict, 197 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 198 int[] word, int probability, int timestamp); 199 private static native boolean removeNgramEntryNative(long dict, 200 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); 201 private static native int addMultipleDictionaryEntriesNative(long dict, 202 LanguageModelParam[] languageModelParams, int startIndex); 203 private static native String getPropertyNative(long dict, String query); 204 private static native boolean isCorruptedNative(long dict); 205 private static native boolean migrateNative(long dict, String dictFilePath, 206 long newFormatVersion); 207 208 // TODO: Move native dict into session 209 private final void loadDictionary(final String path, final long startOffset, 210 final long length, final boolean isUpdatable) { 211 mHasUpdated = false; 212 mNativeDict = openNative(path, startOffset, length, isUpdatable); 213 } 214 215 // TODO: Check isCorrupted() for main dictionaries. 216 public boolean isCorrupted() { 217 if (!isValidDictionary()) { 218 return false; 219 } 220 if (!isCorruptedNative(mNativeDict)) { 221 return false; 222 } 223 // TODO: Record the corruption. 224 Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted."); 225 Log.e(TAG, "locale: " + mLocale); 226 Log.e(TAG, "dict size: " + mDictSize); 227 Log.e(TAG, "updatable: " + mIsUpdatable); 228 return true; 229 } 230 231 public DictionaryHeader getHeader() throws UnsupportedFormatException { 232 if (mNativeDict == 0) { 233 return null; 234 } 235 final int[] outHeaderSize = new int[1]; 236 final int[] outFormatVersion = new int[1]; 237 final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); 238 final ArrayList<int[]> outAttributeValues = new ArrayList<>(); 239 getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, 240 outAttributeValues); 241 final HashMap<String, String> attributes = new HashMap<>(); 242 for (int i = 0; i < outAttributeKeys.size(); i++) { 243 final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( 244 outAttributeKeys.get(i)); 245 final String attributeValue = StringUtils.getStringFromNullTerminatedCodePointArray( 246 outAttributeValues.get(i)); 247 attributes.put(attributeKey, attributeValue); 248 } 249 final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( 250 attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); 251 return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), 252 new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); 253 } 254 255 @Override 256 public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, 257 final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, 258 final SettingsValuesForSuggestion settingsValuesForSuggestion, 259 final int sessionId, final float[] inOutLanguageWeight) { 260 if (!isValidDictionary()) { 261 return null; 262 } 263 final DicTraverseSession session = getTraverseSession(sessionId); 264 Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); 265 prevWordsInfo.outputToArray(session.mPrevWordCodePointArrays, 266 session.mIsBeginningOfSentenceArray); 267 final InputPointers inputPointers = composer.getInputPointers(); 268 final boolean isGesture = composer.isBatchMode(); 269 final int inputSize; 270 if (!isGesture) { 271 inputSize = composer.copyCodePointsExceptTrailingSingleQuotesAndReturnCodePointCount( 272 session.mInputCodePoints); 273 if (inputSize < 0) { 274 return null; 275 } 276 } else { 277 inputSize = inputPointers.getPointerSize(); 278 } 279 session.mNativeSuggestOptions.setUseFullEditDistance(mUseFullEditDistance); 280 session.mNativeSuggestOptions.setIsGesture(isGesture); 281 session.mNativeSuggestOptions.setBlockOffensiveWords( 282 settingsValuesForSuggestion.mBlockPotentiallyOffensive); 283 session.mNativeSuggestOptions.setSpaceAwareGestureEnabled( 284 settingsValuesForSuggestion.mSpaceAwareGestureEnabled); 285 session.mNativeSuggestOptions.setAdditionalFeaturesOptions( 286 settingsValuesForSuggestion.mAdditionalFeaturesSettingValues); 287 if (inOutLanguageWeight != null) { 288 session.mInputOutputLanguageWeight[0] = inOutLanguageWeight[0]; 289 } else { 290 session.mInputOutputLanguageWeight[0] = Dictionary.NOT_A_LANGUAGE_WEIGHT; 291 } 292 // TOOD: Pass multiple previous words information for n-gram. 293 getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(), 294 getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), 295 inputPointers.getYCoordinates(), inputPointers.getTimes(), 296 inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, 297 session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays, 298 session.mIsBeginningOfSentenceArray, session.mOutputSuggestionCount, 299 session.mOutputCodePoints, session.mOutputScores, session.mSpaceIndices, 300 session.mOutputTypes, session.mOutputAutoCommitFirstWordConfidence, 301 session.mInputOutputLanguageWeight); 302 if (inOutLanguageWeight != null) { 303 inOutLanguageWeight[0] = session.mInputOutputLanguageWeight[0]; 304 } 305 final int count = session.mOutputSuggestionCount[0]; 306 final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); 307 for (int j = 0; j < count; ++j) { 308 final int start = j * Constants.DICTIONARY_MAX_WORD_LENGTH; 309 int len = 0; 310 while (len < Constants.DICTIONARY_MAX_WORD_LENGTH 311 && session.mOutputCodePoints[start + len] != 0) { 312 ++len; 313 } 314 if (len > 0) { 315 suggestions.add(new SuggestedWordInfo( 316 new String(session.mOutputCodePoints, start, len), 317 session.mOutputScores[j], session.mOutputTypes[j], this /* sourceDict */, 318 session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, 319 session.mOutputAutoCommitFirstWordConfidence[0])); 320 } 321 } 322 return suggestions; 323 } 324 325 public boolean isValidDictionary() { 326 return mNativeDict != 0; 327 } 328 329 public int getFormatVersion() { 330 return getFormatVersionNative(mNativeDict); 331 } 332 333 @Override 334 public boolean isInDictionary(final String word) { 335 return getFrequency(word) != NOT_A_PROBABILITY; 336 } 337 338 @Override 339 public int getFrequency(final String word) { 340 if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; 341 int[] codePoints = StringUtils.toCodePointArray(word); 342 return getProbabilityNative(mNativeDict, codePoints); 343 } 344 345 @Override 346 public int getMaxFrequencyOfExactMatches(final String word) { 347 if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; 348 int[] codePoints = StringUtils.toCodePointArray(word); 349 return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); 350 } 351 352 @UsedForTesting 353 public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) { 354 return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY; 355 } 356 357 public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) { 358 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 359 return NOT_A_PROBABILITY; 360 } 361 final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; 362 final boolean[] isBeginningOfSentenceArray = 363 new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; 364 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 365 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 366 return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays, 367 isBeginningOfSentenceArray, wordCodePoints); 368 } 369 370 public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { 371 if (word == null) { 372 return null; 373 } 374 final int[] codePoints = StringUtils.toCodePointArray(word); 375 final int[] outCodePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH]; 376 final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; 377 final int[] outProbabilityInfo = 378 new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; 379 final ArrayList<int[]> outBigramTargets = new ArrayList<>(); 380 final ArrayList<int[]> outBigramProbabilityInfo = new ArrayList<>(); 381 final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); 382 final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); 383 getWordPropertyNative(mNativeDict, codePoints, isBeginningOfSentence, outCodePoints, 384 outFlags, outProbabilityInfo, outBigramTargets, outBigramProbabilityInfo, 385 outShortcutTargets, outShortcutProbabilities); 386 return new WordProperty(codePoints, 387 outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], 388 outFlags[FORMAT_WORD_PROPERTY_IS_BLACKLISTED_INDEX], 389 outFlags[FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX], 390 outFlags[FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX], 391 outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, 392 outBigramTargets, outBigramProbabilityInfo, outShortcutTargets, 393 outShortcutProbabilities); 394 } 395 396 public static class GetNextWordPropertyResult { 397 public WordProperty mWordProperty; 398 public int mNextToken; 399 400 public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { 401 mWordProperty = wordProperty; 402 mNextToken = nextToken; 403 } 404 } 405 406 /** 407 * Method to iterate all words in the dictionary for makedict. 408 * If token is 0, this method newly starts iterating the dictionary. 409 */ 410 public GetNextWordPropertyResult getNextWordProperty(final int token) { 411 final int[] codePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH]; 412 final boolean[] isBeginningOfSentence = new boolean[1]; 413 final int nextToken = getNextWordNative(mNativeDict, token, codePoints, 414 isBeginningOfSentence); 415 final String word = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); 416 return new GetNextWordPropertyResult( 417 getWordProperty(word, isBeginningOfSentence[0]), nextToken); 418 } 419 420 // Add a unigram entry to binary dictionary with unigram attributes in native code. 421 public boolean addUnigramEntry(final String word, final int probability, 422 final String shortcutTarget, final int shortcutProbability, 423 final boolean isBeginningOfSentence, final boolean isNotAWord, 424 final boolean isBlacklisted, final int timestamp) { 425 if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { 426 return false; 427 } 428 final int[] codePoints = StringUtils.toCodePointArray(word); 429 final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? 430 StringUtils.toCodePointArray(shortcutTarget) : null; 431 if (!addUnigramEntryNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, 432 shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp)) { 433 return false; 434 } 435 mHasUpdated = true; 436 return true; 437 } 438 439 // Remove a unigram entry from the binary dictionary in native code. 440 public boolean removeUnigramEntry(final String word) { 441 if (TextUtils.isEmpty(word)) { 442 return false; 443 } 444 final int[] codePoints = StringUtils.toCodePointArray(word); 445 if (!removeUnigramEntryNative(mNativeDict, codePoints)) { 446 return false; 447 } 448 mHasUpdated = true; 449 return true; 450 } 451 452 // Add an n-gram entry to the binary dictionary with timestamp in native code. 453 public boolean addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, 454 final int probability, final int timestamp) { 455 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 456 return false; 457 } 458 final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; 459 final boolean[] isBeginningOfSentenceArray = 460 new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; 461 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 462 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 463 if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays, 464 isBeginningOfSentenceArray, wordCodePoints, probability, timestamp)) { 465 return false; 466 } 467 mHasUpdated = true; 468 return true; 469 } 470 471 // Remove an n-gram entry from the binary dictionary in native code. 472 public boolean removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) { 473 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 474 return false; 475 } 476 final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; 477 final boolean[] isBeginningOfSentenceArray = 478 new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; 479 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 480 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 481 if (!removeNgramEntryNative(mNativeDict, prevWordCodePointArrays, 482 isBeginningOfSentenceArray, wordCodePoints)) { 483 return false; 484 } 485 mHasUpdated = true; 486 return true; 487 } 488 489 public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { 490 if (!isValidDictionary()) return; 491 int processedParamCount = 0; 492 while (processedParamCount < languageModelParams.length) { 493 if (needsToRunGC(true /* mindsBlockByGC */)) { 494 flushWithGC(); 495 } 496 processedParamCount = addMultipleDictionaryEntriesNative(mNativeDict, 497 languageModelParams, processedParamCount); 498 mHasUpdated = true; 499 if (processedParamCount <= 0) { 500 return; 501 } 502 } 503 } 504 505 private void reopen() { 506 close(); 507 final File dictFile = new File(mDictFilePath); 508 // WARNING: Because we pass 0 as the offset and file.length() as the length, this can 509 // only be called for actual files. Right now it's only called by the flush() family of 510 // functions, which require an updatable dictionary, so it's okay. But beware. 511 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 512 dictFile.length(), mIsUpdatable); 513 } 514 515 // Flush to dict file if the dictionary has been updated. 516 public boolean flush() { 517 if (!isValidDictionary()) return false; 518 if (mHasUpdated) { 519 if (!flushNative(mNativeDict, mDictFilePath)) { 520 return false; 521 } 522 reopen(); 523 } 524 return true; 525 } 526 527 // Run GC and flush to dict file if the dictionary has been updated. 528 public boolean flushWithGCIfHasUpdated() { 529 if (mHasUpdated) { 530 return flushWithGC(); 531 } 532 return true; 533 } 534 535 // Run GC and flush to dict file. 536 public boolean flushWithGC() { 537 if (!isValidDictionary()) return false; 538 if (!flushWithGCNative(mNativeDict, mDictFilePath)) { 539 return false; 540 } 541 reopen(); 542 return true; 543 } 544 545 /** 546 * Checks whether GC is needed to run or not. 547 * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about 548 * the blocking in some situations such as in idle time or just before closing. 549 * @return whether GC is needed to run or not. 550 */ 551 public boolean needsToRunGC(final boolean mindsBlockByGC) { 552 if (!isValidDictionary()) return false; 553 return needsToRunGCNative(mNativeDict, mindsBlockByGC); 554 } 555 556 public boolean migrateTo(final int newFormatVersion) { 557 if (!isValidDictionary()) { 558 return false; 559 } 560 final File isMigratingDir = 561 new File(mDictFilePath + DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION); 562 if (isMigratingDir.exists()) { 563 isMigratingDir.delete(); 564 Log.e(TAG, "Previous migration attempt failed probably due to a crash. " 565 + "Giving up using the old dictionary (" + mDictFilePath + ")."); 566 return false; 567 } 568 if (!isMigratingDir.mkdir()) { 569 Log.e(TAG, "Cannot create a dir (" + isMigratingDir.getAbsolutePath() 570 + ") to record migration."); 571 return false; 572 } 573 try { 574 final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; 575 if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { 576 return false; 577 } 578 close(); 579 final File dictFile = new File(mDictFilePath); 580 final File tmpDictFile = new File(tmpDictFilePath); 581 if (!FileUtils.deleteRecursively(dictFile)) { 582 return false; 583 } 584 if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { 585 return false; 586 } 587 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 588 dictFile.length(), mIsUpdatable); 589 return true; 590 } finally { 591 isMigratingDir.delete(); 592 } 593 } 594 595 @UsedForTesting 596 public String getPropertyForTest(final String query) { 597 if (!isValidDictionary()) return ""; 598 return getPropertyNative(mNativeDict, query); 599 } 600 601 @Override 602 public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { 603 return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; 604 } 605 606 @Override 607 public void close() { 608 synchronized (mDicTraverseSessions) { 609 final int sessionsSize = mDicTraverseSessions.size(); 610 for (int index = 0; index < sessionsSize; ++index) { 611 final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index); 612 if (traverseSession != null) { 613 traverseSession.close(); 614 } 615 } 616 mDicTraverseSessions.clear(); 617 } 618 closeInternalLocked(); 619 } 620 621 private synchronized void closeInternalLocked() { 622 if (mNativeDict != 0) { 623 closeNative(mNativeDict); 624 mNativeDict = 0; 625 } 626 } 627 628 // TODO: Manage BinaryDictionary instances without using WeakReference or something. 629 @Override 630 protected void finalize() throws Throwable { 631 try { 632 closeInternalLocked(); 633 } finally { 634 super.finalize(); 635 } 636 } 637} 638