BinaryDictionary.java revision bacf2dbac6d27c6a5677ca3f37a6da8bfd0c8db1
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.text.TextUtils; 20import android.util.Log; 21import android.util.SparseArray; 22 23import com.android.inputmethod.annotations.UsedForTesting; 24import com.android.inputmethod.keyboard.ProximityInfo; 25import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 26import com.android.inputmethod.latin.makedict.DictionaryHeader; 27import com.android.inputmethod.latin.makedict.FormatSpec; 28import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; 29import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 30import com.android.inputmethod.latin.makedict.WordProperty; 31import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 32import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; 33import com.android.inputmethod.latin.utils.FileUtils; 34import com.android.inputmethod.latin.utils.JniUtils; 35import com.android.inputmethod.latin.utils.LanguageModelParam; 36import com.android.inputmethod.latin.utils.StringUtils; 37 38import java.io.File; 39import java.util.ArrayList; 40import java.util.Arrays; 41import java.util.HashMap; 42import java.util.Locale; 43import java.util.Map; 44 45/** 46 * Implements a static, compacted, binary dictionary of standard words. 47 */ 48// TODO: All methods which should be locked need to have a suffix "Locked". 49public final class BinaryDictionary extends Dictionary { 50 private static final String TAG = BinaryDictionary.class.getSimpleName(); 51 52 // The cutoff returned by native for auto-commit confidence. 53 // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h 54 private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; 55 56 @UsedForTesting 57 public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 58 @UsedForTesting 59 public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 60 @UsedForTesting 61 public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT"; 62 @UsedForTesting 63 public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; 64 65 public static final int NOT_A_VALID_TIMESTAMP = -1; 66 67 // Format to get unigram flags from native side via getWordPropertyNative(). 68 private static final int FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT = 5; 69 private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; 70 private static final int FORMAT_WORD_PROPERTY_IS_BLACKLISTED_INDEX = 1; 71 private static final int FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX = 2; 72 private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; 73 private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; 74 75 // Format to get probability and historical info from native side via getWordPropertyNative(). 76 public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4; 77 public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0; 78 public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1; 79 public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2; 80 public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3; 81 82 public static final String DICT_FILE_NAME_SUFFIX_FOR_MIGRATION = ".migrate"; 83 public static final String DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION = ".migrating"; 84 85 private long mNativeDict; 86 private final long mDictSize; 87 private final String mDictFilePath; 88 private final boolean mUseFullEditDistance; 89 private final boolean mIsUpdatable; 90 private boolean mHasUpdated; 91 92 private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); 93 94 // TODO: There should be a way to remove used DicTraverseSession objects from 95 // {@code mDicTraverseSessions}. 96 private DicTraverseSession getTraverseSession(final int traverseSessionId) { 97 synchronized(mDicTraverseSessions) { 98 DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); 99 if (traverseSession == null) { 100 traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); 101 mDicTraverseSessions.put(traverseSessionId, traverseSession); 102 } 103 return traverseSession; 104 } 105 } 106 107 /** 108 * Constructs binary dictionary using existing dictionary file. 109 * @param filename the name of the file to read through native code. 110 * @param offset the offset of the dictionary data within the file. 111 * @param length the length of the binary data. 112 * @param useFullEditDistance whether to use the full edit distance in suggestions 113 * @param dictType the dictionary type, as a human-readable string 114 * @param isUpdatable whether to open the dictionary file in writable mode. 115 */ 116 public BinaryDictionary(final String filename, final long offset, final long length, 117 final boolean useFullEditDistance, final Locale locale, final String dictType, 118 final boolean isUpdatable) { 119 super(dictType, locale); 120 mDictSize = length; 121 mDictFilePath = filename; 122 mIsUpdatable = isUpdatable; 123 mHasUpdated = false; 124 mUseFullEditDistance = useFullEditDistance; 125 loadDictionary(filename, offset, length, isUpdatable); 126 } 127 128 /** 129 * Constructs binary dictionary on memory. 130 * @param filename the name of the file used to flush. 131 * @param useFullEditDistance whether to use the full edit distance in suggestions 132 * @param dictType the dictionary type, as a human-readable string 133 * @param formatVersion the format version of the dictionary 134 * @param attributeMap the attributes of the dictionary 135 */ 136 public BinaryDictionary(final String filename, final boolean useFullEditDistance, 137 final Locale locale, final String dictType, final long formatVersion, 138 final Map<String, String> attributeMap) { 139 super(dictType, locale); 140 mDictSize = 0; 141 mDictFilePath = filename; 142 // On memory dictionary is always updatable. 143 mIsUpdatable = true; 144 mHasUpdated = false; 145 mUseFullEditDistance = useFullEditDistance; 146 final String[] keyArray = new String[attributeMap.size()]; 147 final String[] valueArray = new String[attributeMap.size()]; 148 int index = 0; 149 for (final String key : attributeMap.keySet()) { 150 keyArray[index] = key; 151 valueArray[index] = attributeMap.get(key); 152 index++; 153 } 154 mNativeDict = createOnMemoryNative(formatVersion, locale.toString(), keyArray, valueArray); 155 } 156 157 158 static { 159 JniUtils.loadNativeLibrary(); 160 } 161 162 private static native long openNative(String sourceDir, long dictOffset, long dictSize, 163 boolean isUpdatable); 164 private static native long createOnMemoryNative(long formatVersion, 165 String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray); 166 private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, 167 int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, 168 ArrayList<int[]> outAttributeValues); 169 private static native boolean flushNative(long dict, String filePath); 170 private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); 171 private static native boolean flushWithGCNative(long dict, String filePath); 172 private static native void closeNative(long dict); 173 private static native int getFormatVersionNative(long dict); 174 private static native int getProbabilityNative(long dict, int[] word); 175 private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); 176 private static native int getNgramProbabilityNative(long dict, int[][] prevWordCodePointArrays, 177 boolean[] isBeginningOfSentenceArray, int[] word); 178 private static native void getWordPropertyNative(long dict, int[] word, 179 boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, 180 int[] outProbabilityInfo, ArrayList<int[]> outBigramTargets, 181 ArrayList<int[]> outBigramProbabilityInfo, ArrayList<int[]> outShortcutTargets, 182 ArrayList<Integer> outShortcutProbabilities); 183 private static native int getNextWordNative(long dict, int token, int[] outCodePoints, 184 boolean[] outIsBeginningOfSentence); 185 private static native void getSuggestionsNative(long dict, long proximityInfo, 186 long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, 187 int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, 188 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 189 int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints, 190 int[] outputScores, int[] outputIndices, int[] outputTypes, 191 int[] outputAutoCommitFirstWordConfidence, 192 float[] inOutWeightOfLangModelVsSpatialModel); 193 private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, 194 int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, 195 boolean isNotAWord, boolean isBlacklisted, int timestamp); 196 private static native boolean removeUnigramEntryNative(long dict, int[] word); 197 private static native boolean addNgramEntryNative(long dict, 198 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 199 int[] word, int probability, int timestamp); 200 private static native boolean removeNgramEntryNative(long dict, 201 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); 202 private static native int addMultipleDictionaryEntriesNative(long dict, 203 LanguageModelParam[] languageModelParams, int startIndex); 204 private static native String getPropertyNative(long dict, String query); 205 private static native boolean isCorruptedNative(long dict); 206 private static native boolean migrateNative(long dict, String dictFilePath, 207 long newFormatVersion); 208 209 // TODO: Move native dict into session 210 private final void loadDictionary(final String path, final long startOffset, 211 final long length, final boolean isUpdatable) { 212 mHasUpdated = false; 213 mNativeDict = openNative(path, startOffset, length, isUpdatable); 214 } 215 216 // TODO: Check isCorrupted() for main dictionaries. 217 public boolean isCorrupted() { 218 if (!isValidDictionary()) { 219 return false; 220 } 221 if (!isCorruptedNative(mNativeDict)) { 222 return false; 223 } 224 // TODO: Record the corruption. 225 Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted."); 226 Log.e(TAG, "locale: " + mLocale); 227 Log.e(TAG, "dict size: " + mDictSize); 228 Log.e(TAG, "updatable: " + mIsUpdatable); 229 return true; 230 } 231 232 public DictionaryHeader getHeader() throws UnsupportedFormatException { 233 if (mNativeDict == 0) { 234 return null; 235 } 236 final int[] outHeaderSize = new int[1]; 237 final int[] outFormatVersion = new int[1]; 238 final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); 239 final ArrayList<int[]> outAttributeValues = new ArrayList<>(); 240 getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, 241 outAttributeValues); 242 final HashMap<String, String> attributes = new HashMap<>(); 243 for (int i = 0; i < outAttributeKeys.size(); i++) { 244 final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( 245 outAttributeKeys.get(i)); 246 final String attributeValue = StringUtils.getStringFromNullTerminatedCodePointArray( 247 outAttributeValues.get(i)); 248 attributes.put(attributeKey, attributeValue); 249 } 250 final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( 251 attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); 252 return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), 253 new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); 254 } 255 256 @Override 257 public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, 258 final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, 259 final SettingsValuesForSuggestion settingsValuesForSuggestion, 260 final int sessionId, final float weightForLocale, 261 final float[] inOutWeightOfLangModelVsSpatialModel) { 262 if (!isValidDictionary()) { 263 return null; 264 } 265 final DicTraverseSession session = getTraverseSession(sessionId); 266 Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); 267 prevWordsInfo.outputToArray(session.mPrevWordCodePointArrays, 268 session.mIsBeginningOfSentenceArray); 269 final InputPointers inputPointers = composer.getInputPointers(); 270 final boolean isGesture = composer.isBatchMode(); 271 final int inputSize; 272 if (!isGesture) { 273 inputSize = composer.copyCodePointsExceptTrailingSingleQuotesAndReturnCodePointCount( 274 session.mInputCodePoints); 275 if (inputSize < 0) { 276 return null; 277 } 278 } else { 279 inputSize = inputPointers.getPointerSize(); 280 } 281 session.mNativeSuggestOptions.setUseFullEditDistance(mUseFullEditDistance); 282 session.mNativeSuggestOptions.setIsGesture(isGesture); 283 session.mNativeSuggestOptions.setBlockOffensiveWords( 284 settingsValuesForSuggestion.mBlockPotentiallyOffensive); 285 session.mNativeSuggestOptions.setSpaceAwareGestureEnabled( 286 settingsValuesForSuggestion.mSpaceAwareGestureEnabled); 287 session.mNativeSuggestOptions.setAdditionalFeaturesOptions( 288 settingsValuesForSuggestion.mAdditionalFeaturesSettingValues); 289 if (inOutWeightOfLangModelVsSpatialModel != null) { 290 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 291 inOutWeightOfLangModelVsSpatialModel[0]; 292 } else { 293 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 294 Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL; 295 } 296 // TOOD: Pass multiple previous words information for n-gram. 297 getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(), 298 getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), 299 inputPointers.getYCoordinates(), inputPointers.getTimes(), 300 inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, 301 session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays, 302 session.mIsBeginningOfSentenceArray, prevWordsInfo.getPrevWordCount(), 303 session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, 304 session.mSpaceIndices, session.mOutputTypes, 305 session.mOutputAutoCommitFirstWordConfidence, 306 session.mInputOutputWeightOfLangModelVsSpatialModel); 307 if (inOutWeightOfLangModelVsSpatialModel != null) { 308 inOutWeightOfLangModelVsSpatialModel[0] = 309 session.mInputOutputWeightOfLangModelVsSpatialModel[0]; 310 } 311 final int count = session.mOutputSuggestionCount[0]; 312 final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); 313 for (int j = 0; j < count; ++j) { 314 final int start = j * Constants.DICTIONARY_MAX_WORD_LENGTH; 315 int len = 0; 316 while (len < Constants.DICTIONARY_MAX_WORD_LENGTH 317 && session.mOutputCodePoints[start + len] != 0) { 318 ++len; 319 } 320 if (len > 0) { 321 suggestions.add(new SuggestedWordInfo( 322 new String(session.mOutputCodePoints, start, len), 323 (int)(session.mOutputScores[j] * weightForLocale), session.mOutputTypes[j], 324 this /* sourceDict */, 325 session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, 326 session.mOutputAutoCommitFirstWordConfidence[0])); 327 } 328 } 329 return suggestions; 330 } 331 332 public boolean isValidDictionary() { 333 return mNativeDict != 0; 334 } 335 336 public int getFormatVersion() { 337 return getFormatVersionNative(mNativeDict); 338 } 339 340 @Override 341 public boolean isInDictionary(final String word) { 342 return getFrequency(word) != NOT_A_PROBABILITY; 343 } 344 345 @Override 346 public int getFrequency(final String word) { 347 if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; 348 int[] codePoints = StringUtils.toCodePointArray(word); 349 return getProbabilityNative(mNativeDict, codePoints); 350 } 351 352 @Override 353 public int getMaxFrequencyOfExactMatches(final String word) { 354 if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; 355 int[] codePoints = StringUtils.toCodePointArray(word); 356 return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); 357 } 358 359 @UsedForTesting 360 public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) { 361 return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY; 362 } 363 364 public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) { 365 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 366 return NOT_A_PROBABILITY; 367 } 368 final int[][] prevWordCodePointArrays = new int[prevWordsInfo.getPrevWordCount()][]; 369 final boolean[] isBeginningOfSentenceArray = new boolean[prevWordsInfo.getPrevWordCount()]; 370 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 371 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 372 return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays, 373 isBeginningOfSentenceArray, wordCodePoints); 374 } 375 376 public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { 377 if (word == null) { 378 return null; 379 } 380 final int[] codePoints = StringUtils.toCodePointArray(word); 381 final int[] outCodePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH]; 382 final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; 383 final int[] outProbabilityInfo = 384 new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; 385 final ArrayList<int[]> outBigramTargets = new ArrayList<>(); 386 final ArrayList<int[]> outBigramProbabilityInfo = new ArrayList<>(); 387 final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); 388 final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); 389 getWordPropertyNative(mNativeDict, codePoints, isBeginningOfSentence, outCodePoints, 390 outFlags, outProbabilityInfo, outBigramTargets, outBigramProbabilityInfo, 391 outShortcutTargets, outShortcutProbabilities); 392 return new WordProperty(codePoints, 393 outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], 394 outFlags[FORMAT_WORD_PROPERTY_IS_BLACKLISTED_INDEX], 395 outFlags[FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX], 396 outFlags[FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX], 397 outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, 398 outBigramTargets, outBigramProbabilityInfo, outShortcutTargets, 399 outShortcutProbabilities); 400 } 401 402 public static class GetNextWordPropertyResult { 403 public WordProperty mWordProperty; 404 public int mNextToken; 405 406 public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { 407 mWordProperty = wordProperty; 408 mNextToken = nextToken; 409 } 410 } 411 412 /** 413 * Method to iterate all words in the dictionary for makedict. 414 * If token is 0, this method newly starts iterating the dictionary. 415 */ 416 public GetNextWordPropertyResult getNextWordProperty(final int token) { 417 final int[] codePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH]; 418 final boolean[] isBeginningOfSentence = new boolean[1]; 419 final int nextToken = getNextWordNative(mNativeDict, token, codePoints, 420 isBeginningOfSentence); 421 final String word = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); 422 return new GetNextWordPropertyResult( 423 getWordProperty(word, isBeginningOfSentence[0]), nextToken); 424 } 425 426 // Add a unigram entry to binary dictionary with unigram attributes in native code. 427 public boolean addUnigramEntry(final String word, final int probability, 428 final String shortcutTarget, final int shortcutProbability, 429 final boolean isBeginningOfSentence, final boolean isNotAWord, 430 final boolean isBlacklisted, final int timestamp) { 431 if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { 432 return false; 433 } 434 final int[] codePoints = StringUtils.toCodePointArray(word); 435 final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? 436 StringUtils.toCodePointArray(shortcutTarget) : null; 437 if (!addUnigramEntryNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, 438 shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp)) { 439 return false; 440 } 441 mHasUpdated = true; 442 return true; 443 } 444 445 // Remove a unigram entry from the binary dictionary in native code. 446 public boolean removeUnigramEntry(final String word) { 447 if (TextUtils.isEmpty(word)) { 448 return false; 449 } 450 final int[] codePoints = StringUtils.toCodePointArray(word); 451 if (!removeUnigramEntryNative(mNativeDict, codePoints)) { 452 return false; 453 } 454 mHasUpdated = true; 455 return true; 456 } 457 458 // Add an n-gram entry to the binary dictionary with timestamp in native code. 459 public boolean addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, 460 final int probability, final int timestamp) { 461 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 462 return false; 463 } 464 final int[][] prevWordCodePointArrays = new int[prevWordsInfo.getPrevWordCount()][]; 465 final boolean[] isBeginningOfSentenceArray = new boolean[prevWordsInfo.getPrevWordCount()]; 466 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 467 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 468 if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays, 469 isBeginningOfSentenceArray, wordCodePoints, probability, timestamp)) { 470 return false; 471 } 472 mHasUpdated = true; 473 return true; 474 } 475 476 // Remove an n-gram entry from the binary dictionary in native code. 477 public boolean removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) { 478 if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { 479 return false; 480 } 481 final int[][] prevWordCodePointArrays = new int[prevWordsInfo.getPrevWordCount()][]; 482 final boolean[] isBeginningOfSentenceArray = new boolean[prevWordsInfo.getPrevWordCount()]; 483 prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 484 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 485 if (!removeNgramEntryNative(mNativeDict, prevWordCodePointArrays, 486 isBeginningOfSentenceArray, wordCodePoints)) { 487 return false; 488 } 489 mHasUpdated = true; 490 return true; 491 } 492 493 @UsedForTesting 494 public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { 495 if (!isValidDictionary()) return; 496 int processedParamCount = 0; 497 while (processedParamCount < languageModelParams.length) { 498 if (needsToRunGC(true /* mindsBlockByGC */)) { 499 flushWithGC(); 500 } 501 processedParamCount = addMultipleDictionaryEntriesNative(mNativeDict, 502 languageModelParams, processedParamCount); 503 mHasUpdated = true; 504 if (processedParamCount <= 0) { 505 return; 506 } 507 } 508 } 509 510 private void reopen() { 511 close(); 512 final File dictFile = new File(mDictFilePath); 513 // WARNING: Because we pass 0 as the offset and file.length() as the length, this can 514 // only be called for actual files. Right now it's only called by the flush() family of 515 // functions, which require an updatable dictionary, so it's okay. But beware. 516 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 517 dictFile.length(), mIsUpdatable); 518 } 519 520 // Flush to dict file if the dictionary has been updated. 521 public boolean flush() { 522 if (!isValidDictionary()) return false; 523 if (mHasUpdated) { 524 if (!flushNative(mNativeDict, mDictFilePath)) { 525 return false; 526 } 527 reopen(); 528 } 529 return true; 530 } 531 532 // Run GC and flush to dict file if the dictionary has been updated. 533 public boolean flushWithGCIfHasUpdated() { 534 if (mHasUpdated) { 535 return flushWithGC(); 536 } 537 return true; 538 } 539 540 // Run GC and flush to dict file. 541 public boolean flushWithGC() { 542 if (!isValidDictionary()) return false; 543 if (!flushWithGCNative(mNativeDict, mDictFilePath)) { 544 return false; 545 } 546 reopen(); 547 return true; 548 } 549 550 /** 551 * Checks whether GC is needed to run or not. 552 * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about 553 * the blocking in some situations such as in idle time or just before closing. 554 * @return whether GC is needed to run or not. 555 */ 556 public boolean needsToRunGC(final boolean mindsBlockByGC) { 557 if (!isValidDictionary()) return false; 558 return needsToRunGCNative(mNativeDict, mindsBlockByGC); 559 } 560 561 public boolean migrateTo(final int newFormatVersion) { 562 if (!isValidDictionary()) { 563 return false; 564 } 565 final File isMigratingDir = 566 new File(mDictFilePath + DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION); 567 if (isMigratingDir.exists()) { 568 isMigratingDir.delete(); 569 Log.e(TAG, "Previous migration attempt failed probably due to a crash. " 570 + "Giving up using the old dictionary (" + mDictFilePath + ")."); 571 return false; 572 } 573 if (!isMigratingDir.mkdir()) { 574 Log.e(TAG, "Cannot create a dir (" + isMigratingDir.getAbsolutePath() 575 + ") to record migration."); 576 return false; 577 } 578 try { 579 final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; 580 if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { 581 return false; 582 } 583 close(); 584 final File dictFile = new File(mDictFilePath); 585 final File tmpDictFile = new File(tmpDictFilePath); 586 if (!FileUtils.deleteRecursively(dictFile)) { 587 return false; 588 } 589 if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { 590 return false; 591 } 592 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 593 dictFile.length(), mIsUpdatable); 594 return true; 595 } finally { 596 isMigratingDir.delete(); 597 } 598 } 599 600 @UsedForTesting 601 public String getPropertyForTest(final String query) { 602 if (!isValidDictionary()) return ""; 603 return getPropertyNative(mNativeDict, query); 604 } 605 606 @Override 607 public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { 608 return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; 609 } 610 611 @Override 612 public void close() { 613 synchronized (mDicTraverseSessions) { 614 final int sessionsSize = mDicTraverseSessions.size(); 615 for (int index = 0; index < sessionsSize; ++index) { 616 final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index); 617 if (traverseSession != null) { 618 traverseSession.close(); 619 } 620 } 621 mDicTraverseSessions.clear(); 622 } 623 closeInternalLocked(); 624 } 625 626 private synchronized void closeInternalLocked() { 627 if (mNativeDict != 0) { 628 closeNative(mNativeDict); 629 mNativeDict = 0; 630 } 631 } 632 633 // TODO: Manage BinaryDictionary instances without using WeakReference or something. 634 @Override 635 protected void finalize() throws Throwable { 636 try { 637 closeInternalLocked(); 638 } finally { 639 super.finalize(); 640 } 641 } 642} 643