BinaryDictionary.java revision 29500ef4ba8e01f4c467a62399c8249d532ee82c
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.text.TextUtils; 20import android.util.Log; 21import android.util.SparseArray; 22 23import com.android.inputmethod.annotations.UsedForTesting; 24import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 25import com.android.inputmethod.latin.common.ComposedData; 26import com.android.inputmethod.latin.common.Constants; 27import com.android.inputmethod.latin.common.FileUtils; 28import com.android.inputmethod.latin.common.InputPointers; 29import com.android.inputmethod.latin.common.StringUtils; 30import com.android.inputmethod.latin.define.DecoderSpecificConstants; 31import com.android.inputmethod.latin.makedict.DictionaryHeader; 32import com.android.inputmethod.latin.makedict.FormatSpec; 33import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; 34import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 35import com.android.inputmethod.latin.makedict.WordProperty; 36import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 37import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; 38import com.android.inputmethod.latin.utils.JniUtils; 39import com.android.inputmethod.latin.utils.WordInputEventForPersonalization; 40 41import java.io.File; 42import java.util.ArrayList; 43import java.util.Arrays; 44import java.util.HashMap; 45import java.util.Locale; 46import java.util.Map; 47 48import javax.annotation.Nonnull; 49 50/** 51 * Implements a static, compacted, binary dictionary of standard words. 52 */ 53// TODO: All methods which should be locked need to have a suffix "Locked". 54public final class BinaryDictionary extends Dictionary { 55 private static final String TAG = BinaryDictionary.class.getSimpleName(); 56 57 // The cutoff returned by native for auto-commit confidence. 58 // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h 59 private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; 60 61 static final int DICTIONARY_MAX_WORD_LENGTH = 48; 62 63 @UsedForTesting 64 public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 65 @UsedForTesting 66 public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 67 @UsedForTesting 68 public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT"; 69 @UsedForTesting 70 public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; 71 72 public static final int NOT_A_VALID_TIMESTAMP = -1; 73 74 // Format to get unigram flags from native side via getWordPropertyNative(). 75 private static final int FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT = 5; 76 private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; 77 private static final int FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX = 1; 78 private static final int FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX = 2; 79 private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; // DEPRECATED 80 private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; 81 82 // Format to get probability and historical info from native side via getWordPropertyNative(). 83 public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4; 84 public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0; 85 public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1; 86 public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2; 87 public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3; 88 89 public static final String DICT_FILE_NAME_SUFFIX_FOR_MIGRATION = ".migrate"; 90 public static final String DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION = ".migrating"; 91 92 private long mNativeDict; 93 private final long mDictSize; 94 private final String mDictFilePath; 95 private final boolean mUseFullEditDistance; 96 private final boolean mIsUpdatable; 97 private boolean mHasUpdated; 98 99 private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); 100 101 // TODO: There should be a way to remove used DicTraverseSession objects from 102 // {@code mDicTraverseSessions}. 103 private DicTraverseSession getTraverseSession(final int traverseSessionId) { 104 synchronized(mDicTraverseSessions) { 105 DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); 106 if (traverseSession == null) { 107 traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); 108 mDicTraverseSessions.put(traverseSessionId, traverseSession); 109 } 110 return traverseSession; 111 } 112 } 113 114 /** 115 * Constructs binary dictionary using existing dictionary file. 116 * @param filename the name of the file to read through native code. 117 * @param offset the offset of the dictionary data within the file. 118 * @param length the length of the binary data. 119 * @param useFullEditDistance whether to use the full edit distance in suggestions 120 * @param dictType the dictionary type, as a human-readable string 121 * @param isUpdatable whether to open the dictionary file in writable mode. 122 */ 123 public BinaryDictionary(final String filename, final long offset, final long length, 124 final boolean useFullEditDistance, final Locale locale, final String dictType, 125 final boolean isUpdatable) { 126 super(dictType, locale); 127 mDictSize = length; 128 mDictFilePath = filename; 129 mIsUpdatable = isUpdatable; 130 mHasUpdated = false; 131 mUseFullEditDistance = useFullEditDistance; 132 loadDictionary(filename, offset, length, isUpdatable); 133 } 134 135 /** 136 * Constructs binary dictionary on memory. 137 * @param filename the name of the file used to flush. 138 * @param useFullEditDistance whether to use the full edit distance in suggestions 139 * @param dictType the dictionary type, as a human-readable string 140 * @param formatVersion the format version of the dictionary 141 * @param attributeMap the attributes of the dictionary 142 */ 143 public BinaryDictionary(final String filename, final boolean useFullEditDistance, 144 final Locale locale, final String dictType, final long formatVersion, 145 final Map<String, String> attributeMap) { 146 super(dictType, locale); 147 mDictSize = 0; 148 mDictFilePath = filename; 149 // On memory dictionary is always updatable. 150 mIsUpdatable = true; 151 mHasUpdated = false; 152 mUseFullEditDistance = useFullEditDistance; 153 final String[] keyArray = new String[attributeMap.size()]; 154 final String[] valueArray = new String[attributeMap.size()]; 155 int index = 0; 156 for (final String key : attributeMap.keySet()) { 157 keyArray[index] = key; 158 valueArray[index] = attributeMap.get(key); 159 index++; 160 } 161 mNativeDict = createOnMemoryNative(formatVersion, locale.toString(), keyArray, valueArray); 162 } 163 164 165 static { 166 JniUtils.loadNativeLibrary(); 167 } 168 169 private static native long openNative(String sourceDir, long dictOffset, long dictSize, 170 boolean isUpdatable); 171 private static native long createOnMemoryNative(long formatVersion, 172 String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray); 173 private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, 174 int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, 175 ArrayList<int[]> outAttributeValues); 176 private static native boolean flushNative(long dict, String filePath); 177 private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); 178 private static native boolean flushWithGCNative(long dict, String filePath); 179 private static native void closeNative(long dict); 180 private static native int getFormatVersionNative(long dict); 181 private static native int getProbabilityNative(long dict, int[] word); 182 private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); 183 private static native int getNgramProbabilityNative(long dict, int[][] prevWordCodePointArrays, 184 boolean[] isBeginningOfSentenceArray, int[] word); 185 private static native void getWordPropertyNative(long dict, int[] word, 186 boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, 187 int[] outProbabilityInfo, ArrayList<int[][]> outNgramPrevWordsArray, 188 ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray, 189 ArrayList<int[]> outNgramTargets, ArrayList<int[]> outNgramProbabilityInfo, 190 ArrayList<int[]> outShortcutTargets, ArrayList<Integer> outShortcutProbabilities); 191 private static native int getNextWordNative(long dict, int token, int[] outCodePoints, 192 boolean[] outIsBeginningOfSentence); 193 private static native void getSuggestionsNative(long dict, long proximityInfo, 194 long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, 195 int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, 196 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 197 int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints, 198 int[] outputScores, int[] outputIndices, int[] outputTypes, 199 int[] outputAutoCommitFirstWordConfidence, 200 float[] inOutWeightOfLangModelVsSpatialModel); 201 private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, 202 int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, 203 boolean isNotAWord, boolean isPossiblyOffensive, int timestamp); 204 private static native boolean removeUnigramEntryNative(long dict, int[] word); 205 private static native boolean addNgramEntryNative(long dict, 206 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 207 int[] word, int probability, int timestamp); 208 private static native boolean removeNgramEntryNative(long dict, 209 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); 210 private static native boolean updateEntriesForWordWithNgramContextNative(long dict, 211 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 212 int[] word, boolean isValidWord, int count, int timestamp); 213 private static native int updateEntriesForInputEventsNative(long dict, 214 WordInputEventForPersonalization[] inputEvents, int startIndex); 215 private static native String getPropertyNative(long dict, String query); 216 private static native boolean isCorruptedNative(long dict); 217 private static native boolean migrateNative(long dict, String dictFilePath, 218 long newFormatVersion); 219 220 // TODO: Move native dict into session 221 private void loadDictionary(final String path, final long startOffset, 222 final long length, final boolean isUpdatable) { 223 mHasUpdated = false; 224 mNativeDict = openNative(path, startOffset, length, isUpdatable); 225 } 226 227 // TODO: Check isCorrupted() for main dictionaries. 228 public boolean isCorrupted() { 229 if (!isValidDictionary()) { 230 return false; 231 } 232 if (!isCorruptedNative(mNativeDict)) { 233 return false; 234 } 235 // TODO: Record the corruption. 236 Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted."); 237 Log.e(TAG, "locale: " + mLocale); 238 Log.e(TAG, "dict size: " + mDictSize); 239 Log.e(TAG, "updatable: " + mIsUpdatable); 240 return true; 241 } 242 243 public DictionaryHeader getHeader() throws UnsupportedFormatException { 244 if (mNativeDict == 0) { 245 return null; 246 } 247 final int[] outHeaderSize = new int[1]; 248 final int[] outFormatVersion = new int[1]; 249 final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); 250 final ArrayList<int[]> outAttributeValues = new ArrayList<>(); 251 getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, 252 outAttributeValues); 253 final HashMap<String, String> attributes = new HashMap<>(); 254 for (int i = 0; i < outAttributeKeys.size(); i++) { 255 final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( 256 outAttributeKeys.get(i)); 257 final String attributeValue = StringUtils.getStringFromNullTerminatedCodePointArray( 258 outAttributeValues.get(i)); 259 attributes.put(attributeKey, attributeValue); 260 } 261 final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( 262 attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); 263 return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), 264 new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); 265 } 266 267 @Override 268 public ArrayList<SuggestedWordInfo> getSuggestions(final ComposedData composedData, 269 final NgramContext ngramContext, final long proximityInfoHandle, 270 final SettingsValuesForSuggestion settingsValuesForSuggestion, 271 final int sessionId, final float weightForLocale, 272 final float[] inOutWeightOfLangModelVsSpatialModel) { 273 if (!isValidDictionary()) { 274 return null; 275 } 276 final DicTraverseSession session = getTraverseSession(sessionId); 277 Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); 278 ngramContext.outputToArray(session.mPrevWordCodePointArrays, 279 session.mIsBeginningOfSentenceArray); 280 final InputPointers inputPointers = composedData.mInputPointers; 281 final boolean isGesture = composedData.mIsBatchMode; 282 final int inputSize; 283 if (!isGesture) { 284 inputSize = 285 composedData.copyCodePointsExceptTrailingSingleQuotesAndReturnCodePointCount( 286 session.mInputCodePoints); 287 if (inputSize < 0) { 288 return null; 289 } 290 } else { 291 inputSize = inputPointers.getPointerSize(); 292 } 293 session.mNativeSuggestOptions.setUseFullEditDistance(mUseFullEditDistance); 294 session.mNativeSuggestOptions.setIsGesture(isGesture); 295 session.mNativeSuggestOptions.setBlockOffensiveWords( 296 settingsValuesForSuggestion.mBlockPotentiallyOffensive); 297 session.mNativeSuggestOptions.setWeightForLocale(weightForLocale); 298 if (inOutWeightOfLangModelVsSpatialModel != null) { 299 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 300 inOutWeightOfLangModelVsSpatialModel[0]; 301 } else { 302 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 303 Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL; 304 } 305 // TOOD: Pass multiple previous words information for n-gram. 306 getSuggestionsNative(mNativeDict, proximityInfoHandle, 307 getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), 308 inputPointers.getYCoordinates(), inputPointers.getTimes(), 309 inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, 310 session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays, 311 session.mIsBeginningOfSentenceArray, ngramContext.getPrevWordCount(), 312 session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, 313 session.mSpaceIndices, session.mOutputTypes, 314 session.mOutputAutoCommitFirstWordConfidence, 315 session.mInputOutputWeightOfLangModelVsSpatialModel); 316 if (inOutWeightOfLangModelVsSpatialModel != null) { 317 inOutWeightOfLangModelVsSpatialModel[0] = 318 session.mInputOutputWeightOfLangModelVsSpatialModel[0]; 319 } 320 final int count = session.mOutputSuggestionCount[0]; 321 final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); 322 for (int j = 0; j < count; ++j) { 323 final int start = j * DICTIONARY_MAX_WORD_LENGTH; 324 int len = 0; 325 while (len < DICTIONARY_MAX_WORD_LENGTH 326 && session.mOutputCodePoints[start + len] != 0) { 327 ++len; 328 } 329 if (len > 0) { 330 suggestions.add(new SuggestedWordInfo( 331 new String(session.mOutputCodePoints, start, len), 332 (int)(session.mOutputScores[j] * weightForLocale), session.mOutputTypes[j], 333 this /* sourceDict */, 334 session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, 335 session.mOutputAutoCommitFirstWordConfidence[0])); 336 } 337 } 338 return suggestions; 339 } 340 341 public boolean isValidDictionary() { 342 return mNativeDict != 0; 343 } 344 345 public int getFormatVersion() { 346 return getFormatVersionNative(mNativeDict); 347 } 348 349 @Override 350 public boolean isInDictionary(final String word) { 351 return getFrequency(word) != NOT_A_PROBABILITY; 352 } 353 354 @Override 355 public int getFrequency(final String word) { 356 if (TextUtils.isEmpty(word)) { 357 return NOT_A_PROBABILITY; 358 } 359 final int[] codePoints = StringUtils.toCodePointArray(word); 360 return getProbabilityNative(mNativeDict, codePoints); 361 } 362 363 @Override 364 public int getMaxFrequencyOfExactMatches(final String word) { 365 if (TextUtils.isEmpty(word)) { 366 return NOT_A_PROBABILITY; 367 } 368 final int[] codePoints = StringUtils.toCodePointArray(word); 369 return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); 370 } 371 372 @UsedForTesting 373 public boolean isValidNgram(final NgramContext ngramContext, final String word) { 374 return getNgramProbability(ngramContext, word) != NOT_A_PROBABILITY; 375 } 376 377 public int getNgramProbability(final NgramContext ngramContext, final String word) { 378 if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 379 return NOT_A_PROBABILITY; 380 } 381 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 382 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 383 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 384 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 385 return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays, 386 isBeginningOfSentenceArray, wordCodePoints); 387 } 388 389 public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { 390 if (word == null) { 391 return null; 392 } 393 final int[] codePoints = StringUtils.toCodePointArray(word); 394 final int[] outCodePoints = new int[DICTIONARY_MAX_WORD_LENGTH]; 395 final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; 396 final int[] outProbabilityInfo = 397 new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; 398 final ArrayList<int[][]> outNgramPrevWordsArray = new ArrayList<>(); 399 final ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray = 400 new ArrayList<>(); 401 final ArrayList<int[]> outNgramTargets = new ArrayList<>(); 402 final ArrayList<int[]> outNgramProbabilityInfo = new ArrayList<>(); 403 final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); 404 final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); 405 getWordPropertyNative(mNativeDict, codePoints, isBeginningOfSentence, outCodePoints, 406 outFlags, outProbabilityInfo, outNgramPrevWordsArray, 407 outNgramPrevWordIsBeginningOfSentenceArray, outNgramTargets, 408 outNgramProbabilityInfo, outShortcutTargets, outShortcutProbabilities); 409 return new WordProperty(codePoints, 410 outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], 411 outFlags[FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX], 412 outFlags[FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX], 413 outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, 414 outNgramPrevWordsArray, outNgramPrevWordIsBeginningOfSentenceArray, 415 outNgramTargets, outNgramProbabilityInfo); 416 } 417 418 public static class GetNextWordPropertyResult { 419 public WordProperty mWordProperty; 420 public int mNextToken; 421 422 public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { 423 mWordProperty = wordProperty; 424 mNextToken = nextToken; 425 } 426 } 427 428 /** 429 * Method to iterate all words in the dictionary for makedict. 430 * If token is 0, this method newly starts iterating the dictionary. 431 */ 432 public GetNextWordPropertyResult getNextWordProperty(final int token) { 433 final int[] codePoints = new int[DICTIONARY_MAX_WORD_LENGTH]; 434 final boolean[] isBeginningOfSentence = new boolean[1]; 435 final int nextToken = getNextWordNative(mNativeDict, token, codePoints, 436 isBeginningOfSentence); 437 final String word = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); 438 return new GetNextWordPropertyResult( 439 getWordProperty(word, isBeginningOfSentence[0]), nextToken); 440 } 441 442 // Add a unigram entry to binary dictionary with unigram attributes in native code. 443 public boolean addUnigramEntry( 444 final String word, final int probability, final boolean isBeginningOfSentence, 445 final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { 446 if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { 447 return false; 448 } 449 final int[] codePoints = StringUtils.toCodePointArray(word); 450 if (!addUnigramEntryNative(mNativeDict, codePoints, probability, 451 null /* shortcutTargetCodePoints */, 0 /* shortcutProbability */, 452 isBeginningOfSentence, isNotAWord, isPossiblyOffensive, timestamp)) { 453 return false; 454 } 455 mHasUpdated = true; 456 return true; 457 } 458 459 // Remove a unigram entry from the binary dictionary in native code. 460 public boolean removeUnigramEntry(final String word) { 461 if (TextUtils.isEmpty(word)) { 462 return false; 463 } 464 final int[] codePoints = StringUtils.toCodePointArray(word); 465 if (!removeUnigramEntryNative(mNativeDict, codePoints)) { 466 return false; 467 } 468 mHasUpdated = true; 469 return true; 470 } 471 472 // Add an n-gram entry to the binary dictionary with timestamp in native code. 473 public boolean addNgramEntry(final NgramContext ngramContext, final String word, 474 final int probability, final int timestamp) { 475 if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 476 return false; 477 } 478 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 479 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 480 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 481 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 482 if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays, 483 isBeginningOfSentenceArray, wordCodePoints, probability, timestamp)) { 484 return false; 485 } 486 mHasUpdated = true; 487 return true; 488 } 489 490 // Update entries for the word occurrence with the ngramContext. 491 public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext, 492 final String word, final boolean isValidWord, final int count, final int timestamp) { 493 if (TextUtils.isEmpty(word)) { 494 return false; 495 } 496 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 497 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 498 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 499 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 500 if (!updateEntriesForWordWithNgramContextNative(mNativeDict, prevWordCodePointArrays, 501 isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { 502 return false; 503 } 504 mHasUpdated = true; 505 return true; 506 } 507 508 @UsedForTesting 509 public void updateEntriesForInputEvents(final WordInputEventForPersonalization[] inputEvents) { 510 if (!isValidDictionary()) { 511 return; 512 } 513 int processedEventCount = 0; 514 while (processedEventCount < inputEvents.length) { 515 if (needsToRunGC(true /* mindsBlockByGC */)) { 516 flushWithGC(); 517 } 518 processedEventCount = updateEntriesForInputEventsNative(mNativeDict, inputEvents, 519 processedEventCount); 520 mHasUpdated = true; 521 if (processedEventCount <= 0) { 522 return; 523 } 524 } 525 } 526 527 private void reopen() { 528 close(); 529 final File dictFile = new File(mDictFilePath); 530 // WARNING: Because we pass 0 as the offset and file.length() as the length, this can 531 // only be called for actual files. Right now it's only called by the flush() family of 532 // functions, which require an updatable dictionary, so it's okay. But beware. 533 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 534 dictFile.length(), mIsUpdatable); 535 } 536 537 // Flush to dict file if the dictionary has been updated. 538 public boolean flush() { 539 if (!isValidDictionary()) { 540 return false; 541 } 542 if (mHasUpdated) { 543 if (!flushNative(mNativeDict, mDictFilePath)) { 544 return false; 545 } 546 reopen(); 547 } 548 return true; 549 } 550 551 // Run GC and flush to dict file if the dictionary has been updated. 552 public boolean flushWithGCIfHasUpdated() { 553 if (mHasUpdated) { 554 return flushWithGC(); 555 } 556 return true; 557 } 558 559 // Run GC and flush to dict file. 560 public boolean flushWithGC() { 561 if (!isValidDictionary()) { 562 return false; 563 } 564 if (!flushWithGCNative(mNativeDict, mDictFilePath)) { 565 return false; 566 } 567 reopen(); 568 return true; 569 } 570 571 /** 572 * Checks whether GC is needed to run or not. 573 * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about 574 * the blocking in some situations such as in idle time or just before closing. 575 * @return whether GC is needed to run or not. 576 */ 577 public boolean needsToRunGC(final boolean mindsBlockByGC) { 578 if (!isValidDictionary()) { 579 return false; 580 } 581 return needsToRunGCNative(mNativeDict, mindsBlockByGC); 582 } 583 584 public boolean migrateTo(final int newFormatVersion) { 585 if (!isValidDictionary()) { 586 return false; 587 } 588 final File isMigratingDir = 589 new File(mDictFilePath + DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION); 590 if (isMigratingDir.exists()) { 591 isMigratingDir.delete(); 592 Log.e(TAG, "Previous migration attempt failed probably due to a crash. " 593 + "Giving up using the old dictionary (" + mDictFilePath + ")."); 594 return false; 595 } 596 if (!isMigratingDir.mkdir()) { 597 Log.e(TAG, "Cannot create a dir (" + isMigratingDir.getAbsolutePath() 598 + ") to record migration."); 599 return false; 600 } 601 try { 602 final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; 603 if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { 604 return false; 605 } 606 close(); 607 final File dictFile = new File(mDictFilePath); 608 final File tmpDictFile = new File(tmpDictFilePath); 609 if (!FileUtils.deleteRecursively(dictFile)) { 610 return false; 611 } 612 if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { 613 return false; 614 } 615 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 616 dictFile.length(), mIsUpdatable); 617 return true; 618 } finally { 619 isMigratingDir.delete(); 620 } 621 } 622 623 @UsedForTesting 624 public String getPropertyForGettingStats(final String query) { 625 if (!isValidDictionary()) { 626 return ""; 627 } 628 return getPropertyNative(mNativeDict, query); 629 } 630 631 @Override 632 public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { 633 return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; 634 } 635 636 @Override 637 public void close() { 638 synchronized (mDicTraverseSessions) { 639 final int sessionsSize = mDicTraverseSessions.size(); 640 for (int index = 0; index < sessionsSize; ++index) { 641 final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index); 642 if (traverseSession != null) { 643 traverseSession.close(); 644 } 645 } 646 mDicTraverseSessions.clear(); 647 } 648 closeInternalLocked(); 649 } 650 651 private synchronized void closeInternalLocked() { 652 if (mNativeDict != 0) { 653 closeNative(mNativeDict); 654 mNativeDict = 0; 655 } 656 } 657 658 // TODO: Manage BinaryDictionary instances without using WeakReference or something. 659 @Override 660 protected void finalize() throws Throwable { 661 try { 662 closeInternalLocked(); 663 } finally { 664 super.finalize(); 665 } 666 } 667} 668