BinaryDictionary.java revision 4e0af43673936c336fff298bc9aeee1d75c3bc92
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.text.TextUtils; 20import android.util.Log; 21import android.util.SparseArray; 22 23import com.android.inputmethod.annotations.UsedForTesting; 24import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 25import com.android.inputmethod.latin.common.ComposedData; 26import com.android.inputmethod.latin.common.Constants; 27import com.android.inputmethod.latin.common.FileUtils; 28import com.android.inputmethod.latin.common.InputPointers; 29import com.android.inputmethod.latin.common.StringUtils; 30import com.android.inputmethod.latin.define.DecoderSpecificConstants; 31import com.android.inputmethod.latin.makedict.DictionaryHeader; 32import com.android.inputmethod.latin.makedict.FormatSpec; 33import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; 34import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 35import com.android.inputmethod.latin.makedict.WordProperty; 36import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 37import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; 38import com.android.inputmethod.latin.utils.JniUtils; 39import com.android.inputmethod.latin.utils.WordInputEventForPersonalization; 40 41import java.io.File; 42import java.util.ArrayList; 43import java.util.Arrays; 44import java.util.HashMap; 45import java.util.Locale; 46import java.util.Map; 47 48import javax.annotation.Nonnull; 49 50/** 51 * Implements a static, compacted, binary dictionary of standard words. 52 */ 53// TODO: All methods which should be locked need to have a suffix "Locked". 54public final class BinaryDictionary extends Dictionary { 55 private static final String TAG = BinaryDictionary.class.getSimpleName(); 56 57 // The cutoff returned by native for auto-commit confidence. 58 // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h 59 private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; 60 61 @UsedForTesting 62 public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 63 @UsedForTesting 64 public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 65 @UsedForTesting 66 public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT"; 67 @UsedForTesting 68 public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; 69 70 public static final int NOT_A_VALID_TIMESTAMP = -1; 71 72 // Format to get unigram flags from native side via getWordPropertyNative(). 73 private static final int FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT = 5; 74 private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; 75 private static final int FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX = 1; 76 private static final int FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX = 2; 77 private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; // DEPRECATED 78 private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; 79 80 // Format to get probability and historical info from native side via getWordPropertyNative(). 81 public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4; 82 public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0; 83 public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1; 84 public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2; 85 public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3; 86 87 public static final String DICT_FILE_NAME_SUFFIX_FOR_MIGRATION = ".migrate"; 88 public static final String DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION = ".migrating"; 89 90 private long mNativeDict; 91 private final long mDictSize; 92 private final String mDictFilePath; 93 private final boolean mUseFullEditDistance; 94 private final boolean mIsUpdatable; 95 private boolean mHasUpdated; 96 97 private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); 98 99 // TODO: There should be a way to remove used DicTraverseSession objects from 100 // {@code mDicTraverseSessions}. 101 private DicTraverseSession getTraverseSession(final int traverseSessionId) { 102 synchronized(mDicTraverseSessions) { 103 DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); 104 if (traverseSession == null) { 105 traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); 106 mDicTraverseSessions.put(traverseSessionId, traverseSession); 107 } 108 return traverseSession; 109 } 110 } 111 112 /** 113 * Constructs binary dictionary using existing dictionary file. 114 * @param filename the name of the file to read through native code. 115 * @param offset the offset of the dictionary data within the file. 116 * @param length the length of the binary data. 117 * @param useFullEditDistance whether to use the full edit distance in suggestions 118 * @param dictType the dictionary type, as a human-readable string 119 * @param isUpdatable whether to open the dictionary file in writable mode. 120 */ 121 public BinaryDictionary(final String filename, final long offset, final long length, 122 final boolean useFullEditDistance, final Locale locale, final String dictType, 123 final boolean isUpdatable) { 124 super(dictType, locale); 125 mDictSize = length; 126 mDictFilePath = filename; 127 mIsUpdatable = isUpdatable; 128 mHasUpdated = false; 129 mUseFullEditDistance = useFullEditDistance; 130 loadDictionary(filename, offset, length, isUpdatable); 131 } 132 133 /** 134 * Constructs binary dictionary on memory. 135 * @param filename the name of the file used to flush. 136 * @param useFullEditDistance whether to use the full edit distance in suggestions 137 * @param dictType the dictionary type, as a human-readable string 138 * @param formatVersion the format version of the dictionary 139 * @param attributeMap the attributes of the dictionary 140 */ 141 public BinaryDictionary(final String filename, final boolean useFullEditDistance, 142 final Locale locale, final String dictType, final long formatVersion, 143 final Map<String, String> attributeMap) { 144 super(dictType, locale); 145 mDictSize = 0; 146 mDictFilePath = filename; 147 // On memory dictionary is always updatable. 148 mIsUpdatable = true; 149 mHasUpdated = false; 150 mUseFullEditDistance = useFullEditDistance; 151 final String[] keyArray = new String[attributeMap.size()]; 152 final String[] valueArray = new String[attributeMap.size()]; 153 int index = 0; 154 for (final String key : attributeMap.keySet()) { 155 keyArray[index] = key; 156 valueArray[index] = attributeMap.get(key); 157 index++; 158 } 159 mNativeDict = createOnMemoryNative(formatVersion, locale.toString(), keyArray, valueArray); 160 } 161 162 163 static { 164 JniUtils.loadNativeLibrary(); 165 } 166 167 private static native long openNative(String sourceDir, long dictOffset, long dictSize, 168 boolean isUpdatable); 169 private static native long createOnMemoryNative(long formatVersion, 170 String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray); 171 private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, 172 int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, 173 ArrayList<int[]> outAttributeValues); 174 private static native boolean flushNative(long dict, String filePath); 175 private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); 176 private static native boolean flushWithGCNative(long dict, String filePath); 177 private static native void closeNative(long dict); 178 private static native int getFormatVersionNative(long dict); 179 private static native int getProbabilityNative(long dict, int[] word); 180 private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); 181 private static native int getNgramProbabilityNative(long dict, int[][] prevWordCodePointArrays, 182 boolean[] isBeginningOfSentenceArray, int[] word); 183 private static native void getWordPropertyNative(long dict, int[] word, 184 boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, 185 int[] outProbabilityInfo, ArrayList<int[][]> outNgramPrevWordsArray, 186 ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray, 187 ArrayList<int[]> outNgramTargets, ArrayList<int[]> outNgramProbabilityInfo, 188 ArrayList<int[]> outShortcutTargets, ArrayList<Integer> outShortcutProbabilities); 189 private static native int getNextWordNative(long dict, int token, int[] outCodePoints, 190 boolean[] outIsBeginningOfSentence); 191 private static native void getSuggestionsNative(long dict, long proximityInfo, 192 long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, 193 int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, 194 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 195 int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints, 196 int[] outputScores, int[] outputIndices, int[] outputTypes, 197 int[] outputAutoCommitFirstWordConfidence, 198 float[] inOutWeightOfLangModelVsSpatialModel); 199 private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, 200 int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, 201 boolean isNotAWord, boolean isPossiblyOffensive, int timestamp); 202 private static native boolean removeUnigramEntryNative(long dict, int[] word); 203 private static native boolean addNgramEntryNative(long dict, 204 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 205 int[] word, int probability, int timestamp); 206 private static native boolean removeNgramEntryNative(long dict, 207 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); 208 private static native boolean updateEntriesForWordWithNgramContextNative(long dict, 209 int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 210 int[] word, boolean isValidWord, int count, int timestamp); 211 private static native int updateEntriesForInputEventsNative(long dict, 212 WordInputEventForPersonalization[] inputEvents, int startIndex); 213 private static native String getPropertyNative(long dict, String query); 214 private static native boolean isCorruptedNative(long dict); 215 private static native boolean migrateNative(long dict, String dictFilePath, 216 long newFormatVersion); 217 218 // TODO: Move native dict into session 219 private void loadDictionary(final String path, final long startOffset, 220 final long length, final boolean isUpdatable) { 221 mHasUpdated = false; 222 mNativeDict = openNative(path, startOffset, length, isUpdatable); 223 } 224 225 // TODO: Check isCorrupted() for main dictionaries. 226 public boolean isCorrupted() { 227 if (!isValidDictionary()) { 228 return false; 229 } 230 if (!isCorruptedNative(mNativeDict)) { 231 return false; 232 } 233 // TODO: Record the corruption. 234 Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted."); 235 Log.e(TAG, "locale: " + mLocale); 236 Log.e(TAG, "dict size: " + mDictSize); 237 Log.e(TAG, "updatable: " + mIsUpdatable); 238 return true; 239 } 240 241 public DictionaryHeader getHeader() throws UnsupportedFormatException { 242 if (mNativeDict == 0) { 243 return null; 244 } 245 final int[] outHeaderSize = new int[1]; 246 final int[] outFormatVersion = new int[1]; 247 final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); 248 final ArrayList<int[]> outAttributeValues = new ArrayList<>(); 249 getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, 250 outAttributeValues); 251 final HashMap<String, String> attributes = new HashMap<>(); 252 for (int i = 0; i < outAttributeKeys.size(); i++) { 253 final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( 254 outAttributeKeys.get(i)); 255 final String attributeValue = StringUtils.getStringFromNullTerminatedCodePointArray( 256 outAttributeValues.get(i)); 257 attributes.put(attributeKey, attributeValue); 258 } 259 final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( 260 attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); 261 return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), 262 new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); 263 } 264 265 @Override 266 public ArrayList<SuggestedWordInfo> getSuggestions(final ComposedData composedData, 267 final NgramContext ngramContext, final long proximityInfoHandle, 268 final SettingsValuesForSuggestion settingsValuesForSuggestion, 269 final int sessionId, final float weightForLocale, 270 final float[] inOutWeightOfLangModelVsSpatialModel) { 271 if (!isValidDictionary()) { 272 return null; 273 } 274 final DicTraverseSession session = getTraverseSession(sessionId); 275 Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); 276 ngramContext.outputToArray(session.mPrevWordCodePointArrays, 277 session.mIsBeginningOfSentenceArray); 278 final InputPointers inputPointers = composedData.mInputPointers; 279 final boolean isGesture = composedData.mIsBatchMode; 280 final int inputSize; 281 if (!isGesture) { 282 inputSize = 283 composedData.copyCodePointsExceptTrailingSingleQuotesAndReturnCodePointCount( 284 session.mInputCodePoints); 285 if (inputSize < 0) { 286 return null; 287 } 288 } else { 289 inputSize = inputPointers.getPointerSize(); 290 } 291 session.mNativeSuggestOptions.setUseFullEditDistance(mUseFullEditDistance); 292 session.mNativeSuggestOptions.setIsGesture(isGesture); 293 session.mNativeSuggestOptions.setBlockOffensiveWords( 294 settingsValuesForSuggestion.mBlockPotentiallyOffensive); 295 session.mNativeSuggestOptions.setWeightForLocale(weightForLocale); 296 if (inOutWeightOfLangModelVsSpatialModel != null) { 297 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 298 inOutWeightOfLangModelVsSpatialModel[0]; 299 } else { 300 session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 301 Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL; 302 } 303 // TOOD: Pass multiple previous words information for n-gram. 304 getSuggestionsNative(mNativeDict, proximityInfoHandle, 305 getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), 306 inputPointers.getYCoordinates(), inputPointers.getTimes(), 307 inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, 308 session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays, 309 session.mIsBeginningOfSentenceArray, ngramContext.getPrevWordCount(), 310 session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, 311 session.mSpaceIndices, session.mOutputTypes, 312 session.mOutputAutoCommitFirstWordConfidence, 313 session.mInputOutputWeightOfLangModelVsSpatialModel); 314 if (inOutWeightOfLangModelVsSpatialModel != null) { 315 inOutWeightOfLangModelVsSpatialModel[0] = 316 session.mInputOutputWeightOfLangModelVsSpatialModel[0]; 317 } 318 final int count = session.mOutputSuggestionCount[0]; 319 final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); 320 for (int j = 0; j < count; ++j) { 321 final int start = j * DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH; 322 int len = 0; 323 while (len < DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH 324 && session.mOutputCodePoints[start + len] != 0) { 325 ++len; 326 } 327 if (len > 0) { 328 suggestions.add(new SuggestedWordInfo( 329 new String(session.mOutputCodePoints, start, len), 330 (int)(session.mOutputScores[j] * weightForLocale), session.mOutputTypes[j], 331 this /* sourceDict */, 332 session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, 333 session.mOutputAutoCommitFirstWordConfidence[0])); 334 } 335 } 336 return suggestions; 337 } 338 339 public boolean isValidDictionary() { 340 return mNativeDict != 0; 341 } 342 343 public int getFormatVersion() { 344 return getFormatVersionNative(mNativeDict); 345 } 346 347 @Override 348 public boolean isInDictionary(final String word) { 349 return getFrequency(word) != NOT_A_PROBABILITY; 350 } 351 352 @Override 353 public int getFrequency(final String word) { 354 if (TextUtils.isEmpty(word)) { 355 return NOT_A_PROBABILITY; 356 } 357 final int[] codePoints = StringUtils.toCodePointArray(word); 358 return getProbabilityNative(mNativeDict, codePoints); 359 } 360 361 @Override 362 public int getMaxFrequencyOfExactMatches(final String word) { 363 if (TextUtils.isEmpty(word)) { 364 return NOT_A_PROBABILITY; 365 } 366 final int[] codePoints = StringUtils.toCodePointArray(word); 367 return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); 368 } 369 370 @UsedForTesting 371 public boolean isValidNgram(final NgramContext ngramContext, final String word) { 372 return getNgramProbability(ngramContext, word) != NOT_A_PROBABILITY; 373 } 374 375 public int getNgramProbability(final NgramContext ngramContext, final String word) { 376 if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 377 return NOT_A_PROBABILITY; 378 } 379 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 380 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 381 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 382 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 383 return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays, 384 isBeginningOfSentenceArray, wordCodePoints); 385 } 386 387 public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { 388 if (word == null) { 389 return null; 390 } 391 final int[] codePoints = StringUtils.toCodePointArray(word); 392 final int[] outCodePoints = new int[DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH]; 393 final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; 394 final int[] outProbabilityInfo = 395 new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; 396 final ArrayList<int[][]> outNgramPrevWordsArray = new ArrayList<>(); 397 final ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray = 398 new ArrayList<>(); 399 final ArrayList<int[]> outNgramTargets = new ArrayList<>(); 400 final ArrayList<int[]> outNgramProbabilityInfo = new ArrayList<>(); 401 final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); 402 final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); 403 getWordPropertyNative(mNativeDict, codePoints, isBeginningOfSentence, outCodePoints, 404 outFlags, outProbabilityInfo, outNgramPrevWordsArray, 405 outNgramPrevWordIsBeginningOfSentenceArray, outNgramTargets, 406 outNgramProbabilityInfo, outShortcutTargets, outShortcutProbabilities); 407 return new WordProperty(codePoints, 408 outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], 409 outFlags[FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX], 410 outFlags[FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX], 411 outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, 412 outNgramPrevWordsArray, outNgramPrevWordIsBeginningOfSentenceArray, 413 outNgramTargets, outNgramProbabilityInfo); 414 } 415 416 public static class GetNextWordPropertyResult { 417 public WordProperty mWordProperty; 418 public int mNextToken; 419 420 public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { 421 mWordProperty = wordProperty; 422 mNextToken = nextToken; 423 } 424 } 425 426 /** 427 * Method to iterate all words in the dictionary for makedict. 428 * If token is 0, this method newly starts iterating the dictionary. 429 */ 430 public GetNextWordPropertyResult getNextWordProperty(final int token) { 431 final int[] codePoints = new int[DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH]; 432 final boolean[] isBeginningOfSentence = new boolean[1]; 433 final int nextToken = getNextWordNative(mNativeDict, token, codePoints, 434 isBeginningOfSentence); 435 final String word = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); 436 return new GetNextWordPropertyResult( 437 getWordProperty(word, isBeginningOfSentence[0]), nextToken); 438 } 439 440 // Add a unigram entry to binary dictionary with unigram attributes in native code. 441 public boolean addUnigramEntry( 442 final String word, final int probability, final boolean isBeginningOfSentence, 443 final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { 444 if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { 445 return false; 446 } 447 final int[] codePoints = StringUtils.toCodePointArray(word); 448 if (!addUnigramEntryNative(mNativeDict, codePoints, probability, 449 null /* shortcutTargetCodePoints */, 0 /* shortcutProbability */, 450 isBeginningOfSentence, isNotAWord, isPossiblyOffensive, timestamp)) { 451 return false; 452 } 453 mHasUpdated = true; 454 return true; 455 } 456 457 // Remove a unigram entry from the binary dictionary in native code. 458 public boolean removeUnigramEntry(final String word) { 459 if (TextUtils.isEmpty(word)) { 460 return false; 461 } 462 final int[] codePoints = StringUtils.toCodePointArray(word); 463 if (!removeUnigramEntryNative(mNativeDict, codePoints)) { 464 return false; 465 } 466 mHasUpdated = true; 467 return true; 468 } 469 470 // Add an n-gram entry to the binary dictionary with timestamp in native code. 471 public boolean addNgramEntry(final NgramContext ngramContext, final String word, 472 final int probability, final int timestamp) { 473 if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 474 return false; 475 } 476 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 477 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 478 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 479 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 480 if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays, 481 isBeginningOfSentenceArray, wordCodePoints, probability, timestamp)) { 482 return false; 483 } 484 mHasUpdated = true; 485 return true; 486 } 487 488 // Update entries for the word occurrence with the ngramContext. 489 public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext, 490 final String word, final boolean isValidWord, final int count, final int timestamp) { 491 if (TextUtils.isEmpty(word)) { 492 return false; 493 } 494 final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 495 final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 496 ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 497 final int[] wordCodePoints = StringUtils.toCodePointArray(word); 498 if (!updateEntriesForWordWithNgramContextNative(mNativeDict, prevWordCodePointArrays, 499 isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { 500 return false; 501 } 502 mHasUpdated = true; 503 return true; 504 } 505 506 @UsedForTesting 507 public void updateEntriesForInputEvents(final WordInputEventForPersonalization[] inputEvents) { 508 if (!isValidDictionary()) { 509 return; 510 } 511 int processedEventCount = 0; 512 while (processedEventCount < inputEvents.length) { 513 if (needsToRunGC(true /* mindsBlockByGC */)) { 514 flushWithGC(); 515 } 516 processedEventCount = updateEntriesForInputEventsNative(mNativeDict, inputEvents, 517 processedEventCount); 518 mHasUpdated = true; 519 if (processedEventCount <= 0) { 520 return; 521 } 522 } 523 } 524 525 private void reopen() { 526 close(); 527 final File dictFile = new File(mDictFilePath); 528 // WARNING: Because we pass 0 as the offset and file.length() as the length, this can 529 // only be called for actual files. Right now it's only called by the flush() family of 530 // functions, which require an updatable dictionary, so it's okay. But beware. 531 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 532 dictFile.length(), mIsUpdatable); 533 } 534 535 // Flush to dict file if the dictionary has been updated. 536 public boolean flush() { 537 if (!isValidDictionary()) { 538 return false; 539 } 540 if (mHasUpdated) { 541 if (!flushNative(mNativeDict, mDictFilePath)) { 542 return false; 543 } 544 reopen(); 545 } 546 return true; 547 } 548 549 // Run GC and flush to dict file if the dictionary has been updated. 550 public boolean flushWithGCIfHasUpdated() { 551 if (mHasUpdated) { 552 return flushWithGC(); 553 } 554 return true; 555 } 556 557 // Run GC and flush to dict file. 558 public boolean flushWithGC() { 559 if (!isValidDictionary()) { 560 return false; 561 } 562 if (!flushWithGCNative(mNativeDict, mDictFilePath)) { 563 return false; 564 } 565 reopen(); 566 return true; 567 } 568 569 /** 570 * Checks whether GC is needed to run or not. 571 * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about 572 * the blocking in some situations such as in idle time or just before closing. 573 * @return whether GC is needed to run or not. 574 */ 575 public boolean needsToRunGC(final boolean mindsBlockByGC) { 576 if (!isValidDictionary()) { 577 return false; 578 } 579 return needsToRunGCNative(mNativeDict, mindsBlockByGC); 580 } 581 582 public boolean migrateTo(final int newFormatVersion) { 583 if (!isValidDictionary()) { 584 return false; 585 } 586 final File isMigratingDir = 587 new File(mDictFilePath + DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION); 588 if (isMigratingDir.exists()) { 589 isMigratingDir.delete(); 590 Log.e(TAG, "Previous migration attempt failed probably due to a crash. " 591 + "Giving up using the old dictionary (" + mDictFilePath + ")."); 592 return false; 593 } 594 if (!isMigratingDir.mkdir()) { 595 Log.e(TAG, "Cannot create a dir (" + isMigratingDir.getAbsolutePath() 596 + ") to record migration."); 597 return false; 598 } 599 try { 600 final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; 601 if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { 602 return false; 603 } 604 close(); 605 final File dictFile = new File(mDictFilePath); 606 final File tmpDictFile = new File(tmpDictFilePath); 607 if (!FileUtils.deleteRecursively(dictFile)) { 608 return false; 609 } 610 if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { 611 return false; 612 } 613 loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 614 dictFile.length(), mIsUpdatable); 615 return true; 616 } finally { 617 isMigratingDir.delete(); 618 } 619 } 620 621 @UsedForTesting 622 public String getPropertyForGettingStats(final String query) { 623 if (!isValidDictionary()) { 624 return ""; 625 } 626 return getPropertyNative(mNativeDict, query); 627 } 628 629 @Override 630 public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { 631 return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; 632 } 633 634 @Override 635 public void close() { 636 synchronized (mDicTraverseSessions) { 637 final int sessionsSize = mDicTraverseSessions.size(); 638 for (int index = 0; index < sessionsSize; ++index) { 639 final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index); 640 if (traverseSession != null) { 641 traverseSession.close(); 642 } 643 } 644 mDicTraverseSessions.clear(); 645 } 646 closeInternalLocked(); 647 } 648 649 private synchronized void closeInternalLocked() { 650 if (mNativeDict != 0) { 651 closeNative(mNativeDict); 652 mNativeDict = 0; 653 } 654 } 655 656 // TODO: Manage BinaryDictionary instances without using WeakReference or something. 657 @Override 658 protected void finalize() throws Throwable { 659 try { 660 closeInternalLocked(); 661 } finally { 662 super.finalize(); 663 } 664 } 665} 666