ExpandableBinaryDictionary.java revision 0cda0e8a9ceaeab5a0e918c4fc76f77770d89b2c
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.content.Context; 20import android.util.Log; 21 22import com.android.inputmethod.annotations.UsedForTesting; 23import com.android.inputmethod.keyboard.ProximityInfo; 24import com.android.inputmethod.latin.makedict.DictionaryHeader; 25import com.android.inputmethod.latin.makedict.FormatSpec; 26import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 27import com.android.inputmethod.latin.makedict.WordProperty; 28import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 29import com.android.inputmethod.latin.utils.AsyncResultHolder; 30import com.android.inputmethod.latin.utils.CollectionUtils; 31import com.android.inputmethod.latin.utils.CombinedFormatUtils; 32import com.android.inputmethod.latin.utils.FileUtils; 33import com.android.inputmethod.latin.utils.LanguageModelParam; 34import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; 35 36import java.io.File; 37import java.util.ArrayList; 38import java.util.HashMap; 39import java.util.Locale; 40import java.util.Map; 41import java.util.concurrent.ConcurrentHashMap; 42import java.util.concurrent.CountDownLatch; 43import java.util.concurrent.TimeUnit; 44import java.util.concurrent.atomic.AtomicBoolean; 45import java.util.concurrent.atomic.AtomicReference; 46 47/** 48 * Abstract base class for an expandable dictionary that can be created and updated dynamically 49 * during runtime. When updated it automatically generates a new binary dictionary to handle future 50 * queries in native code. This binary dictionary is written to internal storage, and potentially 51 * shared across multiple ExpandableBinaryDictionary instances. Updates to each dictionary filename 52 * are controlled across multiple instances to ensure that only one instance can update the same 53 * dictionary at the same time. 54 */ 55abstract public class ExpandableBinaryDictionary extends Dictionary { 56 57 /** Used for Log actions from this class */ 58 private static final String TAG = ExpandableBinaryDictionary.class.getSimpleName(); 59 60 /** Whether to print debug output to log */ 61 private static boolean DEBUG = false; 62 private static final boolean DBG_STRESS_TEST = false; 63 64 private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; 65 private static final int TIMEOUT_FOR_READ_OPS_FOR_TESTS_IN_MILLISECONDS = 1000; 66 67 /** 68 * The maximum length of a word in this dictionary. 69 */ 70 protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; 71 72 private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4; 73 74 /** 75 * A static map of update controllers, each of which records the time of accesses to a single 76 * binary dictionary file and tracks whether the file is regenerating. The key for this map is 77 * the dictionary name and the value is the shared dictionary time recorder associated with 78 * that dictionary name. 79 */ 80 private static final ConcurrentHashMap<String, DictionaryUpdateController> 81 sDictNameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); 82 83 private static final ConcurrentHashMap<String, PrioritizedSerialExecutor> 84 sDictNameExecutorMap = CollectionUtils.newConcurrentHashMap(); 85 86 /** The application context. */ 87 protected final Context mContext; 88 89 /** 90 * The binary dictionary generated dynamically from the fusion dictionary. This is used to 91 * answer unigram and bigram queries. 92 */ 93 private BinaryDictionary mBinaryDictionary; 94 95 // TODO: Remove and handle dictionaries in native code. 96 /** The in-memory dictionary used to generate the binary dictionary. */ 97 protected AbstractDictionaryWriter mDictionaryWriter; 98 99 /** 100 * The name of this dictionary, used as a part of the filename for storing the binary 101 * dictionary. Multiple dictionary instances with the same name is supported, with access 102 * controlled by DictionaryUpdateController. 103 */ 104 private final String mDictName; 105 106 /** Dictionary locale */ 107 private final Locale mLocale; 108 109 /** Whether to support dynamically updating the dictionary */ 110 private final boolean mIsUpdatable; 111 112 /** Dictionary file */ 113 private final File mDictFile; 114 115 // TODO: remove, once dynamic operations is serialized 116 /** Controls updating the shared binary dictionary file across multiple instances. */ 117 private final DictionaryUpdateController mDictNameDictionaryUpdateController; 118 119 // TODO: remove, once dynamic operations is serialized 120 /** Controls updating the local binary dictionary for this instance. */ 121 private final DictionaryUpdateController mPerInstanceDictionaryUpdateController = 122 new DictionaryUpdateController(); 123 124 /* A extension for a binary dictionary file. */ 125 protected static final String DICT_FILE_EXTENSION = ".dict"; 126 127 private final AtomicReference<Runnable> mUnfinishedFlushingTask = 128 new AtomicReference<Runnable>(); 129 130 /** 131 * Abstract method for loading the unigrams and bigrams of a given dictionary in a background 132 * thread. 133 */ 134 protected abstract void loadDictionaryAsync(); 135 136 /** 137 * Indicates that the source dictionary content has changed and a rebuild of the binary file is 138 * required. If it returns false, the next reload will only read the current binary dictionary 139 * from file. Note that the shared binary dictionary is locked when this is called. 140 */ 141 protected abstract boolean hasContentChanged(); 142 143 private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { 144 return formatVersion == FormatSpec.VERSION4; 145 } 146 147 public boolean isValidDictionary() { 148 return mBinaryDictionary.isValidDictionary(); 149 } 150 151 /** 152 * Gets the dictionary update controller for the given dictionary name. 153 */ 154 private static DictionaryUpdateController getDictionaryUpdateController( 155 final String dictName) { 156 DictionaryUpdateController recorder = sDictNameDictionaryUpdateControllerMap.get(dictName); 157 if (recorder == null) { 158 synchronized(sDictNameDictionaryUpdateControllerMap) { 159 recorder = new DictionaryUpdateController(); 160 sDictNameDictionaryUpdateControllerMap.put(dictName, recorder); 161 } 162 } 163 return recorder; 164 } 165 166 /** 167 * Gets the executor for the given dictionary name. 168 */ 169 private static PrioritizedSerialExecutor getExecutor(final String dictName) { 170 PrioritizedSerialExecutor executor = sDictNameExecutorMap.get(dictName); 171 if (executor == null) { 172 synchronized(sDictNameExecutorMap) { 173 executor = new PrioritizedSerialExecutor(); 174 sDictNameExecutorMap.put(dictName, executor); 175 } 176 } 177 return executor; 178 } 179 180 /** 181 * Shutdowns all executors and removes all executors from the executor map for testing. 182 */ 183 @UsedForTesting 184 public static void shutdownAllExecutors() { 185 synchronized(sDictNameExecutorMap) { 186 for (final PrioritizedSerialExecutor executor : sDictNameExecutorMap.values()) { 187 executor.shutdown(); 188 sDictNameExecutorMap.remove(executor); 189 } 190 } 191 } 192 193 private static AbstractDictionaryWriter getDictionaryWriter( 194 final boolean isDynamicPersonalizationDictionary) { 195 if (isDynamicPersonalizationDictionary) { 196 return null; 197 } else { 198 return new DictionaryWriter(); 199 } 200 } 201 202 /** 203 * Creates a new expandable binary dictionary. 204 * 205 * @param context The application context of the parent. 206 * @param dictName The name of the dictionary. Multiple instances with the same 207 * name is supported. 208 * @param locale the dictionary locale. 209 * @param dictType the dictionary type, as a human-readable string 210 * @param isUpdatable whether to support dynamically updating the dictionary. Please note that 211 * dynamic dictionary has negative effects on memory space and computation time. 212 * @param dictFile dictionary file path. if null, use default dictionary path based on 213 * dictionary type. 214 */ 215 public ExpandableBinaryDictionary(final Context context, final String dictName, 216 final Locale locale, final String dictType, final boolean isUpdatable, 217 final File dictFile) { 218 super(dictType); 219 mDictName = dictName; 220 mContext = context; 221 mLocale = locale; 222 mIsUpdatable = isUpdatable; 223 mDictFile = getDictFile(context, dictName, dictFile); 224 mBinaryDictionary = null; 225 mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName); 226 // Currently, only dynamic personalization dictionary is updatable. 227 mDictionaryWriter = getDictionaryWriter(isUpdatable); 228 } 229 230 public static File getDictFile(final Context context, final String dictName, 231 final File dictFile) { 232 return (dictFile != null) ? dictFile 233 : new File(context.getFilesDir(), dictName + DICT_FILE_EXTENSION); 234 } 235 236 public static String getDictName(final String name, final Locale locale, 237 final File dictFile) { 238 return dictFile != null ? dictFile.getName() : name + "." + locale.toString(); 239 } 240 241 /** 242 * Closes and cleans up the binary dictionary. 243 */ 244 @Override 245 public void close() { 246 getExecutor(mDictName).execute(new Runnable() { 247 @Override 248 public void run() { 249 if (mBinaryDictionary!= null) { 250 mBinaryDictionary.close(); 251 mBinaryDictionary = null; 252 } 253 } 254 }); 255 } 256 257 protected void closeBinaryDictionary() { 258 // Ensure that no other threads are accessing the local binary dictionary. 259 getExecutor(mDictName).execute(new Runnable() { 260 @Override 261 public void run() { 262 if (mBinaryDictionary != null) { 263 mBinaryDictionary.close(); 264 mBinaryDictionary = null; 265 } 266 } 267 }); 268 } 269 270 protected Map<String, String> getHeaderAttributeMap() { 271 HashMap<String, String> attributeMap = new HashMap<String, String>(); 272 attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); 273 attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); 274 attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, 275 String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); 276 return attributeMap; 277 } 278 279 protected void clear() { 280 final File dictFile = mDictFile; 281 getExecutor(mDictName).execute(new Runnable() { 282 @Override 283 public void run() { 284 if (mDictionaryWriter == null) { 285 if (mBinaryDictionary != null) { 286 mBinaryDictionary.close(); 287 } 288 if (dictFile.exists() && !FileUtils.deleteRecursively(dictFile)) { 289 Log.e(TAG, "Can't remove a file: " + dictFile.getName()); 290 } 291 BinaryDictionary.createEmptyDictFile(dictFile.getAbsolutePath(), 292 DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); 293 mBinaryDictionary = new BinaryDictionary( 294 dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), 295 true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable); 296 } else { 297 mDictionaryWriter.clear(); 298 } 299 } 300 }); 301 } 302 303 /** 304 * Adds a word unigram to the dictionary. Used for loading a dictionary. 305 * @param word The word to add. 306 * @param shortcutTarget A shortcut target for this word, or null if none. 307 * @param frequency The frequency for this unigram. 308 * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored 309 * if shortcutTarget is null. 310 * @param isNotAWord true if this is not a word, i.e. shortcut only. 311 */ 312 protected void addWord(final String word, final String shortcutTarget, 313 final int frequency, final int shortcutFreq, final boolean isNotAWord) { 314 mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord); 315 } 316 317 /** 318 * Adds a word bigram in the dictionary. Used for loading a dictionary. 319 */ 320 protected void addBigram(final String prevWord, final String word, final int frequency, 321 final long lastModifiedTime) { 322 mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */, 323 lastModifiedTime); 324 } 325 326 /** 327 * Check whether GC is needed and run GC if required. 328 */ 329 protected void runGCIfRequired(final boolean mindsBlockByGC) { 330 getExecutor(mDictName).execute(new Runnable() { 331 @Override 332 public void run() { 333 runGCIfRequiredInternalLocked(mindsBlockByGC); 334 } 335 }); 336 } 337 338 private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { 339 // Calls to needsToRunGC() need to be serialized. 340 if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { 341 if (setProcessingLargeTaskIfNot()) { 342 // Run GC after currently existing time sensitive operations. 343 getExecutor(mDictName).executePrioritized(new Runnable() { 344 @Override 345 public void run() { 346 try { 347 mBinaryDictionary.flushWithGC(); 348 } finally { 349 mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); 350 } 351 } 352 }); 353 } 354 } 355 } 356 357 /** 358 * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. 359 */ 360 protected void addWordDynamically(final String word, final int frequency, 361 final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, 362 final boolean isBlacklisted, final int timestamp) { 363 if (!mIsUpdatable) { 364 Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName); 365 return; 366 } 367 getExecutor(mDictName).execute(new Runnable() { 368 @Override 369 public void run() { 370 runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); 371 mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, 372 isNotAWord, isBlacklisted, timestamp); 373 } 374 }); 375 } 376 377 /** 378 * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. 379 */ 380 protected void addBigramDynamically(final String word0, final String word1, 381 final int frequency, final int timestamp) { 382 if (!mIsUpdatable) { 383 Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " 384 + mDictName); 385 return; 386 } 387 getExecutor(mDictName).execute(new Runnable() { 388 @Override 389 public void run() { 390 runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); 391 mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); 392 } 393 }); 394 } 395 396 /** 397 * Dynamically remove a word bigram in the dictionary. 398 */ 399 protected void removeBigramDynamically(final String word0, final String word1) { 400 if (!mIsUpdatable) { 401 Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " 402 + mDictName); 403 return; 404 } 405 getExecutor(mDictName).execute(new Runnable() { 406 @Override 407 public void run() { 408 runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); 409 mBinaryDictionary.removeBigramWords(word0, word1); 410 } 411 }); 412 } 413 414 public interface AddMultipleDictionaryEntriesCallback { 415 public void onFinished(); 416 } 417 418 /** 419 * Dynamically add multiple entries to the dictionary. 420 */ 421 protected void addMultipleDictionaryEntriesDynamically( 422 final ArrayList<LanguageModelParam> languageModelParams, 423 final AddMultipleDictionaryEntriesCallback callback) { 424 if (!mIsUpdatable) { 425 Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " + 426 "dictionary: " + mDictName); 427 return; 428 } 429 getExecutor(mDictName).execute(new Runnable() { 430 @Override 431 public void run() { 432 final boolean locked = setProcessingLargeTaskIfNot(); 433 try { 434 mBinaryDictionary.addMultipleDictionaryEntries( 435 languageModelParams.toArray( 436 new LanguageModelParam[languageModelParams.size()])); 437 } finally { 438 if (callback != null) { 439 callback.onFinished(); 440 } 441 if (locked) { 442 mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); 443 } 444 } 445 } 446 }); 447 } 448 449 @Override 450 public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, 451 final String prevWord, final ProximityInfo proximityInfo, 452 final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, 453 final int sessionId) { 454 reloadDictionaryIfRequired(); 455 if (processingLargeTask()) { 456 return null; 457 } 458 final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = 459 new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); 460 getExecutor(mDictName).executePrioritized(new Runnable() { 461 @Override 462 public void run() { 463 if (mBinaryDictionary == null) { 464 holder.set(null); 465 return; 466 } 467 final ArrayList<SuggestedWordInfo> binarySuggestion = 468 mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, 469 proximityInfo, blockOffensiveWords, additionalFeaturesOptions, 470 sessionId); 471 holder.set(binarySuggestion); 472 } 473 }); 474 return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); 475 } 476 477 @Override 478 public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, 479 final String prevWord, final ProximityInfo proximityInfo, 480 final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { 481 return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, 482 additionalFeaturesOptions, 0 /* sessionId */); 483 } 484 485 @Override 486 public boolean isValidWord(final String word) { 487 reloadDictionaryIfRequired(); 488 return isValidWordInner(word); 489 } 490 491 protected boolean isValidWordInner(final String word) { 492 if (processingLargeTask()) { 493 return false; 494 } 495 final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); 496 getExecutor(mDictName).executePrioritized(new Runnable() { 497 @Override 498 public void run() { 499 holder.set(isValidWordLocked(word)); 500 } 501 }); 502 return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); 503 } 504 505 protected boolean isValidWordLocked(final String word) { 506 if (mBinaryDictionary == null) return false; 507 return mBinaryDictionary.isValidWord(word); 508 } 509 510 protected boolean isValidBigramLocked(final String word1, final String word2) { 511 if (mBinaryDictionary == null) return false; 512 return mBinaryDictionary.isValidBigram(word1, word2); 513 } 514 515 /** 516 * Load the current binary dictionary from internal storage in a background thread. If no binary 517 * dictionary exists, this method will generate one. 518 */ 519 protected void loadDictionary() { 520 mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = System.currentTimeMillis(); 521 reloadDictionaryIfRequired(); 522 } 523 524 /** 525 * Loads the current binary dictionary from internal storage. Assumes the dictionary file 526 * exists. 527 */ 528 private void loadBinaryDictionary() { 529 if (DEBUG) { 530 Log.d(TAG, "Loading binary dictionary: " + mDictName + " request=" 531 + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" 532 + mDictNameDictionaryUpdateController.mLastUpdateTime); 533 } 534 if (DBG_STRESS_TEST) { 535 // Test if this class does not cause problems when it takes long time to load binary 536 // dictionary. 537 try { 538 Log.w(TAG, "Start stress in loading: " + mDictName); 539 Thread.sleep(15000); 540 Log.w(TAG, "End stress in loading"); 541 } catch (InterruptedException e) { 542 } 543 } 544 545 final String filename = mDictFile.getAbsolutePath(); 546 final long length = mDictFile.length(); 547 548 // Build the new binary dictionary 549 final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */, 550 length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); 551 552 // Ensure all threads accessing the current dictionary have finished before 553 // swapping in the new one. 554 // TODO: Ensure multi-thread assignment of mBinaryDictionary. 555 final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; 556 getExecutor(mDictName).executePrioritized(new Runnable() { 557 @Override 558 public void run() { 559 mBinaryDictionary = newBinaryDictionary; 560 if (oldBinaryDictionary != null) { 561 oldBinaryDictionary.close(); 562 } 563 } 564 }); 565 } 566 567 /** 568 * Abstract method for checking if it is required to reload the dictionary before writing 569 * a binary dictionary. 570 */ 571 abstract protected boolean needsToReloadBeforeWriting(); 572 573 /** 574 * Writes a new binary dictionary based on the contents of the fusion dictionary. 575 */ 576 private void writeBinaryDictionary() { 577 if (DEBUG) { 578 Log.d(TAG, "Generating binary dictionary: " + mDictName + " request=" 579 + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" 580 + mDictNameDictionaryUpdateController.mLastUpdateTime); 581 } 582 if (needsToReloadBeforeWriting()) { 583 mDictionaryWriter.clear(); 584 loadDictionaryAsync(); 585 mDictionaryWriter.write(mDictFile, getHeaderAttributeMap()); 586 } else { 587 if (mBinaryDictionary == null || !isValidDictionary() 588 // TODO: remove the check below 589 || !matchesExpectedBinaryDictFormatVersionForThisType( 590 mBinaryDictionary.getFormatVersion())) { 591 if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) { 592 Log.e(TAG, "Can't remove a file: " + mDictFile.getName()); 593 } 594 BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), 595 DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); 596 } else { 597 if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { 598 mBinaryDictionary.flushWithGC(); 599 } else { 600 mBinaryDictionary.flush(); 601 } 602 } 603 } 604 } 605 606 /** 607 * Marks that the dictionary is out of date and requires a reload. 608 * 609 * @param requiresRebuild Indicates that the source dictionary content has changed and a rebuild 610 * of the binary file is required. If not true, the next reload process will only read 611 * the current binary dictionary from file. 612 */ 613 protected void setRequiresReload(final boolean requiresRebuild) { 614 final long time = System.currentTimeMillis(); 615 mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time; 616 mDictNameDictionaryUpdateController.mLastUpdateRequestTime = time; 617 if (DEBUG) { 618 Log.d(TAG, "Reload request: " + mDictName + ": request=" + time + " update=" 619 + mDictNameDictionaryUpdateController.mLastUpdateTime); 620 } 621 } 622 623 /** 624 * Reloads the dictionary if required. 625 */ 626 public final void reloadDictionaryIfRequired() { 627 if (!isReloadRequired()) return; 628 if (setProcessingLargeTaskIfNot()) { 629 reloadDictionary(); 630 } 631 } 632 633 /** 634 * Returns whether a dictionary reload is required. 635 */ 636 private boolean isReloadRequired() { 637 return mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.isOutOfDate(); 638 } 639 640 private boolean processingLargeTask() { 641 return mDictNameDictionaryUpdateController.mProcessingLargeTask.get(); 642 } 643 644 // Returns whether the dictionary is being used for a large task. If true, we should not use 645 // this dictionary for latency sensitive operations. 646 private boolean setProcessingLargeTaskIfNot() { 647 return mDictNameDictionaryUpdateController.mProcessingLargeTask.compareAndSet( 648 false /* expect */ , true /* update */); 649 } 650 651 /** 652 * Reloads the dictionary. Access is controlled on a per dictionary file basis and supports 653 * concurrent calls from multiple instances that share the same dictionary file. 654 */ 655 private final void reloadDictionary() { 656 // Ensure that only one thread attempts to read or write to the shared binary dictionary 657 // file at the same time. 658 getExecutor(mDictName).execute(new Runnable() { 659 @Override 660 public void run() { 661 try { 662 final long time = System.currentTimeMillis(); 663 final boolean dictionaryFileExists = dictionaryFileExists(); 664 if (mDictNameDictionaryUpdateController.isOutOfDate() 665 || !dictionaryFileExists) { 666 // If the shared dictionary file does not exist or is out of date, the 667 // first instance that acquires the lock will generate a new one. 668 if (hasContentChanged() || !dictionaryFileExists) { 669 // If the source content has changed or the dictionary does not exist, 670 // rebuild the binary dictionary. Empty dictionaries are supported (in 671 // the case where loadDictionaryAsync() adds nothing) in order to 672 // provide a uniform framework. 673 mDictNameDictionaryUpdateController.mLastUpdateTime = time; 674 writeBinaryDictionary(); 675 loadBinaryDictionary(); 676 } else { 677 // If not, the reload request was unnecessary so revert 678 // LastUpdateRequestTime to LastUpdateTime. 679 mDictNameDictionaryUpdateController.mLastUpdateRequestTime = 680 mDictNameDictionaryUpdateController.mLastUpdateTime; 681 } 682 } else if (mBinaryDictionary == null || 683 mPerInstanceDictionaryUpdateController.mLastUpdateTime 684 < mDictNameDictionaryUpdateController.mLastUpdateTime) { 685 // Otherwise, if the local dictionary is older than the shared dictionary, 686 // load the shared dictionary. 687 loadBinaryDictionary(); 688 } 689 // If we just loaded the binary dictionary, then mBinaryDictionary is not 690 // up-to-date yet so it's useless to test it right away. Schedule the check 691 // for right after it's loaded instead. 692 getExecutor(mDictName).executePrioritized(new Runnable() { 693 @Override 694 public void run() { 695 if (mBinaryDictionary != null && !(isValidDictionary() 696 // TODO: remove the check below 697 && matchesExpectedBinaryDictFormatVersionForThisType( 698 mBinaryDictionary.getFormatVersion()))) { 699 // Binary dictionary or its format version is not valid. Regenerate 700 // the dictionary file. writeBinaryDictionary will remove the 701 // existing files if appropriate. 702 mDictNameDictionaryUpdateController.mLastUpdateTime = time; 703 writeBinaryDictionary(); 704 loadBinaryDictionary(); 705 } 706 mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; 707 } 708 }); 709 } finally { 710 mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); 711 } 712 } 713 }); 714 } 715 716 // TODO: cache the file's existence so that we avoid doing a disk access each time. 717 private boolean dictionaryFileExists() { 718 return mDictFile.exists(); 719 } 720 721 /** 722 * Generate binary dictionary using DictionaryWriter. 723 */ 724 protected void asyncFlushBinaryDictionary() { 725 final Runnable newTask = new Runnable() { 726 @Override 727 public void run() { 728 writeBinaryDictionary(); 729 } 730 }; 731 final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); 732 getExecutor(mDictName).replaceAndExecute(oldTask, newTask); 733 } 734 735 /** 736 * For tracking whether the dictionary is out of date and the dictionary is used in a large 737 * task. Can be shared across multiple dictionary instances that access the same filename. 738 */ 739 private static class DictionaryUpdateController { 740 public volatile long mLastUpdateTime = 0; 741 public volatile long mLastUpdateRequestTime = 0; 742 public volatile AtomicBoolean mProcessingLargeTask = new AtomicBoolean(); 743 744 public boolean isOutOfDate() { 745 return (mLastUpdateRequestTime > mLastUpdateTime); 746 } 747 } 748 749 // TODO: Implement BinaryDictionary.isInDictionary(). 750 @UsedForTesting 751 public boolean isInUnderlyingBinaryDictionaryForTests(final String word) { 752 final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); 753 getExecutor(mDictName).executePrioritized(new Runnable() { 754 @Override 755 public void run() { 756 if (mDictType == Dictionary.TYPE_USER_HISTORY) { 757 holder.set(mBinaryDictionary.isValidWord(word)); 758 } 759 } 760 }); 761 return holder.get(false, TIMEOUT_FOR_READ_OPS_FOR_TESTS_IN_MILLISECONDS); 762 } 763 764 @UsedForTesting 765 public void waitAllTasksForTests() { 766 final CountDownLatch countDownLatch = new CountDownLatch(1); 767 getExecutor(mDictName).execute(new Runnable() { 768 @Override 769 public void run() { 770 countDownLatch.countDown(); 771 } 772 }); 773 try { 774 countDownLatch.await(); 775 } catch (InterruptedException e) { 776 Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); 777 } 778 } 779 780 @UsedForTesting 781 public void dumpAllWordsForDebug() { 782 reloadDictionaryIfRequired(); 783 getExecutor(mDictName).execute(new Runnable() { 784 @Override 785 public void run() { 786 Log.d(TAG, "Dump dictionary: " + mDictName); 787 try { 788 final DictionaryHeader header = mBinaryDictionary.getHeader(); 789 Log.d(TAG, CombinedFormatUtils.formatAttributeMap( 790 header.mDictionaryOptions.mAttributes)); 791 } catch (final UnsupportedFormatException e) { 792 Log.d(TAG, "Cannot fetch header information.", e); 793 } 794 int token = 0; 795 do { 796 final BinaryDictionary.GetNextWordPropertyResult result = 797 mBinaryDictionary.getNextWordProperty(token); 798 final WordProperty wordProperty = result.mWordProperty; 799 if (wordProperty == null) { 800 Log.d(TAG, " dictionary is empty."); 801 break; 802 } 803 Log.d(TAG, wordProperty.toString()); 804 token = result.mNextToken; 805 } while (token != 0); 806 } 807 }); 808 } 809} 810