ExpandableBinaryDictionary.java revision c3b151957cd2130cbf781fd815a7f7322308f542
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.content.Context; 20import android.util.Log; 21 22import com.android.inputmethod.annotations.UsedForTesting; 23import com.android.inputmethod.keyboard.ProximityInfo; 24import com.android.inputmethod.latin.makedict.DictionaryHeader; 25import com.android.inputmethod.latin.makedict.FormatSpec; 26import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 27import com.android.inputmethod.latin.makedict.WordProperty; 28import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 29import com.android.inputmethod.latin.utils.CombinedFormatUtils; 30import com.android.inputmethod.latin.utils.ExecutorUtils; 31import com.android.inputmethod.latin.utils.FileUtils; 32import com.android.inputmethod.latin.utils.LanguageModelParam; 33 34import java.io.File; 35import java.util.ArrayList; 36import java.util.HashMap; 37import java.util.Locale; 38import java.util.Map; 39import java.util.concurrent.CountDownLatch; 40import java.util.concurrent.TimeUnit; 41import java.util.concurrent.atomic.AtomicBoolean; 42import java.util.concurrent.locks.Lock; 43import java.util.concurrent.locks.ReentrantReadWriteLock; 44 45/** 46 * Abstract base class for an expandable dictionary that can be created and updated dynamically 47 * during runtime. When updated it automatically generates a new binary dictionary to handle future 48 * queries in native code. This binary dictionary is written to internal storage. 49 */ 50abstract public class ExpandableBinaryDictionary extends Dictionary { 51 52 /** Used for Log actions from this class */ 53 private static final String TAG = ExpandableBinaryDictionary.class.getSimpleName(); 54 55 /** Whether to print debug output to log */ 56 private static boolean DEBUG = false; 57 private static final boolean DBG_STRESS_TEST = false; 58 59 private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; 60 61 private static final int DEFAULT_MAX_UNIGRAM_COUNT = 10000; 62 private static final int DEFAULT_MAX_BIGRAM_COUNT = 10000; 63 64 /** 65 * The maximum length of a word in this dictionary. 66 */ 67 protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; 68 69 private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4; 70 71 /** The application context. */ 72 protected final Context mContext; 73 74 /** 75 * The binary dictionary generated dynamically from the fusion dictionary. This is used to 76 * answer unigram and bigram queries. 77 */ 78 private BinaryDictionary mBinaryDictionary; 79 80 /** 81 * The name of this dictionary, used as a part of the filename for storing the binary 82 * dictionary. 83 */ 84 private final String mDictName; 85 86 /** Dictionary locale */ 87 private final Locale mLocale; 88 89 /** Dictionary file */ 90 private final File mDictFile; 91 92 /** Indicates whether a task for reloading the dictionary has been scheduled. */ 93 private final AtomicBoolean mIsReloading; 94 95 /** Indicates whether the current dictionary needs to be reloaded. */ 96 private boolean mNeedsToReload; 97 98 private final ReentrantReadWriteLock mLock; 99 100 private Map<String, String> mAdditionalAttributeMap = null; 101 102 /* A extension for a binary dictionary file. */ 103 protected static final String DICT_FILE_EXTENSION = ".dict"; 104 105 /** 106 * Abstract method for loading initial contents of a given dictionary. 107 */ 108 protected abstract void loadInitialContentsLocked(); 109 110 /** 111 * Indicates that the source dictionary contents have changed and a rebuild of the binary file 112 * is required. If it returns false, the next reload will only read the current binary 113 * dictionary from file. 114 */ 115 protected abstract boolean haveContentsChanged(); 116 117 private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { 118 return formatVersion == FormatSpec.VERSION4; 119 } 120 121 private boolean needsToMigrateDictionary(final int formatVersion) { 122 // TODO: Check version. 123 return false; 124 } 125 126 public boolean isValidDictionaryLocked() { 127 return mBinaryDictionary.isValidDictionary(); 128 } 129 130 /** 131 * Creates a new expandable binary dictionary. 132 * 133 * @param context The application context of the parent. 134 * @param dictName The name of the dictionary. Multiple instances with the same 135 * name is supported. 136 * @param locale the dictionary locale. 137 * @param dictType the dictionary type, as a human-readable string 138 * @param dictFile dictionary file path. if null, use default dictionary path based on 139 * dictionary type. 140 */ 141 public ExpandableBinaryDictionary(final Context context, final String dictName, 142 final Locale locale, final String dictType, final File dictFile) { 143 super(dictType); 144 mDictName = dictName; 145 mContext = context; 146 mLocale = locale; 147 mDictFile = getDictFile(context, dictName, dictFile); 148 mBinaryDictionary = null; 149 mIsReloading = new AtomicBoolean(); 150 mNeedsToReload = false; 151 mLock = new ReentrantReadWriteLock(); 152 } 153 154 public static File getDictFile(final Context context, final String dictName, 155 final File dictFile) { 156 return (dictFile != null) ? dictFile 157 : new File(context.getFilesDir(), dictName + DICT_FILE_EXTENSION); 158 } 159 160 public static String getDictName(final String name, final Locale locale, 161 final File dictFile) { 162 return dictFile != null ? dictFile.getName() : name + "." + locale.toString(); 163 } 164 165 private void asyncExecuteTaskWithWriteLock(final Runnable task) { 166 asyncExecuteTaskWithLock(mLock.writeLock(), task); 167 } 168 169 private void asyncExecuteTaskWithLock(final Lock lock, final Runnable task) { 170 ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { 171 @Override 172 public void run() { 173 lock.lock(); 174 try { 175 task.run(); 176 } finally { 177 lock.unlock(); 178 } 179 } 180 }); 181 } 182 183 /** 184 * Closes and cleans up the binary dictionary. 185 */ 186 @Override 187 public void close() { 188 asyncExecuteTaskWithWriteLock(new Runnable() { 189 @Override 190 public void run() { 191 if (mBinaryDictionary != null) { 192 mBinaryDictionary.close(); 193 mBinaryDictionary = null; 194 } 195 } 196 }); 197 } 198 199 protected Map<String, String> getHeaderAttributeMap() { 200 HashMap<String, String> attributeMap = new HashMap<String, String>(); 201 if (mAdditionalAttributeMap != null) { 202 attributeMap.putAll(mAdditionalAttributeMap); 203 } 204 attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); 205 attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); 206 attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, 207 String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); 208 attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, 209 String.valueOf(DEFAULT_MAX_UNIGRAM_COUNT)); 210 attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, 211 String.valueOf(DEFAULT_MAX_BIGRAM_COUNT)); 212 return attributeMap; 213 } 214 215 private void removeBinaryDictionary() { 216 asyncExecuteTaskWithWriteLock(new Runnable() { 217 @Override 218 public void run() { 219 removeBinaryDictionaryLocked(); 220 } 221 }); 222 } 223 224 private void removeBinaryDictionaryLocked() { 225 if (mBinaryDictionary != null) { 226 mBinaryDictionary.close(); 227 } 228 if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) { 229 Log.e(TAG, "Can't remove a file: " + mDictFile.getName()); 230 } 231 mBinaryDictionary = null; 232 } 233 234 private void openBinaryDictionaryLocked() { 235 mBinaryDictionary = new BinaryDictionary( 236 mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), 237 true /* useFullEditDistance */, mLocale, mDictType, true /* isUpdatable */); 238 } 239 240 private void createOnMemoryBinaryDictionaryLocked() { 241 mBinaryDictionary = new BinaryDictionary( 242 mDictFile.getAbsolutePath(), true /* useFullEditDistance */, mLocale, mDictType, 243 DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); 244 } 245 246 public void clear() { 247 asyncExecuteTaskWithWriteLock(new Runnable() { 248 @Override 249 public void run() { 250 removeBinaryDictionaryLocked(); 251 createOnMemoryBinaryDictionaryLocked(); 252 } 253 }); 254 } 255 256 /** 257 * Check whether GC is needed and run GC if required. 258 */ 259 protected void runGCIfRequired(final boolean mindsBlockByGC) { 260 asyncExecuteTaskWithWriteLock(new Runnable() { 261 @Override 262 public void run() { 263 if (mBinaryDictionary == null) { 264 return; 265 } 266 runGCIfRequiredLocked(mindsBlockByGC); 267 } 268 }); 269 } 270 271 protected void runGCIfRequiredLocked(final boolean mindsBlockByGC) { 272 if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { 273 mBinaryDictionary.flushWithGC(); 274 } 275 } 276 277 /** 278 * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. 279 */ 280 public void addWordDynamically(final String word, final int frequency, 281 final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, 282 final boolean isBlacklisted, final int timestamp) { 283 reloadDictionaryIfRequired(); 284 asyncExecuteTaskWithWriteLock(new Runnable() { 285 @Override 286 public void run() { 287 if (mBinaryDictionary == null) { 288 return; 289 } 290 runGCIfRequiredLocked(true /* mindsBlockByGC */); 291 addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, 292 isNotAWord, isBlacklisted, timestamp); 293 } 294 }); 295 } 296 297 protected void addWordDynamicallyLocked(final String word, final int frequency, 298 final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, 299 final boolean isBlacklisted, final int timestamp) { 300 mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, 301 isNotAWord, isBlacklisted, timestamp); 302 } 303 304 /** 305 * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. 306 */ 307 public void addBigramDynamically(final String word0, final String word1, 308 final int frequency, final int timestamp) { 309 reloadDictionaryIfRequired(); 310 asyncExecuteTaskWithWriteLock(new Runnable() { 311 @Override 312 public void run() { 313 if (mBinaryDictionary == null) { 314 return; 315 } 316 runGCIfRequiredLocked(true /* mindsBlockByGC */); 317 addBigramDynamicallyLocked(word0, word1, frequency, timestamp); 318 } 319 }); 320 } 321 322 protected void addBigramDynamicallyLocked(final String word0, final String word1, 323 final int frequency, final int timestamp) { 324 mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); 325 } 326 327 /** 328 * Dynamically remove a word bigram in the dictionary. 329 */ 330 public void removeBigramDynamically(final String word0, final String word1) { 331 reloadDictionaryIfRequired(); 332 asyncExecuteTaskWithWriteLock(new Runnable() { 333 @Override 334 public void run() { 335 if (mBinaryDictionary == null) { 336 return; 337 } 338 runGCIfRequiredLocked(true /* mindsBlockByGC */); 339 mBinaryDictionary.removeBigramWords(word0, word1); 340 } 341 }); 342 } 343 344 public interface AddMultipleDictionaryEntriesCallback { 345 public void onFinished(); 346 } 347 348 /** 349 * Dynamically add multiple entries to the dictionary. 350 */ 351 public void addMultipleDictionaryEntriesDynamically( 352 final ArrayList<LanguageModelParam> languageModelParams, 353 final AddMultipleDictionaryEntriesCallback callback) { 354 reloadDictionaryIfRequired(); 355 asyncExecuteTaskWithWriteLock(new Runnable() { 356 @Override 357 public void run() { 358 try { 359 if (mBinaryDictionary == null) { 360 return; 361 } 362 mBinaryDictionary.addMultipleDictionaryEntries( 363 languageModelParams.toArray( 364 new LanguageModelParam[languageModelParams.size()])); 365 } finally { 366 if (callback != null) { 367 callback.onFinished(); 368 } 369 } 370 } 371 }); 372 } 373 374 @Override 375 public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, 376 final String prevWord, final ProximityInfo proximityInfo, 377 final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, 378 final int sessionId, final float[] inOutLanguageWeight) { 379 reloadDictionaryIfRequired(); 380 boolean lockAcquired = false; 381 try { 382 lockAcquired = mLock.readLock().tryLock( 383 TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS, TimeUnit.MILLISECONDS); 384 if (lockAcquired) { 385 if (mBinaryDictionary == null) { 386 return null; 387 } 388 final ArrayList<SuggestedWordInfo> suggestions = 389 mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, 390 proximityInfo, blockOffensiveWords, additionalFeaturesOptions, 391 sessionId, inOutLanguageWeight); 392 if (mBinaryDictionary.isCorrupted()) { 393 Log.i(TAG, "Dictionary (" + mDictName +") is corrupted. " 394 + "Remove and regenerate it."); 395 removeBinaryDictionary(); 396 } 397 return suggestions; 398 } 399 } catch (final InterruptedException e) { 400 Log.e(TAG, "Interrupted tryLock() in getSuggestionsWithSessionId().", e); 401 } finally { 402 if (lockAcquired) { 403 mLock.readLock().unlock(); 404 } 405 } 406 return null; 407 } 408 409 @Override 410 public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, 411 final String prevWord, final ProximityInfo proximityInfo, 412 final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, 413 final float[] inOutLanguageWeight) { 414 return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, 415 additionalFeaturesOptions, 0 /* sessionId */, inOutLanguageWeight); 416 } 417 418 @Override 419 public boolean isValidWord(final String word) { 420 reloadDictionaryIfRequired(); 421 boolean lockAcquired = false; 422 try { 423 lockAcquired = mLock.readLock().tryLock( 424 TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS, TimeUnit.MILLISECONDS); 425 if (lockAcquired) { 426 if (mBinaryDictionary == null) { 427 return false; 428 } 429 return isValidWordLocked(word); 430 } 431 } catch (final InterruptedException e) { 432 Log.e(TAG, "Interrupted tryLock() in isValidWord().", e); 433 } finally { 434 if (lockAcquired) { 435 mLock.readLock().unlock(); 436 } 437 } 438 return false; 439 } 440 441 protected boolean isValidWordLocked(final String word) { 442 if (mBinaryDictionary == null) return false; 443 return mBinaryDictionary.isValidWord(word); 444 } 445 446 protected boolean isValidBigramLocked(final String word1, final String word2) { 447 if (mBinaryDictionary == null) return false; 448 return mBinaryDictionary.isValidBigram(word1, word2); 449 } 450 451 /** 452 * Loads the current binary dictionary from internal storage. Assumes the dictionary file 453 * exists. 454 */ 455 private void loadBinaryDictionaryLocked() { 456 if (DBG_STRESS_TEST) { 457 // Test if this class does not cause problems when it takes long time to load binary 458 // dictionary. 459 try { 460 Log.w(TAG, "Start stress in loading: " + mDictName); 461 Thread.sleep(15000); 462 Log.w(TAG, "End stress in loading"); 463 } catch (InterruptedException e) { 464 } 465 } 466 final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; 467 openBinaryDictionaryLocked(); 468 if (oldBinaryDictionary != null) { 469 oldBinaryDictionary.close(); 470 } 471 if (mBinaryDictionary.isValidDictionary() 472 && needsToMigrateDictionary(mBinaryDictionary.getFormatVersion())) { 473 mBinaryDictionary.migrateTo(DICTIONARY_FORMAT_VERSION); 474 } 475 } 476 477 /** 478 * Create a new binary dictionary and load initial contents. 479 */ 480 private void createNewDictionaryLocked() { 481 removeBinaryDictionaryLocked(); 482 createOnMemoryBinaryDictionaryLocked(); 483 loadInitialContentsLocked(); 484 // Run GC and flush to file when initial contents have been loaded. 485 mBinaryDictionary.flushWithGCIfHasUpdated(); 486 } 487 488 /** 489 * Marks that the dictionary needs to be reloaded. 490 * 491 */ 492 protected void setNeedsToReload() { 493 mNeedsToReload = true; 494 } 495 496 /** 497 * Load the current binary dictionary from internal storage. If the dictionary file doesn't 498 * exists or needs to be regenerated, the new dictionary file will be asynchronously generated. 499 * However, the dictionary itself is accessible even before the new dictionary file is actually 500 * generated. It may return a null result for getSuggestions() in that case by design. 501 */ 502 public final void reloadDictionaryIfRequired() { 503 if (!isReloadRequired()) return; 504 asyncReloadDictionary(); 505 } 506 507 /** 508 * Returns whether a dictionary reload is required. 509 */ 510 private boolean isReloadRequired() { 511 return mBinaryDictionary == null || mNeedsToReload; 512 } 513 514 /** 515 * Reloads the dictionary. Access is controlled on a per dictionary file basis. 516 */ 517 private final void asyncReloadDictionary() { 518 if (mIsReloading.compareAndSet(false, true)) { 519 asyncExecuteTaskWithWriteLock(new Runnable() { 520 @Override 521 public void run() { 522 try { 523 // TODO: Quit checking contents in ExpandableBinaryDictionary. 524 if (!mDictFile.exists() || (mNeedsToReload && haveContentsChanged())) { 525 // If the dictionary file does not exist or contents have been updated, 526 // generate a new one. 527 createNewDictionaryLocked(); 528 } else if (mBinaryDictionary == null) { 529 // Otherwise, load the existing dictionary. 530 loadBinaryDictionaryLocked(); 531 if (mBinaryDictionary != null && !(isValidDictionaryLocked() 532 // TODO: remove the check below 533 && matchesExpectedBinaryDictFormatVersionForThisType( 534 mBinaryDictionary.getFormatVersion()))) { 535 // Binary dictionary or its format version is not valid. Regenerate 536 // the dictionary file. writeBinaryDictionary will remove the 537 // existing files if appropriate. 538 createNewDictionaryLocked(); 539 } 540 } 541 mNeedsToReload = false; 542 } finally { 543 mIsReloading.set(false); 544 } 545 } 546 }); 547 } 548 } 549 550 /** 551 * Flush binary dictionary to dictionary file. 552 */ 553 public void asyncFlushBinaryDictionary() { 554 asyncExecuteTaskWithWriteLock(new Runnable() { 555 @Override 556 public void run() { 557 if (mBinaryDictionary == null) { 558 return; 559 } 560 if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { 561 mBinaryDictionary.flushWithGC(); 562 } else { 563 mBinaryDictionary.flush(); 564 } 565 } 566 }); 567 } 568 569 // TODO: Implement BinaryDictionary.isInDictionary(). 570 @UsedForTesting 571 public boolean isInUnderlyingBinaryDictionaryForTests(final String word) { 572 mLock.readLock().lock(); 573 try { 574 if (mBinaryDictionary != null && mDictType == Dictionary.TYPE_USER_HISTORY) { 575 return mBinaryDictionary.isValidWord(word); 576 } 577 return false; 578 } finally { 579 mLock.readLock().unlock(); 580 } 581 } 582 583 @UsedForTesting 584 public void waitAllTasksForTests() { 585 final CountDownLatch countDownLatch = new CountDownLatch(1); 586 ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { 587 @Override 588 public void run() { 589 countDownLatch.countDown(); 590 } 591 }); 592 try { 593 countDownLatch.await(); 594 } catch (InterruptedException e) { 595 Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); 596 } 597 } 598 599 @UsedForTesting 600 public void clearAndFlushDictionaryWithAdditionalAttributes( 601 final Map<String, String> attributeMap) { 602 mAdditionalAttributeMap = attributeMap; 603 clear(); 604 } 605 606 public void dumpAllWordsForDebug() { 607 reloadDictionaryIfRequired(); 608 asyncExecuteTaskWithLock(mLock.readLock(), new Runnable() { 609 @Override 610 public void run() { 611 Log.d(TAG, "Dump dictionary: " + mDictName); 612 try { 613 final DictionaryHeader header = mBinaryDictionary.getHeader(); 614 Log.d(TAG, CombinedFormatUtils.formatAttributeMap( 615 header.mDictionaryOptions.mAttributes)); 616 } catch (final UnsupportedFormatException e) { 617 Log.d(TAG, "Cannot fetch header information.", e); 618 } 619 int token = 0; 620 do { 621 final BinaryDictionary.GetNextWordPropertyResult result = 622 mBinaryDictionary.getNextWordProperty(token); 623 final WordProperty wordProperty = result.mWordProperty; 624 if (wordProperty == null) { 625 Log.d(TAG, " dictionary is empty."); 626 break; 627 } 628 Log.d(TAG, wordProperty.toString()); 629 token = result.mNextToken; 630 } while (token != 0); 631 } 632 }); 633 } 634} 635