DictionaryFacilitator.java revision e708b1bc2e11285ad404133b8de21719ce08acb5
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.content.Context; 20import android.text.TextUtils; 21import android.util.Log; 22import android.view.inputmethod.InputMethodSubtype; 23 24import com.android.inputmethod.annotations.UsedForTesting; 25import com.android.inputmethod.keyboard.ProximityInfo; 26import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; 27import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 28import com.android.inputmethod.latin.personalization.ContextualDictionary; 29import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; 30import com.android.inputmethod.latin.personalization.PersonalizationDictionary; 31import com.android.inputmethod.latin.personalization.UserHistoryDictionary; 32import com.android.inputmethod.latin.settings.SpacingAndPunctuations; 33import com.android.inputmethod.latin.utils.DistracterFilter; 34import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; 35import com.android.inputmethod.latin.utils.ExecutorUtils; 36import com.android.inputmethod.latin.utils.LanguageModelParam; 37import com.android.inputmethod.latin.utils.SuggestionResults; 38 39import java.io.File; 40import java.lang.reflect.InvocationTargetException; 41import java.lang.reflect.Method; 42import java.util.ArrayList; 43import java.util.Arrays; 44import java.util.HashMap; 45import java.util.HashSet; 46import java.util.List; 47import java.util.Locale; 48import java.util.Map; 49import java.util.concurrent.ConcurrentHashMap; 50import java.util.concurrent.CountDownLatch; 51import java.util.concurrent.TimeUnit; 52 53// TODO: Consolidate dictionaries in native code. 54public class DictionaryFacilitator { 55 public static final String TAG = DictionaryFacilitator.class.getSimpleName(); 56 57 // HACK: This threshold is being used when adding a capitalized entry in the User History 58 // dictionary. 59 private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140; 60 61 private Dictionaries mDictionaries = new Dictionaries(); 62 private boolean mIsUserDictEnabled = false; 63 private volatile CountDownLatch mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); 64 // To synchronize assigning mDictionaries to ensure closing dictionaries. 65 private final Object mLock = new Object(); 66 private final DistracterFilter mDistracterFilter; 67 68 private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS = 69 new String[] { 70 Dictionary.TYPE_MAIN, 71 Dictionary.TYPE_USER_HISTORY, 72 Dictionary.TYPE_PERSONALIZATION, 73 Dictionary.TYPE_USER, 74 Dictionary.TYPE_CONTACTS, 75 Dictionary.TYPE_CONTEXTUAL 76 }; 77 78 public static final Map<String, Class<? extends ExpandableBinaryDictionary>> 79 DICT_TYPE_TO_CLASS = new HashMap<>(); 80 81 static { 82 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER_HISTORY, UserHistoryDictionary.class); 83 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_PERSONALIZATION, PersonalizationDictionary.class); 84 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER, UserBinaryDictionary.class); 85 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTACTS, ContactsBinaryDictionary.class); 86 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTEXTUAL, ContextualDictionary.class); 87 } 88 89 private static final String DICT_FACTORY_METHOD_NAME = "getDictionary"; 90 private static final Class<?>[] DICT_FACTORY_METHOD_ARG_TYPES = 91 new Class[] { Context.class, Locale.class, File.class, String.class }; 92 93 private static final String[] SUB_DICT_TYPES = 94 Arrays.copyOfRange(DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS, 1 /* start */, 95 DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS.length); 96 97 /** 98 * Class contains dictionaries for a locale. 99 */ 100 private static class Dictionaries { 101 public final Locale mLocale; 102 private Dictionary mMainDict; 103 public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap = 104 new ConcurrentHashMap<>(); 105 106 public Dictionaries() { 107 mLocale = null; 108 } 109 110 public Dictionaries(final Locale locale, final Dictionary mainDict, 111 final Map<String, ExpandableBinaryDictionary> subDicts) { 112 mLocale = locale; 113 // Main dictionary can be asynchronously loaded. 114 setMainDict(mainDict); 115 for (final Map.Entry<String, ExpandableBinaryDictionary> entry : subDicts.entrySet()) { 116 setSubDict(entry.getKey(), entry.getValue()); 117 } 118 } 119 120 private void setSubDict(final String dictType, final ExpandableBinaryDictionary dict) { 121 if (dict != null) { 122 mSubDictMap.put(dictType, dict); 123 } 124 } 125 126 public void setMainDict(final Dictionary mainDict) { 127 // Close old dictionary if exists. Main dictionary can be assigned multiple times. 128 final Dictionary oldDict = mMainDict; 129 mMainDict = mainDict; 130 if (oldDict != null && mainDict != oldDict) { 131 oldDict.close(); 132 } 133 } 134 135 public Dictionary getDict(final String dictType) { 136 if (Dictionary.TYPE_MAIN.equals(dictType)) { 137 return mMainDict; 138 } else { 139 return getSubDict(dictType); 140 } 141 } 142 143 public ExpandableBinaryDictionary getSubDict(final String dictType) { 144 return mSubDictMap.get(dictType); 145 } 146 147 public boolean hasDict(final String dictType) { 148 if (Dictionary.TYPE_MAIN.equals(dictType)) { 149 return mMainDict != null; 150 } else { 151 return mSubDictMap.containsKey(dictType); 152 } 153 } 154 155 public void closeDict(final String dictType) { 156 final Dictionary dict; 157 if (Dictionary.TYPE_MAIN.equals(dictType)) { 158 dict = mMainDict; 159 } else { 160 dict = mSubDictMap.remove(dictType); 161 } 162 if (dict != null) { 163 dict.close(); 164 } 165 } 166 } 167 168 public interface DictionaryInitializationListener { 169 public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); 170 } 171 172 public DictionaryFacilitator() { 173 mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; 174 } 175 176 public DictionaryFacilitator(final DistracterFilter distracterFilter) { 177 mDistracterFilter = distracterFilter; 178 } 179 180 public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { 181 mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); 182 } 183 184 public Locale getLocale() { 185 return mDictionaries.mLocale; 186 } 187 188 private static ExpandableBinaryDictionary getSubDict(final String dictType, 189 final Context context, final Locale locale, final File dictFile, 190 final String dictNamePrefix) { 191 final Class<? extends ExpandableBinaryDictionary> dictClass = 192 DICT_TYPE_TO_CLASS.get(dictType); 193 if (dictClass == null) { 194 return null; 195 } 196 try { 197 final Method factoryMethod = dictClass.getMethod(DICT_FACTORY_METHOD_NAME, 198 DICT_FACTORY_METHOD_ARG_TYPES); 199 final Object dict = factoryMethod.invoke(null /* obj */, 200 new Object[] { context, locale, dictFile, dictNamePrefix }); 201 return (ExpandableBinaryDictionary) dict; 202 } catch (final NoSuchMethodException | SecurityException | IllegalAccessException 203 | IllegalArgumentException | InvocationTargetException e) { 204 Log.e(TAG, "Cannot create dictionary: " + dictType, e); 205 return null; 206 } 207 } 208 209 public void resetDictionaries(final Context context, final Locale newLocale, 210 final boolean useContactsDict, final boolean usePersonalizedDicts, 211 final boolean forceReloadMainDictionary, 212 final DictionaryInitializationListener listener) { 213 resetDictionariesWithDictNamePrefix(context, newLocale, useContactsDict, 214 usePersonalizedDicts, forceReloadMainDictionary, listener, "" /* dictNamePrefix */); 215 } 216 217 public void resetDictionariesWithDictNamePrefix(final Context context, final Locale newLocale, 218 final boolean useContactsDict, final boolean usePersonalizedDicts, 219 final boolean forceReloadMainDictionary, 220 final DictionaryInitializationListener listener, 221 final String dictNamePrefix) { 222 final boolean localeHasBeenChanged = !newLocale.equals(mDictionaries.mLocale); 223 // We always try to have the main dictionary. Other dictionaries can be unused. 224 final boolean reloadMainDictionary = localeHasBeenChanged || forceReloadMainDictionary; 225 // TODO: Make subDictTypesToUse configurable by resource or a static final list. 226 final HashSet<String> subDictTypesToUse = new HashSet<>(); 227 if (useContactsDict) { 228 subDictTypesToUse.add(Dictionary.TYPE_CONTACTS); 229 } 230 subDictTypesToUse.add(Dictionary.TYPE_USER); 231 if (usePersonalizedDicts) { 232 subDictTypesToUse.add(Dictionary.TYPE_USER_HISTORY); 233 subDictTypesToUse.add(Dictionary.TYPE_PERSONALIZATION); 234 subDictTypesToUse.add(Dictionary.TYPE_CONTEXTUAL); 235 } 236 237 final Dictionary newMainDict; 238 if (reloadMainDictionary) { 239 // The main dictionary will be asynchronously loaded. 240 newMainDict = null; 241 } else { 242 newMainDict = mDictionaries.getDict(Dictionary.TYPE_MAIN); 243 } 244 245 final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>(); 246 for (final String dictType : SUB_DICT_TYPES) { 247 if (!subDictTypesToUse.contains(dictType)) { 248 // This dictionary will not be used. 249 continue; 250 } 251 final ExpandableBinaryDictionary dict; 252 if (!localeHasBeenChanged && mDictionaries.hasDict(dictType)) { 253 // Continue to use current dictionary. 254 dict = mDictionaries.getSubDict(dictType); 255 } else { 256 // Start to use new dictionary. 257 dict = getSubDict(dictType, context, newLocale, null /* dictFile */, 258 dictNamePrefix); 259 } 260 subDicts.put(dictType, dict); 261 } 262 263 // Replace Dictionaries. 264 final Dictionaries newDictionaries = new Dictionaries(newLocale, newMainDict, subDicts); 265 final Dictionaries oldDictionaries; 266 synchronized (mLock) { 267 oldDictionaries = mDictionaries; 268 mDictionaries = newDictionaries; 269 mIsUserDictEnabled = UserBinaryDictionary.isEnabled(context); 270 if (reloadMainDictionary) { 271 asyncReloadMainDictionary(context, newLocale, listener); 272 } 273 } 274 if (listener != null) { 275 listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); 276 } 277 // Clean up old dictionaries. 278 if (reloadMainDictionary) { 279 oldDictionaries.closeDict(Dictionary.TYPE_MAIN); 280 } 281 for (final String dictType : SUB_DICT_TYPES) { 282 if (localeHasBeenChanged || !subDictTypesToUse.contains(dictType)) { 283 oldDictionaries.closeDict(dictType); 284 } 285 } 286 oldDictionaries.mSubDictMap.clear(); 287 } 288 289 private void asyncReloadMainDictionary(final Context context, final Locale locale, 290 final DictionaryInitializationListener listener) { 291 final CountDownLatch latchForWaitingLoadingMainDictionary = new CountDownLatch(1); 292 mLatchForWaitingLoadingMainDictionary = latchForWaitingLoadingMainDictionary; 293 ExecutorUtils.getExecutor("InitializeBinaryDictionary").execute(new Runnable() { 294 @Override 295 public void run() { 296 final Dictionary mainDict = 297 DictionaryFactory.createMainDictionaryFromManager(context, locale); 298 synchronized (mLock) { 299 if (locale.equals(mDictionaries.mLocale)) { 300 mDictionaries.setMainDict(mainDict); 301 } else { 302 // Dictionary facilitator has been reset for another locale. 303 mainDict.close(); 304 } 305 } 306 if (listener != null) { 307 listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); 308 } 309 latchForWaitingLoadingMainDictionary.countDown(); 310 } 311 }); 312 } 313 314 @UsedForTesting 315 public void resetDictionariesForTesting(final Context context, final Locale locale, 316 final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles, 317 final Map<String, Map<String, String>> additionalDictAttributes) { 318 Dictionary mainDictionary = null; 319 final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>(); 320 321 for (final String dictType : dictionaryTypes) { 322 if (dictType.equals(Dictionary.TYPE_MAIN)) { 323 mainDictionary = DictionaryFactory.createMainDictionaryFromManager(context, locale); 324 } else { 325 final File dictFile = dictionaryFiles.get(dictType); 326 final ExpandableBinaryDictionary dict = getSubDict( 327 dictType, context, locale, dictFile, "" /* dictNamePrefix */); 328 if (additionalDictAttributes.containsKey(dictType)) { 329 dict.clearAndFlushDictionaryWithAdditionalAttributes( 330 additionalDictAttributes.get(dictType)); 331 } 332 if (dict == null) { 333 throw new RuntimeException("Unknown dictionary type: " + dictType); 334 } 335 dict.reloadDictionaryIfRequired(); 336 dict.waitAllTasksForTests(); 337 subDicts.put(dictType, dict); 338 } 339 } 340 mDictionaries = new Dictionaries(locale, mainDictionary, subDicts); 341 } 342 343 public void closeDictionaries() { 344 final Dictionaries dictionaries; 345 synchronized (mLock) { 346 dictionaries = mDictionaries; 347 mDictionaries = new Dictionaries(); 348 } 349 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 350 dictionaries.closeDict(dictType); 351 } 352 mDistracterFilter.close(); 353 } 354 355 @UsedForTesting 356 public ExpandableBinaryDictionary getSubDictForTesting(final String dictName) { 357 return mDictionaries.getSubDict(dictName); 358 } 359 360 // The main dictionary could have been loaded asynchronously. Don't cache the return value 361 // of this method. 362 public boolean hasInitializedMainDictionary() { 363 final Dictionary mainDict = mDictionaries.getDict(Dictionary.TYPE_MAIN); 364 return mainDict != null && mainDict.isInitialized(); 365 } 366 367 public boolean hasPersonalizationDictionary() { 368 return mDictionaries.hasDict(Dictionary.TYPE_PERSONALIZATION); 369 } 370 371 public void flushPersonalizationDictionary() { 372 final ExpandableBinaryDictionary personalizationDict = 373 mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); 374 if (personalizationDict != null) { 375 personalizationDict.asyncFlushBinaryDictionary(); 376 } 377 } 378 379 public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit) 380 throws InterruptedException { 381 mLatchForWaitingLoadingMainDictionary.await(timeout, unit); 382 } 383 384 @UsedForTesting 385 public void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit) 386 throws InterruptedException { 387 waitForLoadingMainDictionary(timeout, unit); 388 final Map<String, ExpandableBinaryDictionary> dictMap = mDictionaries.mSubDictMap; 389 for (final ExpandableBinaryDictionary dict : dictMap.values()) { 390 dict.waitAllTasksForTests(); 391 } 392 } 393 394 public boolean isUserDictionaryEnabled() { 395 return mIsUserDictEnabled; 396 } 397 398 public void addWordToUserDictionary(final Context context, final String word) { 399 final Locale locale = getLocale(); 400 if (locale == null) { 401 return; 402 } 403 UserBinaryDictionary.addWordToUserDictionary(context, locale, word); 404 } 405 406 public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, 407 final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds, 408 final boolean blockPotentiallyOffensive) { 409 final Dictionaries dictionaries = mDictionaries; 410 final String[] words = suggestion.split(Constants.WORD_SEPARATOR); 411 PrevWordsInfo prevWordsInfoForCurrentWord = prevWordsInfo; 412 for (int i = 0; i < words.length; i++) { 413 final String currentWord = words[i]; 414 final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false; 415 addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord, 416 wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive); 417 prevWordsInfoForCurrentWord = 418 prevWordsInfoForCurrentWord.getNextPrevWordsInfo(new WordInfo(currentWord)); 419 } 420 } 421 422 private void addWordToUserHistory(final Dictionaries dictionaries, 423 final PrevWordsInfo prevWordsInfo, final String word, final boolean wasAutoCapitalized, 424 final int timeStampInSeconds, final boolean blockPotentiallyOffensive) { 425 final ExpandableBinaryDictionary userHistoryDictionary = 426 dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); 427 if (userHistoryDictionary == null) { 428 return; 429 } 430 final int maxFreq = getFrequency(word); 431 if (maxFreq == 0 && blockPotentiallyOffensive) { 432 return; 433 } 434 final String lowerCasedWord = word.toLowerCase(dictionaries.mLocale); 435 final String secondWord; 436 if (wasAutoCapitalized) { 437 if (isValidWord(word, false /* ignoreCase */) 438 && !isValidWord(lowerCasedWord, false /* ignoreCase */)) { 439 // If the word was auto-capitalized and exists only as a capitalized word in the 440 // dictionary, then we must not downcase it before registering it. For example, 441 // the name of the contacts in start-of-sentence position would come here with the 442 // wasAutoCapitalized flag: if we downcase it, we'd register a lower-case version 443 // of that contact's name which would end up popping in suggestions. 444 secondWord = word; 445 } else { 446 // If however the word is not in the dictionary, or exists as a lower-case word 447 // only, then we consider that was a lower-case word that had been auto-capitalized. 448 secondWord = lowerCasedWord; 449 } 450 } else { 451 // HACK: We'd like to avoid adding the capitalized form of common words to the User 452 // History dictionary in order to avoid suggesting them until the dictionary 453 // consolidation is done. 454 // TODO: Remove this hack when ready. 455 final int lowerCaseFreqInMainDict = dictionaries.hasDict(Dictionary.TYPE_MAIN) ? 456 dictionaries.getDict(Dictionary.TYPE_MAIN).getFrequency(lowerCasedWord) : 457 Dictionary.NOT_A_PROBABILITY; 458 if (maxFreq < lowerCaseFreqInMainDict 459 && lowerCaseFreqInMainDict >= CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT) { 460 // Use lower cased word as the word can be a distracter of the popular word. 461 secondWord = lowerCasedWord; 462 } else { 463 secondWord = word; 464 } 465 } 466 // We demote unrecognized words (frequency < 0, below) by specifying them as "invalid". 467 // We don't add words with 0-frequency (assuming they would be profanity etc.). 468 final boolean isValid = maxFreq > 0; 469 UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord, 470 isValid, timeStampInSeconds, mDistracterFilter); 471 } 472 473 private void removeWord(final String dictName, final String word) { 474 final ExpandableBinaryDictionary dictionary = mDictionaries.getSubDict(dictName); 475 if (dictionary != null) { 476 dictionary.removeUnigramEntryDynamically(word); 477 } 478 } 479 480 public void removeWordFromPersonalizedDicts(final String word) { 481 removeWord(Dictionary.TYPE_USER_HISTORY, word); 482 removeWord(Dictionary.TYPE_PERSONALIZATION, word); 483 removeWord(Dictionary.TYPE_CONTEXTUAL, word); 484 } 485 486 // TODO: Revise the way to fusion suggestion results. 487 public SuggestionResults getSuggestionResults(final WordComposer composer, 488 final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, 489 final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, 490 final int sessionId, final ArrayList<SuggestedWordInfo> rawSuggestions) { 491 final Dictionaries dictionaries = mDictionaries; 492 final SuggestionResults suggestionResults = 493 new SuggestionResults(dictionaries.mLocale, SuggestedWords.MAX_SUGGESTIONS); 494 final float[] languageWeight = new float[] { Dictionary.NOT_A_LANGUAGE_WEIGHT }; 495 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 496 final Dictionary dictionary = dictionaries.getDict(dictType); 497 if (null == dictionary) continue; 498 final ArrayList<SuggestedWordInfo> dictionarySuggestions = 499 dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, 500 blockOffensiveWords, additionalFeaturesOptions, sessionId, 501 languageWeight); 502 if (null == dictionarySuggestions) continue; 503 suggestionResults.addAll(dictionarySuggestions); 504 if (null != rawSuggestions) { 505 rawSuggestions.addAll(dictionarySuggestions); 506 } 507 } 508 return suggestionResults; 509 } 510 511 public boolean isValidMainDictWord(final String word) { 512 final Dictionary mainDict = mDictionaries.getDict(Dictionary.TYPE_MAIN); 513 if (TextUtils.isEmpty(word) || mainDict == null) { 514 return false; 515 } 516 return mainDict.isValidWord(word); 517 } 518 519 public boolean isValidWord(final String word, final boolean ignoreCase) { 520 if (TextUtils.isEmpty(word)) { 521 return false; 522 } 523 final Dictionaries dictionaries = mDictionaries; 524 if (dictionaries.mLocale == null) { 525 return false; 526 } 527 final String lowerCasedWord = word.toLowerCase(dictionaries.mLocale); 528 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 529 final Dictionary dictionary = dictionaries.getDict(dictType); 530 // Ideally the passed map would come out of a {@link java.util.concurrent.Future} and 531 // would be immutable once it's finished initializing, but concretely a null test is 532 // probably good enough for the time being. 533 if (null == dictionary) continue; 534 if (dictionary.isValidWord(word) 535 || (ignoreCase && dictionary.isValidWord(lowerCasedWord))) { 536 return true; 537 } 538 } 539 return false; 540 } 541 542 private int getFrequencyInternal(final String word, 543 final boolean isGettingMaxFrequencyOfExactMatches) { 544 if (TextUtils.isEmpty(word)) { 545 return Dictionary.NOT_A_PROBABILITY; 546 } 547 int maxFreq = Dictionary.NOT_A_PROBABILITY; 548 final Dictionaries dictionaries = mDictionaries; 549 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 550 final Dictionary dictionary = dictionaries.getDict(dictType); 551 if (dictionary == null) continue; 552 final int tempFreq; 553 if (isGettingMaxFrequencyOfExactMatches) { 554 tempFreq = dictionary.getMaxFrequencyOfExactMatches(word); 555 } else { 556 tempFreq = dictionary.getFrequency(word); 557 } 558 if (tempFreq >= maxFreq) { 559 maxFreq = tempFreq; 560 } 561 } 562 return maxFreq; 563 } 564 565 public int getFrequency(final String word) { 566 return getFrequencyInternal(word, false /* isGettingMaxFrequencyOfExactMatches */); 567 } 568 569 public int getMaxFrequencyOfExactMatches(final String word) { 570 return getFrequencyInternal(word, true /* isGettingMaxFrequencyOfExactMatches */); 571 } 572 573 private void clearSubDictionary(final String dictName) { 574 final ExpandableBinaryDictionary dictionary = mDictionaries.getSubDict(dictName); 575 if (dictionary != null) { 576 dictionary.clear(); 577 } 578 } 579 580 public void clearUserHistoryDictionary() { 581 clearSubDictionary(Dictionary.TYPE_USER_HISTORY); 582 } 583 584 // This method gets called only when the IME receives a notification to remove the 585 // personalization dictionary. 586 public void clearPersonalizationDictionary() { 587 clearSubDictionary(Dictionary.TYPE_PERSONALIZATION); 588 } 589 590 public void clearContextualDictionary() { 591 clearSubDictionary(Dictionary.TYPE_CONTEXTUAL); 592 } 593 594 public void addEntriesToPersonalizationDictionary( 595 final PersonalizationDataChunk personalizationDataChunk, 596 final SpacingAndPunctuations spacingAndPunctuations, 597 final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { 598 final ExpandableBinaryDictionary personalizationDict = 599 mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); 600 if (personalizationDict == null) { 601 if (callback != null) { 602 callback.onFinished(); 603 } 604 return; 605 } 606 final ArrayList<LanguageModelParam> languageModelParams = 607 LanguageModelParam.createLanguageModelParamsFrom( 608 personalizationDataChunk.mTokens, 609 personalizationDataChunk.mTimestampInSeconds, 610 this /* dictionaryFacilitator */, spacingAndPunctuations, 611 new DistracterFilterCheckingIsInDictionary( 612 mDistracterFilter, personalizationDict)); 613 if (languageModelParams == null || languageModelParams.isEmpty()) { 614 if (callback != null) { 615 callback.onFinished(); 616 } 617 return; 618 } 619 personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback); 620 } 621 622 public void addPhraseToContextualDictionary(final String[] phrase, final int probability, 623 final int bigramProbabilityForWords, final int bigramProbabilityForPhrases) { 624 final ExpandableBinaryDictionary contextualDict = 625 mDictionaries.getSubDict(Dictionary.TYPE_CONTEXTUAL); 626 if (contextualDict == null) { 627 return; 628 } 629 PrevWordsInfo prevWordsInfo = PrevWordsInfo.BEGINNING_OF_SENTENCE; 630 for (int i = 0; i < phrase.length; i++) { 631 final String[] subPhrase = Arrays.copyOfRange(phrase, i /* start */, phrase.length); 632 final String subPhraseStr = TextUtils.join(Constants.WORD_SEPARATOR, subPhrase); 633 contextualDict.addUnigramEntryWithCheckingDistracter( 634 subPhraseStr, probability, null /* shortcutTarget */, 635 Dictionary.NOT_A_PROBABILITY /* shortcutFreq */, 636 false /* isNotAWord */, false /* isBlacklisted */, 637 BinaryDictionary.NOT_A_VALID_TIMESTAMP, 638 DistracterFilter.EMPTY_DISTRACTER_FILTER); 639 contextualDict.addNgramEntry(prevWordsInfo, subPhraseStr, 640 bigramProbabilityForPhrases, BinaryDictionary.NOT_A_VALID_TIMESTAMP); 641 642 if (i < phrase.length - 1) { 643 contextualDict.addUnigramEntryWithCheckingDistracter( 644 phrase[i], probability, null /* shortcutTarget */, 645 Dictionary.NOT_A_PROBABILITY /* shortcutFreq */, 646 false /* isNotAWord */, false /* isBlacklisted */, 647 BinaryDictionary.NOT_A_VALID_TIMESTAMP, 648 DistracterFilter.EMPTY_DISTRACTER_FILTER); 649 contextualDict.addNgramEntry(prevWordsInfo, phrase[i], 650 bigramProbabilityForWords, BinaryDictionary.NOT_A_VALID_TIMESTAMP); 651 } 652 prevWordsInfo = 653 prevWordsInfo.getNextPrevWordsInfo(new PrevWordsInfo.WordInfo(phrase[i])); 654 } 655 } 656 657 public void dumpDictionaryForDebug(final String dictName) { 658 final ExpandableBinaryDictionary dictToDump = mDictionaries.getSubDict(dictName); 659 if (dictToDump == null) { 660 Log.e(TAG, "Cannot dump " + dictName + ". " 661 + "The dictionary is not being used for suggestion or cannot be dumped."); 662 return; 663 } 664 dictToDump.dumpAllWordsForDebug(); 665 } 666} 667