ExpandableBinaryDictionary.java revision 60eed92dc37e59403142ac35bdf676ae7ceac298
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 * in compliance with the License. You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software distributed under the License 10 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 * or implied. See the License for the specific language governing permissions and limitations under 12 * the License. 13 */ 14 15package com.android.inputmethod.latin; 16 17import android.content.Context; 18import android.os.SystemClock; 19import android.util.Log; 20 21import com.android.inputmethod.keyboard.ProximityInfo; 22import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 23import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; 24import com.android.inputmethod.latin.makedict.FusionDictionary; 25import com.android.inputmethod.latin.makedict.FusionDictionary.Node; 26import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; 27import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 28 29import java.io.File; 30import java.io.FileOutputStream; 31import java.io.IOException; 32import java.util.ArrayList; 33import java.util.HashMap; 34import java.util.concurrent.locks.ReentrantLock; 35 36/** 37 * Abstract base class for an expandable dictionary that can be created and updated dynamically 38 * during runtime. When updated it automatically generates a new binary dictionary to handle future 39 * queries in native code. This binary dictionary is written to internal storage, and potentially 40 * shared across multiple ExpandableBinaryDictionary instances. Updates to each dictionary filename 41 * are controlled across multiple instances to ensure that only one instance can update the same 42 * dictionary at the same time. 43 */ 44abstract public class ExpandableBinaryDictionary extends Dictionary { 45 46 /** Used for Log actions from this class */ 47 private static final String TAG = ExpandableBinaryDictionary.class.getSimpleName(); 48 49 /** Whether to print debug output to log */ 50 private static boolean DEBUG = false; 51 52 /** 53 * The maximum length of a word in this dictionary. This is the same value as the binary 54 * dictionary. 55 */ 56 protected static final int MAX_WORD_LENGTH = BinaryDictionary.MAX_WORD_LENGTH; 57 58 /** 59 * A static map of locks, each of which controls access to a single binary dictionary file. They 60 * ensure that only one instance can update the same dictionary at the same time. The key for 61 * this map is the filename and the value is the shared dictionary controller associated with 62 * that filename. 63 */ 64 private static final HashMap<String, DictionaryController> sSharedDictionaryControllers = 65 new HashMap<String, DictionaryController>(); 66 67 /** The application context. */ 68 protected final Context mContext; 69 70 /** 71 * The binary dictionary generated dynamically from the fusion dictionary. This is used to 72 * answer unigram and bigram queries. 73 */ 74 private BinaryDictionary mBinaryDictionary; 75 76 /** The expandable fusion dictionary used to generate the binary dictionary. */ 77 private FusionDictionary mFusionDictionary; 78 79 /** The dictionary type id. */ 80 public final int mDicTypeId; 81 82 /** 83 * The name of this dictionary, used as the filename for storing the binary dictionary. Multiple 84 * dictionary instances with the same filename is supported, with access controlled by 85 * DictionaryController. 86 */ 87 private final String mFilename; 88 89 /** Controls access to the shared binary dictionary file across multiple instances. */ 90 private final DictionaryController mSharedDictionaryController; 91 92 /** Controls access to the local binary dictionary for this instance. */ 93 private final DictionaryController mLocalDictionaryController = new DictionaryController(); 94 95 /** 96 * Abstract method for loading the unigrams and bigrams of a given dictionary in a background 97 * thread. 98 */ 99 protected abstract void loadDictionaryAsync(); 100 101 /** 102 * Indicates that the source dictionary content has changed and a rebuild of the binary file is 103 * required. If it returns false, the next reload will only read the current binary dictionary 104 * from file. Note that the shared binary dictionary is locked when this is called. 105 */ 106 protected abstract boolean hasContentChanged(); 107 108 /** 109 * Gets the shared dictionary controller for the given filename. 110 */ 111 private static synchronized DictionaryController getSharedDictionaryController( 112 String filename) { 113 DictionaryController controller = sSharedDictionaryControllers.get(filename); 114 if (controller == null) { 115 controller = new DictionaryController(); 116 sSharedDictionaryControllers.put(filename, controller); 117 } 118 return controller; 119 } 120 121 /** 122 * Creates a new expandable binary dictionary. 123 * 124 * @param context The application context of the parent. 125 * @param filename The filename for this binary dictionary. Multiple dictionaries with the same 126 * filename is supported. 127 * @param dictType The type of this dictionary. 128 */ 129 public ExpandableBinaryDictionary( 130 final Context context, final String filename, final int dictType) { 131 mDicTypeId = dictType; 132 mFilename = filename; 133 mContext = context; 134 mBinaryDictionary = null; 135 mSharedDictionaryController = getSharedDictionaryController(filename); 136 clearFusionDictionary(); 137 } 138 139 protected static String getFilenameWithLocale(final String name, final String localeStr) { 140 return name + "." + localeStr + ".dict"; 141 } 142 143 /** 144 * Closes and cleans up the binary dictionary. 145 */ 146 @Override 147 public void close() { 148 // Ensure that no other threads are accessing the local binary dictionary. 149 mLocalDictionaryController.lock(); 150 try { 151 if (mBinaryDictionary != null) { 152 mBinaryDictionary.close(); 153 mBinaryDictionary = null; 154 } 155 } finally { 156 mLocalDictionaryController.unlock(); 157 } 158 } 159 160 /** 161 * Clears the fusion dictionary on the Java side. Note: Does not modify the binary dictionary on 162 * the native side. 163 */ 164 public void clearFusionDictionary() { 165 mFusionDictionary = new FusionDictionary(new Node(), 166 new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, 167 false)); 168 } 169 170 /** 171 * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes 172 * are done to update the binary dictionary. 173 */ 174 // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries, 175 // considering performance regression. 176 protected void addWord(final String word, final String shortcutTarget, final int frequency) { 177 if (shortcutTarget == null) { 178 mFusionDictionary.add(word, frequency, null); 179 } else { 180 // TODO: Do this in the subclass, with this class taking an arraylist. 181 final ArrayList<WeightedString> shortcutTargets = new ArrayList<WeightedString>(); 182 shortcutTargets.add(new WeightedString(shortcutTarget, frequency)); 183 mFusionDictionary.add(word, frequency, shortcutTargets); 184 } 185 } 186 187 /** 188 * Sets a word bigram in the fusion dictionary. Call updateBinaryDictionary when all changes are 189 * done to update the binary dictionary. 190 */ 191 // TODO: Create "cache dictionary" to cache fresh bigrams for frequently updated dictionaries, 192 // considering performance regression. 193 protected void setBigram(final String prevWord, final String word, final int frequency) { 194 mFusionDictionary.setBigram(prevWord, word, frequency); 195 } 196 197 @Override 198 public ArrayList<SuggestedWordInfo> getWords(final WordComposer codes, 199 final CharSequence prevWordForBigrams, final ProximityInfo proximityInfo) { 200 asyncReloadDictionaryIfRequired(); 201 return getWordsInner(codes, prevWordForBigrams, proximityInfo); 202 } 203 204 protected final ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, 205 final CharSequence prevWordForBigrams, final ProximityInfo proximityInfo) { 206 // Ensure that there are no concurrent calls to getWords. If there are, do nothing and 207 // return. 208 if (mLocalDictionaryController.tryLock()) { 209 try { 210 if (mBinaryDictionary != null) { 211 return mBinaryDictionary.getWords(codes, prevWordForBigrams, proximityInfo); 212 } 213 } finally { 214 mLocalDictionaryController.unlock(); 215 } 216 } 217 return null; 218 } 219 220 @Override 221 public ArrayList<SuggestedWordInfo> getBigrams(final WordComposer codes, 222 final CharSequence previousWord, final WordCallback callback) { 223 asyncReloadDictionaryIfRequired(); 224 return getBigramsInner(codes, previousWord, callback); 225 } 226 227 protected ArrayList<SuggestedWordInfo> getBigramsInner(final WordComposer codes, 228 final CharSequence previousWord, final WordCallback callback) { 229 if (mLocalDictionaryController.tryLock()) { 230 try { 231 if (mBinaryDictionary != null) { 232 return mBinaryDictionary.getBigrams(codes, previousWord, callback); 233 } 234 } finally { 235 mLocalDictionaryController.unlock(); 236 } 237 } 238 return null; 239 } 240 241 @Override 242 public boolean isValidWord(final CharSequence word) { 243 asyncReloadDictionaryIfRequired(); 244 return isValidWordInner(word); 245 } 246 247 protected boolean isValidWordInner(final CharSequence word) { 248 if (mLocalDictionaryController.tryLock()) { 249 try { 250 return isValidWordLocked(word); 251 } finally { 252 mLocalDictionaryController.unlock(); 253 } 254 } 255 return false; 256 } 257 258 protected boolean isValidWordLocked(final CharSequence word) { 259 if (mBinaryDictionary == null) return false; 260 return mBinaryDictionary.isValidWord(word); 261 } 262 263 protected boolean isValidBigram(final CharSequence word1, final CharSequence word2) { 264 if (mBinaryDictionary == null) return false; 265 return mBinaryDictionary.isValidBigram(word1, word2); 266 } 267 268 protected boolean isValidBigramInner(final CharSequence word1, final CharSequence word2) { 269 if (mLocalDictionaryController.tryLock()) { 270 try { 271 return isValidBigramLocked(word1, word2); 272 } finally { 273 mLocalDictionaryController.unlock(); 274 } 275 } 276 return false; 277 } 278 279 protected boolean isValidBigramLocked(final CharSequence word1, final CharSequence word2) { 280 if (mBinaryDictionary == null) return false; 281 return mBinaryDictionary.isValidBigram(word1, word2); 282 } 283 284 /** 285 * Load the current binary dictionary from internal storage in a background thread. If no binary 286 * dictionary exists, this method will generate one. 287 */ 288 protected void loadDictionary() { 289 mLocalDictionaryController.mLastUpdateRequestTime = SystemClock.uptimeMillis(); 290 asyncReloadDictionaryIfRequired(); 291 } 292 293 /** 294 * Loads the current binary dictionary from internal storage. Assumes the dictionary file 295 * exists. 296 */ 297 protected void loadBinaryDictionary() { 298 if (DEBUG) { 299 Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" 300 + mSharedDictionaryController.mLastUpdateRequestTime + " update=" 301 + mSharedDictionaryController.mLastUpdateTime); 302 } 303 304 final File file = new File(mContext.getFilesDir(), mFilename); 305 final String filename = file.getAbsolutePath(); 306 final long length = file.length(); 307 308 // Build the new binary dictionary 309 final BinaryDictionary newBinaryDictionary = 310 new BinaryDictionary(mContext, filename, 0, length, true /* useFullEditDistance */, 311 null, mDicTypeId); 312 313 if (mBinaryDictionary != null) { 314 // Ensure all threads accessing the current dictionary have finished before swapping in 315 // the new one. 316 final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; 317 mLocalDictionaryController.lock(); 318 mBinaryDictionary = newBinaryDictionary; 319 mLocalDictionaryController.unlock(); 320 oldBinaryDictionary.close(); 321 } else { 322 mBinaryDictionary = newBinaryDictionary; 323 } 324 } 325 326 /** 327 * Generates and writes a new binary dictionary based on the contents of the fusion dictionary. 328 */ 329 private void generateBinaryDictionary() { 330 if (DEBUG) { 331 Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" 332 + mSharedDictionaryController.mLastUpdateRequestTime + " update=" 333 + mSharedDictionaryController.mLastUpdateTime); 334 } 335 336 loadDictionaryAsync(); 337 338 final String tempFileName = mFilename + ".temp"; 339 final File file = new File(mContext.getFilesDir(), mFilename); 340 final File tempFile = new File(mContext.getFilesDir(), tempFileName); 341 FileOutputStream out = null; 342 try { 343 out = new FileOutputStream(tempFile); 344 BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, 1); 345 out.flush(); 346 out.close(); 347 tempFile.renameTo(file); 348 clearFusionDictionary(); 349 } catch (IOException e) { 350 Log.e(TAG, "IO exception while writing file: " + e); 351 } catch (UnsupportedFormatException e) { 352 Log.e(TAG, "Unsupported format: " + e); 353 } finally { 354 if (out != null) { 355 try { 356 out.close(); 357 } catch (IOException e) { 358 // ignore 359 } 360 } 361 } 362 } 363 364 /** 365 * Marks that the dictionary is out of date and requires a reload. 366 * 367 * @param requiresRebuild Indicates that the source dictionary content has changed and a rebuild 368 * of the binary file is required. If not true, the next reload process will only read 369 * the current binary dictionary from file. 370 */ 371 protected void setRequiresReload(final boolean requiresRebuild) { 372 final long time = SystemClock.uptimeMillis(); 373 mLocalDictionaryController.mLastUpdateRequestTime = time; 374 mSharedDictionaryController.mLastUpdateRequestTime = time; 375 if (DEBUG) { 376 Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update=" 377 + mSharedDictionaryController.mLastUpdateTime); 378 } 379 } 380 381 /** 382 * Reloads the dictionary if required. Reload will occur asynchronously in a separate thread. 383 */ 384 void asyncReloadDictionaryIfRequired() { 385 if (!isReloadRequired()) return; 386 if (DEBUG) { 387 Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename); 388 } 389 new AsyncReloadDictionaryTask().start(); 390 } 391 392 /** 393 * Reloads the dictionary if required. 394 */ 395 protected final void syncReloadDictionaryIfRequired() { 396 if (!isReloadRequired()) return; 397 syncReloadDictionaryInternal(); 398 } 399 400 /** 401 * Returns whether a dictionary reload is required. 402 */ 403 private boolean isReloadRequired() { 404 return mBinaryDictionary == null || mLocalDictionaryController.isOutOfDate(); 405 } 406 407 /** 408 * Reloads the dictionary. Access is controlled on a per dictionary file basis and supports 409 * concurrent calls from multiple instances that share the same dictionary file. 410 */ 411 private final void syncReloadDictionaryInternal() { 412 // Ensure that only one thread attempts to read or write to the shared binary dictionary 413 // file at the same time. 414 mSharedDictionaryController.lock(); 415 try { 416 final long time = SystemClock.uptimeMillis(); 417 final boolean dictionaryFileExists = dictionaryFileExists(); 418 if (mSharedDictionaryController.isOutOfDate() || !dictionaryFileExists) { 419 // If the shared dictionary file does not exist or is out of date, the first 420 // instance that acquires the lock will generate a new one. 421 if (hasContentChanged() || !dictionaryFileExists) { 422 // If the source content has changed or the dictionary does not exist, rebuild 423 // the binary dictionary. Empty dictionaries are supported (in the case where 424 // loadDictionaryAsync() adds nothing) in order to provide a uniform framework. 425 mSharedDictionaryController.mLastUpdateTime = time; 426 generateBinaryDictionary(); 427 loadBinaryDictionary(); 428 } else { 429 // If not, the reload request was unnecessary so revert LastUpdateRequestTime 430 // to LastUpdateTime. 431 mSharedDictionaryController.mLastUpdateRequestTime = 432 mSharedDictionaryController.mLastUpdateTime; 433 } 434 } else if (mBinaryDictionary == null || mLocalDictionaryController.mLastUpdateTime 435 < mSharedDictionaryController.mLastUpdateTime) { 436 // Otherwise, if the local dictionary is older than the shared dictionary, load the 437 // shared dictionary. 438 loadBinaryDictionary(); 439 } 440 mLocalDictionaryController.mLastUpdateTime = time; 441 } finally { 442 mSharedDictionaryController.unlock(); 443 } 444 } 445 446 // TODO: cache the file's existence so that we avoid doing a disk access each time. 447 private boolean dictionaryFileExists() { 448 final File file = new File(mContext.getFilesDir(), mFilename); 449 return file.exists(); 450 } 451 452 /** 453 * Thread class for asynchronously reloading and rewriting the binary dictionary. 454 */ 455 private class AsyncReloadDictionaryTask extends Thread { 456 @Override 457 public void run() { 458 syncReloadDictionaryInternal(); 459 } 460 } 461 462 /** 463 * Lock for controlling access to a given binary dictionary and for tracking whether the 464 * dictionary is out of date. Can be shared across multiple dictionary instances that access the 465 * same filename. 466 */ 467 private static class DictionaryController extends ReentrantLock { 468 private volatile long mLastUpdateTime = 0; 469 private volatile long mLastUpdateRequestTime = 0; 470 471 private boolean isOutOfDate() { 472 return (mLastUpdateRequestTime > mLastUpdateTime); 473 } 474 } 475} 476