BinaryDictionaryFileDumper.java revision 2fa3693c264a4c150ac307d9bb7f6f8f18cc4ffc
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.content.ContentProviderClient; 20import android.content.ContentResolver; 21import android.content.ContentValues; 22import android.content.Context; 23import android.content.res.AssetFileDescriptor; 24import android.database.Cursor; 25import android.net.Uri; 26import android.os.RemoteException; 27import android.text.TextUtils; 28import android.util.Log; 29 30import com.android.inputmethod.dictionarypack.DictionaryPackConstants; 31import com.android.inputmethod.latin.utils.CollectionUtils; 32import com.android.inputmethod.latin.utils.DictionaryInfoUtils; 33import com.android.inputmethod.latin.utils.DictionaryInfoUtils.DictionaryInfo; 34import com.android.inputmethod.latin.utils.FileTransforms; 35import com.android.inputmethod.latin.utils.MetadataFileUriGetter; 36 37import java.io.BufferedInputStream; 38import java.io.BufferedOutputStream; 39import java.io.Closeable; 40import java.io.File; 41import java.io.FileNotFoundException; 42import java.io.FileOutputStream; 43import java.io.IOException; 44import java.io.InputStream; 45import java.util.Arrays; 46import java.util.ArrayList; 47import java.util.Collections; 48import java.util.List; 49import java.util.Locale; 50 51/** 52 * Group class for static methods to help with creation and getting of the binary dictionary 53 * file from the dictionary provider 54 */ 55public final class BinaryDictionaryFileDumper { 56 private static final String TAG = BinaryDictionaryFileDumper.class.getSimpleName(); 57 private static final boolean DEBUG = false; 58 59 /** 60 * The size of the temporary buffer to copy files. 61 */ 62 private static final int FILE_READ_BUFFER_SIZE = 8192; 63 // TODO: make the following data common with the native code 64 private static final byte[] MAGIC_NUMBER_VERSION_1 = 65 new byte[] { (byte)0x78, (byte)0xB1, (byte)0x00, (byte)0x00 }; 66 private static final byte[] MAGIC_NUMBER_VERSION_2 = 67 new byte[] { (byte)0x9B, (byte)0xC1, (byte)0x3A, (byte)0xFE }; 68 69 private static final String DICTIONARY_PROJECTION[] = { "id" }; 70 71 private static final String QUERY_PARAMETER_MAY_PROMPT_USER = "mayPrompt"; 72 private static final String QUERY_PARAMETER_TRUE = "true"; 73 private static final String QUERY_PARAMETER_DELETE_RESULT = "result"; 74 private static final String QUERY_PARAMETER_SUCCESS = "success"; 75 private static final String QUERY_PARAMETER_FAILURE = "failure"; 76 77 // Using protocol version 2 to communicate with the dictionary pack 78 private static final String QUERY_PARAMETER_PROTOCOL = "protocol"; 79 private static final String QUERY_PARAMETER_PROTOCOL_VALUE = "2"; 80 81 // The path fragment to append after the client ID for dictionary info requests. 82 private static final String QUERY_PATH_DICT_INFO = "dict"; 83 // The path fragment to append after the client ID for dictionary datafile requests. 84 private static final String QUERY_PATH_DATAFILE = "datafile"; 85 // The path fragment to append after the client ID for updating the metadata URI. 86 private static final String QUERY_PATH_METADATA = "metadata"; 87 private static final String INSERT_METADATA_CLIENT_ID_COLUMN = "clientid"; 88 private static final String INSERT_METADATA_METADATA_URI_COLUMN = "uri"; 89 private static final String INSERT_METADATA_METADATA_ADDITIONAL_ID_COLUMN = "additionalid"; 90 91 // Prevents this class to be accidentally instantiated. 92 private BinaryDictionaryFileDumper() { 93 } 94 95 /** 96 * Returns a URI builder pointing to the dictionary pack. 97 * 98 * This creates a URI builder able to build a URI pointing to the dictionary 99 * pack content provider for a specific dictionary id. 100 */ 101 public static Uri.Builder getProviderUriBuilder(final String path) { 102 return new Uri.Builder().scheme(ContentResolver.SCHEME_CONTENT) 103 .authority(DictionaryPackConstants.AUTHORITY).appendPath(path); 104 } 105 106 /** 107 * Gets the content URI builder for a specified type. 108 * 109 * Supported types include QUERY_PATH_DICT_INFO, which takes the locale as 110 * the extraPath argument, and QUERY_PATH_DATAFILE, which needs a wordlist ID 111 * as the extraPath argument. 112 * 113 * @param clientId the clientId to use 114 * @param contentProviderClient the instance of content provider client 115 * @param queryPathType the path element encoding the type 116 * @param extraPath optional extra argument for this type (typically word list id) 117 * @return a builder that can build the URI for the best supported protocol version 118 * @throws RemoteException if the client can't be contacted 119 */ 120 private static Uri.Builder getContentUriBuilderForType(final String clientId, 121 final ContentProviderClient contentProviderClient, final String queryPathType, 122 final String extraPath) throws RemoteException { 123 // Check whether protocol v2 is supported by building a v2 URI and calling getType() 124 // on it. If this returns null, v2 is not supported. 125 final Uri.Builder uriV2Builder = getProviderUriBuilder(clientId); 126 uriV2Builder.appendPath(queryPathType); 127 uriV2Builder.appendPath(extraPath); 128 uriV2Builder.appendQueryParameter(QUERY_PARAMETER_PROTOCOL, 129 QUERY_PARAMETER_PROTOCOL_VALUE); 130 if (null != contentProviderClient.getType(uriV2Builder.build())) return uriV2Builder; 131 // Protocol v2 is not supported, so create and return the protocol v1 uri. 132 return getProviderUriBuilder(extraPath); 133 } 134 135 /** 136 * Queries a content provider for the list of word lists for a specific locale 137 * available to copy into Latin IME. 138 */ 139 private static List<WordListInfo> getWordListWordListInfos(final Locale locale, 140 final Context context, final boolean hasDefaultWordList) { 141 final String clientId = context.getString(R.string.dictionary_pack_client_id); 142 final ContentProviderClient client = context.getContentResolver(). 143 acquireContentProviderClient(getProviderUriBuilder("").build()); 144 if (null == client) return Collections.<WordListInfo>emptyList(); 145 146 try { 147 final Uri.Builder builder = getContentUriBuilderForType(clientId, client, 148 QUERY_PATH_DICT_INFO, locale.toString()); 149 if (!hasDefaultWordList) { 150 builder.appendQueryParameter(QUERY_PARAMETER_MAY_PROMPT_USER, 151 QUERY_PARAMETER_TRUE); 152 } 153 final Uri queryUri = builder.build(); 154 final boolean isProtocolV2 = (QUERY_PARAMETER_PROTOCOL_VALUE.equals( 155 queryUri.getQueryParameter(QUERY_PARAMETER_PROTOCOL))); 156 157 Cursor c = client.query(queryUri, DICTIONARY_PROJECTION, null, null, null); 158 if (isProtocolV2 && null == c) { 159 reinitializeClientRecordInDictionaryContentProvider(context, client, clientId); 160 c = client.query(queryUri, DICTIONARY_PROJECTION, null, null, null); 161 } 162 if (null == c) return Collections.<WordListInfo>emptyList(); 163 if (c.getCount() <= 0 || !c.moveToFirst()) { 164 c.close(); 165 return Collections.<WordListInfo>emptyList(); 166 } 167 final ArrayList<WordListInfo> list = CollectionUtils.newArrayList(); 168 do { 169 final String wordListId = c.getString(0); 170 final String wordListLocale = c.getString(1); 171 if (TextUtils.isEmpty(wordListId)) continue; 172 list.add(new WordListInfo(wordListId, wordListLocale)); 173 } while (c.moveToNext()); 174 c.close(); 175 return list; 176 } catch (RemoteException e) { 177 // The documentation is unclear as to in which cases this may happen, but it probably 178 // happens when the content provider got suddenly killed because it crashed or because 179 // the user disabled it through Settings. 180 Log.e(TAG, "RemoteException: communication with the dictionary pack cut", e); 181 return Collections.<WordListInfo>emptyList(); 182 } catch (Exception e) { 183 // A crash here is dangerous because crashing here would brick any encrypted device - 184 // we need the keyboard to be up and working to enter the password, so we don't want 185 // to die no matter what. So let's be as safe as possible. 186 Log.e(TAG, "Unexpected exception communicating with the dictionary pack", e); 187 return Collections.<WordListInfo>emptyList(); 188 } finally { 189 client.release(); 190 } 191 } 192 193 194 /** 195 * Helper method to encapsulate exception handling. 196 */ 197 private static AssetFileDescriptor openAssetFileDescriptor( 198 final ContentProviderClient providerClient, final Uri uri) { 199 try { 200 return providerClient.openAssetFile(uri, "r"); 201 } catch (FileNotFoundException e) { 202 // I don't want to log the word list URI here for security concerns. The exception 203 // contains the name of the file, so let's not pass it to Log.e here. 204 Log.e(TAG, "Could not find a word list from the dictionary provider." 205 /* intentionally don't pass the exception (see comment above) */); 206 return null; 207 } catch (RemoteException e) { 208 Log.e(TAG, "Can't communicate with the dictionary pack", e); 209 return null; 210 } 211 } 212 213 /** 214 * Caches a word list the id of which is passed as an argument. This will write the file 215 * to the cache file name designated by its id and locale, overwriting it if already present 216 * and creating it (and its containing directory) if necessary. 217 */ 218 private static void cacheWordList(final String wordlistId, final String locale, 219 final ContentProviderClient providerClient, final Context context) { 220 final int COMPRESSED_CRYPTED_COMPRESSED = 0; 221 final int CRYPTED_COMPRESSED = 1; 222 final int COMPRESSED_CRYPTED = 2; 223 final int COMPRESSED_ONLY = 3; 224 final int CRYPTED_ONLY = 4; 225 final int NONE = 5; 226 final int MODE_MIN = COMPRESSED_CRYPTED_COMPRESSED; 227 final int MODE_MAX = NONE; 228 229 final String clientId = context.getString(R.string.dictionary_pack_client_id); 230 final Uri.Builder wordListUriBuilder; 231 try { 232 wordListUriBuilder = getContentUriBuilderForType(clientId, 233 providerClient, QUERY_PATH_DATAFILE, wordlistId /* extraPath */); 234 } catch (RemoteException e) { 235 Log.e(TAG, "Can't communicate with the dictionary pack", e); 236 return; 237 } 238 final String finalFileName = 239 DictionaryInfoUtils.getCacheFileName(wordlistId, locale, context); 240 String tempFileName; 241 try { 242 tempFileName = BinaryDictionaryGetter.getTempFileName(wordlistId, context); 243 } catch (IOException e) { 244 Log.e(TAG, "Can't open the temporary file", e); 245 return; 246 } 247 248 for (int mode = MODE_MIN; mode <= MODE_MAX; ++mode) { 249 final InputStream originalSourceStream; 250 InputStream inputStream = null; 251 InputStream uncompressedStream = null; 252 InputStream decryptedStream = null; 253 BufferedInputStream bufferedInputStream = null; 254 File outputFile = null; 255 BufferedOutputStream bufferedOutputStream = null; 256 AssetFileDescriptor afd = null; 257 final Uri wordListUri = wordListUriBuilder.build(); 258 try { 259 // Open input. 260 afd = openAssetFileDescriptor(providerClient, wordListUri); 261 // If we can't open it at all, don't even try a number of times. 262 if (null == afd) return; 263 originalSourceStream = afd.createInputStream(); 264 // Open output. 265 outputFile = new File(tempFileName); 266 // Just to be sure, delete the file. This may fail silently, and return false: this 267 // is the right thing to do, as we just want to continue anyway. 268 outputFile.delete(); 269 // Get the appropriate decryption method for this try 270 switch (mode) { 271 case COMPRESSED_CRYPTED_COMPRESSED: 272 uncompressedStream = 273 FileTransforms.getUncompressedStream(originalSourceStream); 274 decryptedStream = FileTransforms.getDecryptedStream(uncompressedStream); 275 inputStream = FileTransforms.getUncompressedStream(decryptedStream); 276 break; 277 case CRYPTED_COMPRESSED: 278 decryptedStream = FileTransforms.getDecryptedStream(originalSourceStream); 279 inputStream = FileTransforms.getUncompressedStream(decryptedStream); 280 break; 281 case COMPRESSED_CRYPTED: 282 uncompressedStream = 283 FileTransforms.getUncompressedStream(originalSourceStream); 284 inputStream = FileTransforms.getDecryptedStream(uncompressedStream); 285 break; 286 case COMPRESSED_ONLY: 287 inputStream = FileTransforms.getUncompressedStream(originalSourceStream); 288 break; 289 case CRYPTED_ONLY: 290 inputStream = FileTransforms.getDecryptedStream(originalSourceStream); 291 break; 292 case NONE: 293 inputStream = originalSourceStream; 294 break; 295 } 296 bufferedInputStream = new BufferedInputStream(inputStream); 297 bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(outputFile)); 298 checkMagicAndCopyFileTo(bufferedInputStream, bufferedOutputStream); 299 bufferedOutputStream.flush(); 300 bufferedOutputStream.close(); 301 final File finalFile = new File(finalFileName); 302 finalFile.delete(); 303 if (!outputFile.renameTo(finalFile)) { 304 throw new IOException("Can't move the file to its final name"); 305 } 306 wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, 307 QUERY_PARAMETER_SUCCESS); 308 if (0 >= providerClient.delete(wordListUriBuilder.build(), null, null)) { 309 Log.e(TAG, "Could not have the dictionary pack delete a word list"); 310 } 311 BinaryDictionaryGetter.removeFilesWithIdExcept(context, wordlistId, finalFile); 312 Log.e(TAG, "Successfully copied file for wordlist ID " + wordlistId); 313 // Success! Close files (through the finally{} clause) and return. 314 return; 315 } catch (Exception e) { 316 if (DEBUG) { 317 Log.i(TAG, "Can't open word list in mode " + mode, e); 318 } 319 if (null != outputFile) { 320 // This may or may not fail. The file may not have been created if the 321 // exception was thrown before it could be. Hence, both failure and 322 // success are expected outcomes, so we don't check the return value. 323 outputFile.delete(); 324 } 325 // Try the next method. 326 } finally { 327 // Ignore exceptions while closing files. 328 closeAssetFileDescriptorAndReportAnyException(afd); 329 closeCloseableAndReportAnyException(inputStream); 330 closeCloseableAndReportAnyException(uncompressedStream); 331 closeCloseableAndReportAnyException(decryptedStream); 332 closeCloseableAndReportAnyException(bufferedInputStream); 333 closeCloseableAndReportAnyException(bufferedOutputStream); 334 } 335 } 336 337 // We could not copy the file at all. This is very unexpected. 338 // I'd rather not print the word list ID to the log out of security concerns 339 Log.e(TAG, "Could not copy a word list. Will not be able to use it."); 340 // If we can't copy it we should warn the dictionary provider so that it can mark it 341 // as invalid. 342 reportBrokenFileToDictionaryProvider(providerClient, clientId, wordlistId); 343 } 344 345 public static boolean reportBrokenFileToDictionaryProvider( 346 final ContentProviderClient providerClient, final String clientId, 347 final String wordlistId) { 348 try { 349 final Uri.Builder wordListUriBuilder = getContentUriBuilderForType(clientId, 350 providerClient, QUERY_PATH_DATAFILE, wordlistId /* extraPath */); 351 wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, 352 QUERY_PARAMETER_FAILURE); 353 if (0 >= providerClient.delete(wordListUriBuilder.build(), null, null)) { 354 Log.e(TAG, "Unable to delete a word list."); 355 } 356 } catch (RemoteException e) { 357 Log.e(TAG, "Communication with the dictionary provider was cut", e); 358 return false; 359 } 360 return true; 361 } 362 363 // Ideally the two following methods should be merged, but AssetFileDescriptor does not 364 // implement Closeable although it does implement #close(), and Java does not have 365 // structural typing. 366 private static void closeAssetFileDescriptorAndReportAnyException( 367 final AssetFileDescriptor file) { 368 try { 369 if (null != file) file.close(); 370 } catch (Exception e) { 371 Log.e(TAG, "Exception while closing a file", e); 372 } 373 } 374 375 private static void closeCloseableAndReportAnyException(final Closeable file) { 376 try { 377 if (null != file) file.close(); 378 } catch (Exception e) { 379 Log.e(TAG, "Exception while closing a file", e); 380 } 381 } 382 383 /** 384 * Queries a content provider for word list data for some locale and cache the returned files 385 * 386 * This will query a content provider for word list data for a given locale, and copy the 387 * files locally so that they can be mmap'ed. This may overwrite previously cached word lists 388 * with newer versions if a newer version is made available by the content provider. 389 * @throw FileNotFoundException if the provider returns non-existent data. 390 * @throw IOException if the provider-returned data could not be read. 391 */ 392 public static void cacheWordListsFromContentProvider(final Locale locale, 393 final Context context, final boolean hasDefaultWordList) { 394 final ContentProviderClient providerClient; 395 try { 396 providerClient = context.getContentResolver(). 397 acquireContentProviderClient(getProviderUriBuilder("").build()); 398 } catch (final SecurityException e) { 399 Log.e(TAG, "No permission to communicate with the dictionary provider", e); 400 return; 401 } 402 if (null == providerClient) { 403 Log.e(TAG, "Can't establish communication with the dictionary provider"); 404 return; 405 } 406 try { 407 final List<WordListInfo> idList = getWordListWordListInfos(locale, context, 408 hasDefaultWordList); 409 for (WordListInfo id : idList) { 410 cacheWordList(id.mId, id.mLocale, providerClient, context); 411 } 412 } finally { 413 providerClient.release(); 414 } 415 } 416 417 /** 418 * Copies the data in an input stream to a target file if the magic number matches. 419 * 420 * If the magic number does not match the expected value, this method throws an 421 * IOException. Other usual conditions for IOException or FileNotFoundException 422 * also apply. 423 * 424 * @param input the stream to be copied. 425 * @param output an output stream to copy the data to. 426 */ 427 public static void checkMagicAndCopyFileTo(final BufferedInputStream input, 428 final BufferedOutputStream output) throws FileNotFoundException, IOException { 429 // Check the magic number 430 final int length = MAGIC_NUMBER_VERSION_2.length; 431 final byte[] magicNumberBuffer = new byte[length]; 432 final int readMagicNumberSize = input.read(magicNumberBuffer, 0, length); 433 if (readMagicNumberSize < length) { 434 throw new IOException("Less bytes to read than the magic number length"); 435 } 436 if (!Arrays.equals(MAGIC_NUMBER_VERSION_2, magicNumberBuffer)) { 437 if (!Arrays.equals(MAGIC_NUMBER_VERSION_1, magicNumberBuffer)) { 438 throw new IOException("Wrong magic number for downloaded file"); 439 } 440 } 441 output.write(magicNumberBuffer); 442 443 // Actually copy the file 444 final byte[] buffer = new byte[FILE_READ_BUFFER_SIZE]; 445 for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) { 446 output.write(buffer, 0, readBytes); 447 } 448 input.close(); 449 } 450 451 private static void reinitializeClientRecordInDictionaryContentProvider(final Context context, 452 final ContentProviderClient client, final String clientId) throws RemoteException { 453 final String metadataFileUri = MetadataFileUriGetter.getMetadataUri(context); 454 final String metadataAdditionalId = MetadataFileUriGetter.getMetadataAdditionalId(context); 455 // Tell the content provider to reset all information about this client id 456 final Uri metadataContentUri = getProviderUriBuilder(clientId) 457 .appendPath(QUERY_PATH_METADATA) 458 .appendQueryParameter(QUERY_PARAMETER_PROTOCOL, QUERY_PARAMETER_PROTOCOL_VALUE) 459 .build(); 460 client.delete(metadataContentUri, null, null); 461 // Update the metadata URI 462 final ContentValues metadataValues = new ContentValues(); 463 metadataValues.put(INSERT_METADATA_CLIENT_ID_COLUMN, clientId); 464 metadataValues.put(INSERT_METADATA_METADATA_URI_COLUMN, metadataFileUri); 465 metadataValues.put(INSERT_METADATA_METADATA_ADDITIONAL_ID_COLUMN, metadataAdditionalId); 466 client.insert(metadataContentUri, metadataValues); 467 468 // Update the dictionary list. 469 final Uri dictionaryContentUriBase = getProviderUriBuilder(clientId) 470 .appendPath(QUERY_PATH_DICT_INFO) 471 .appendQueryParameter(QUERY_PARAMETER_PROTOCOL, QUERY_PARAMETER_PROTOCOL_VALUE) 472 .build(); 473 final ArrayList<DictionaryInfo> dictionaryList = 474 DictionaryInfoUtils.getCurrentDictionaryFileNameAndVersionInfo(context); 475 final int length = dictionaryList.size(); 476 for (int i = 0; i < length; ++i) { 477 final DictionaryInfo info = dictionaryList.get(i); 478 client.insert(Uri.withAppendedPath(dictionaryContentUriBase, info.mId), 479 info.toContentValues()); 480 } 481 } 482 483 /** 484 * Initialize a client record with the dictionary content provider. 485 * 486 * This merely acquires the content provider and calls 487 * #reinitializeClientRecordInDictionaryContentProvider. 488 * 489 * @param context the context for resources and providers. 490 * @param clientId the client ID to use. 491 */ 492 public static void initializeClientRecordHelper(final Context context, final String clientId) { 493 try { 494 final ContentProviderClient client = context.getContentResolver(). 495 acquireContentProviderClient(getProviderUriBuilder("").build()); 496 if (null == client) return; 497 reinitializeClientRecordInDictionaryContentProvider(context, client, clientId); 498 } catch (RemoteException e) { 499 Log.e(TAG, "Cannot contact the dictionary content provider", e); 500 } 501 } 502} 503