BinaryDictionary.java revision 979f8690967ff5409fe18f5085858ccdb8e0ccf1
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import java.io.ByteArrayInputStream; 20import java.io.InputStream; 21import java.io.IOException; 22import java.nio.ByteBuffer; 23import java.nio.ByteOrder; 24import java.nio.channels.Channels; 25import java.util.Arrays; 26 27import android.content.Context; 28import android.util.Log; 29 30/** 31 * Implements a static, compacted, binary dictionary of standard words. 32 */ 33public class BinaryDictionary extends Dictionary { 34 35 /** 36 * There is difference between what java and native code can handle. 37 * This value should only be used in BinaryDictionary.java 38 * It is necessary to keep it at this value because some languages e.g. German have 39 * really long words. 40 */ 41 protected static final int MAX_WORD_LENGTH = 48; 42 43 private static final String TAG = "BinaryDictionary"; 44 private static final int MAX_ALTERNATIVES = 16; 45 private static final int MAX_WORDS = 18; 46 private static final int MAX_BIGRAMS = 60; 47 48 private static final int TYPED_LETTER_MULTIPLIER = 2; 49 private static final boolean ENABLE_MISSED_CHARACTERS = true; 50 51 private int mDicTypeId; 52 private int mNativeDict; 53 private int mDictLength; 54 private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES]; 55 private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS]; 56 private char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS]; 57 private int[] mFrequencies = new int[MAX_WORDS]; 58 private int[] mFrequencies_bigrams = new int[MAX_BIGRAMS]; 59 // Keep a reference to the native dict direct buffer in Java to avoid 60 // unexpected deallocation of the direct buffer. 61 private ByteBuffer mNativeDictDirectBuffer; 62 63 static { 64 try { 65 System.loadLibrary("jni_latinime"); 66 } catch (UnsatisfiedLinkError ule) { 67 Log.e("BinaryDictionary", "Could not load native library jni_latinime"); 68 } 69 } 70 71 /** 72 * Create a dictionary from a raw resource file 73 * @param context application context for reading resources 74 * @param resId the resource containing the raw binary dictionary 75 */ 76 public BinaryDictionary(Context context, int[] resId, int dicTypeId) { 77 if (resId != null && resId.length > 0 && resId[0] != 0) { 78 loadDictionary(context, resId); 79 } 80 mDicTypeId = dicTypeId; 81 } 82 83 /** 84 * Create a dictionary from a byte buffer. This is used for testing. 85 * @param context application context for reading resources 86 * @param byteBuffer a ByteBuffer containing the binary dictionary 87 */ 88 public BinaryDictionary(Context context, ByteBuffer byteBuffer, int dicTypeId) { 89 if (byteBuffer != null) { 90 if (byteBuffer.isDirect()) { 91 mNativeDictDirectBuffer = byteBuffer; 92 } else { 93 mNativeDictDirectBuffer = ByteBuffer.allocateDirect(byteBuffer.capacity()); 94 byteBuffer.rewind(); 95 mNativeDictDirectBuffer.put(byteBuffer); 96 } 97 mDictLength = byteBuffer.capacity(); 98 mNativeDict = openNative(mNativeDictDirectBuffer, 99 TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER); 100 } 101 mDicTypeId = dicTypeId; 102 } 103 104 private native int openNative(ByteBuffer bb, int typedLetterMultiplier, 105 int fullWordMultiplier); 106 private native void closeNative(int dict); 107 private native boolean isValidWordNative(int nativeData, char[] word, int wordLength); 108 private native int getSuggestionsNative(int dict, int[] inputCodes, int codesSize, 109 char[] outputChars, int[] frequencies, int maxWordLength, int maxWords, 110 int maxAlternatives, int skipPos, int[] nextLettersFrequencies, int nextLettersSize); 111 private native int getBigramsNative(int dict, char[] prevWord, int prevWordLength, 112 int[] inputCodes, int inputCodesLength, char[] outputChars, int[] frequencies, 113 int maxWordLength, int maxBigrams, int maxAlternatives); 114 115 private final void loadDictionary(Context context, int[] resId) { 116 InputStream[] is = null; 117 try { 118 // merging separated dictionary into one if dictionary is separated 119 int total = 0; 120 is = new InputStream[resId.length]; 121 for (int i = 0; i < resId.length; i++) { 122 is[i] = context.getResources().openRawResource(resId[i]); 123 total += is[i].available(); 124 } 125 126 mNativeDictDirectBuffer = 127 ByteBuffer.allocateDirect(total).order(ByteOrder.nativeOrder()); 128 int got = 0; 129 for (int i = 0; i < resId.length; i++) { 130 got += Channels.newChannel(is[i]).read(mNativeDictDirectBuffer); 131 } 132 if (got != total) { 133 Log.e(TAG, "Read " + got + " bytes, expected " + total); 134 } else { 135 mNativeDict = openNative(mNativeDictDirectBuffer, 136 TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER); 137 mDictLength = total; 138 } 139 } catch (IOException e) { 140 Log.w(TAG, "No available memory for binary dictionary"); 141 } finally { 142 try { 143 if (is != null) { 144 for (int i = 0; i < is.length; i++) { 145 is[i].close(); 146 } 147 } 148 } catch (IOException e) { 149 Log.w(TAG, "Failed to close input stream"); 150 } 151 } 152 } 153 154 155 @Override 156 public void getBigrams(final WordComposer codes, final CharSequence previousWord, 157 final WordCallback callback, int[] nextLettersFrequencies) { 158 159 char[] chars = previousWord.toString().toCharArray(); 160 Arrays.fill(mOutputChars_bigrams, (char) 0); 161 Arrays.fill(mFrequencies_bigrams, 0); 162 163 int codesSize = codes.size(); 164 Arrays.fill(mInputCodes, -1); 165 int[] alternatives = codes.getCodesAt(0); 166 System.arraycopy(alternatives, 0, mInputCodes, 0, 167 Math.min(alternatives.length, MAX_ALTERNATIVES)); 168 169 int count = getBigramsNative(mNativeDict, chars, chars.length, mInputCodes, codesSize, 170 mOutputChars_bigrams, mFrequencies_bigrams, MAX_WORD_LENGTH, MAX_BIGRAMS, 171 MAX_ALTERNATIVES); 172 173 for (int j = 0; j < count; j++) { 174 if (mFrequencies_bigrams[j] < 1) break; 175 int start = j * MAX_WORD_LENGTH; 176 int len = 0; 177 while (mOutputChars_bigrams[start + len] != 0) { 178 len++; 179 } 180 if (len > 0) { 181 callback.addWord(mOutputChars_bigrams, start, len, mFrequencies_bigrams[j], 182 mDicTypeId, DataType.BIGRAM); 183 } 184 } 185 } 186 187 @Override 188 public void getWords(final WordComposer codes, final WordCallback callback, 189 int[] nextLettersFrequencies) { 190 final int codesSize = codes.size(); 191 // Won't deal with really long words. 192 if (codesSize > MAX_WORD_LENGTH - 1) return; 193 194 Arrays.fill(mInputCodes, -1); 195 for (int i = 0; i < codesSize; i++) { 196 int[] alternatives = codes.getCodesAt(i); 197 System.arraycopy(alternatives, 0, mInputCodes, i * MAX_ALTERNATIVES, 198 Math.min(alternatives.length, MAX_ALTERNATIVES)); 199 } 200 Arrays.fill(mOutputChars, (char) 0); 201 Arrays.fill(mFrequencies, 0); 202 203 int count = getSuggestionsNative(mNativeDict, mInputCodes, codesSize, 204 mOutputChars, mFrequencies, 205 MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, -1, 206 nextLettersFrequencies, 207 nextLettersFrequencies != null ? nextLettersFrequencies.length : 0); 208 209 // If there aren't sufficient suggestions, search for words by allowing wild cards at 210 // the different character positions. This feature is not ready for prime-time as we need 211 // to figure out the best ranking for such words compared to proximity corrections and 212 // completions. 213 if (ENABLE_MISSED_CHARACTERS && count < 5) { 214 for (int skip = 0; skip < codesSize; skip++) { 215 int tempCount = getSuggestionsNative(mNativeDict, mInputCodes, codesSize, 216 mOutputChars, mFrequencies, 217 MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, skip, 218 null, 0); 219 count = Math.max(count, tempCount); 220 if (tempCount > 0) break; 221 } 222 } 223 224 for (int j = 0; j < count; j++) { 225 if (mFrequencies[j] < 1) break; 226 int start = j * MAX_WORD_LENGTH; 227 int len = 0; 228 while (mOutputChars[start + len] != 0) { 229 len++; 230 } 231 if (len > 0) { 232 callback.addWord(mOutputChars, start, len, mFrequencies[j], mDicTypeId, 233 DataType.UNIGRAM); 234 } 235 } 236 } 237 238 @Override 239 public boolean isValidWord(CharSequence word) { 240 if (word == null) return false; 241 char[] chars = word.toString().toCharArray(); 242 return isValidWordNative(mNativeDict, chars, chars.length); 243 } 244 245 public int getSize() { 246 return mDictLength; // This value is initialized on the call to openNative() 247 } 248 249 @Override 250 public synchronized void close() { 251 if (mNativeDict != 0) { 252 closeNative(mNativeDict); 253 mNativeDict = 0; 254 } 255 } 256 257 @Override 258 protected void finalize() throws Throwable { 259 close(); 260 super.finalize(); 261 } 262} 263