BinaryDictionary.java revision d4952c8fe9e71c2b313a68077e03d64c8b5196db
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16 17package com.android.inputmethod.latin; 18 19import android.content.Context; 20import android.util.Log; 21 22import java.io.IOException; 23import java.io.InputStream; 24import java.nio.ByteBuffer; 25import java.nio.ByteOrder; 26import java.nio.channels.Channels; 27import java.util.Arrays; 28 29/** 30 * Implements a static, compacted, binary dictionary of standard words. 31 */ 32public class BinaryDictionary extends Dictionary { 33 34 /** 35 * There is difference between what java and native code can handle. 36 * This value should only be used in BinaryDictionary.java 37 * It is necessary to keep it at this value because some languages e.g. German have 38 * really long words. 39 */ 40 protected static final int MAX_WORD_LENGTH = 48; 41 42 private static final String TAG = "BinaryDictionary"; 43 private static final int MAX_ALTERNATIVES = 16; 44 private static final int MAX_WORDS = 18; 45 private static final int MAX_BIGRAMS = 60; 46 47 private static final int TYPED_LETTER_MULTIPLIER = 2; 48 49 private int mDicTypeId; 50 private int mNativeDict; 51 private int mDictLength; 52 private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES]; 53 private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS]; 54 private char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS]; 55 private int[] mFrequencies = new int[MAX_WORDS]; 56 private int[] mFrequencies_bigrams = new int[MAX_BIGRAMS]; 57 // Keep a reference to the native dict direct buffer in Java to avoid 58 // unexpected deallocation of the direct buffer. 59 private ByteBuffer mNativeDictDirectBuffer; 60 61 static { 62 try { 63 System.loadLibrary("jni_latinime"); 64 } catch (UnsatisfiedLinkError ule) { 65 Log.e("BinaryDictionary", "Could not load native library jni_latinime"); 66 } 67 } 68 69 /** 70 * Create a dictionary from a raw resource file 71 * @param context application context for reading resources 72 * @param resId the resource containing the raw binary dictionary 73 */ 74 public BinaryDictionary(Context context, int[] resId, int dicTypeId) { 75 if (resId != null && resId.length > 0 && resId[0] != 0) { 76 loadDictionary(context, resId); 77 } 78 mDicTypeId = dicTypeId; 79 } 80 81 /** 82 * Create a dictionary from a byte buffer. This is used for testing. 83 * @param context application context for reading resources 84 * @param byteBuffer a ByteBuffer containing the binary dictionary 85 */ 86 public BinaryDictionary(Context context, ByteBuffer byteBuffer, int dicTypeId) { 87 if (byteBuffer != null) { 88 if (byteBuffer.isDirect()) { 89 mNativeDictDirectBuffer = byteBuffer; 90 } else { 91 mNativeDictDirectBuffer = ByteBuffer.allocateDirect(byteBuffer.capacity()); 92 byteBuffer.rewind(); 93 mNativeDictDirectBuffer.put(byteBuffer); 94 } 95 mDictLength = byteBuffer.capacity(); 96 mNativeDict = openNative(mNativeDictDirectBuffer, 97 TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER); 98 } 99 mDicTypeId = dicTypeId; 100 } 101 102 private native int openNative(ByteBuffer bb, int typedLetterMultiplier, 103 int fullWordMultiplier); 104 private native void closeNative(int dict); 105 private native boolean isValidWordNative(int nativeData, char[] word, int wordLength); 106 private native int getSuggestionsNative(int dict, int[] inputCodes, int codesSize, 107 char[] outputChars, int[] frequencies, int maxWordLength, int maxWords, 108 int maxAlternatives, int skipPos, int[] nextLettersFrequencies, int nextLettersSize); 109 private native int getBigramsNative(int dict, char[] prevWord, int prevWordLength, 110 int[] inputCodes, int inputCodesLength, char[] outputChars, int[] frequencies, 111 int maxWordLength, int maxBigrams, int maxAlternatives); 112 113 private final void loadDictionary(Context context, int[] resId) { 114 InputStream[] is = null; 115 try { 116 // merging separated dictionary into one if dictionary is separated 117 int total = 0; 118 is = new InputStream[resId.length]; 119 for (int i = 0; i < resId.length; i++) { 120 is[i] = context.getResources().openRawResource(resId[i]); 121 total += is[i].available(); 122 } 123 124 mNativeDictDirectBuffer = 125 ByteBuffer.allocateDirect(total).order(ByteOrder.nativeOrder()); 126 int got = 0; 127 for (int i = 0; i < resId.length; i++) { 128 got += Channels.newChannel(is[i]).read(mNativeDictDirectBuffer); 129 } 130 if (got != total) { 131 Log.e(TAG, "Read " + got + " bytes, expected " + total); 132 } else { 133 mNativeDict = openNative(mNativeDictDirectBuffer, 134 TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER); 135 mDictLength = total; 136 } 137 } catch (IOException e) { 138 Log.w(TAG, "No available memory for binary dictionary"); 139 } finally { 140 try { 141 if (is != null) { 142 for (int i = 0; i < is.length; i++) { 143 is[i].close(); 144 } 145 } 146 } catch (IOException e) { 147 Log.w(TAG, "Failed to close input stream"); 148 } 149 } 150 } 151 152 153 @Override 154 public void getBigrams(final WordComposer codes, final CharSequence previousWord, 155 final WordCallback callback, int[] nextLettersFrequencies) { 156 157 char[] chars = previousWord.toString().toCharArray(); 158 Arrays.fill(mOutputChars_bigrams, (char) 0); 159 Arrays.fill(mFrequencies_bigrams, 0); 160 161 int codesSize = codes.size(); 162 Arrays.fill(mInputCodes, -1); 163 int[] alternatives = codes.getCodesAt(0); 164 System.arraycopy(alternatives, 0, mInputCodes, 0, 165 Math.min(alternatives.length, MAX_ALTERNATIVES)); 166 167 int count = getBigramsNative(mNativeDict, chars, chars.length, mInputCodes, codesSize, 168 mOutputChars_bigrams, mFrequencies_bigrams, MAX_WORD_LENGTH, MAX_BIGRAMS, 169 MAX_ALTERNATIVES); 170 171 for (int j = 0; j < count; j++) { 172 if (mFrequencies_bigrams[j] < 1) break; 173 int start = j * MAX_WORD_LENGTH; 174 int len = 0; 175 while (mOutputChars_bigrams[start + len] != 0) { 176 len++; 177 } 178 if (len > 0) { 179 callback.addWord(mOutputChars_bigrams, start, len, mFrequencies_bigrams[j], 180 mDicTypeId, DataType.BIGRAM); 181 } 182 } 183 } 184 185 @Override 186 public void getWords(final WordComposer codes, final WordCallback callback, 187 int[] nextLettersFrequencies) { 188 final int codesSize = codes.size(); 189 // Won't deal with really long words. 190 if (codesSize > MAX_WORD_LENGTH - 1) return; 191 192 Arrays.fill(mInputCodes, -1); 193 for (int i = 0; i < codesSize; i++) { 194 int[] alternatives = codes.getCodesAt(i); 195 System.arraycopy(alternatives, 0, mInputCodes, i * MAX_ALTERNATIVES, 196 Math.min(alternatives.length, MAX_ALTERNATIVES)); 197 } 198 Arrays.fill(mOutputChars, (char) 0); 199 Arrays.fill(mFrequencies, 0); 200 201 int count = getSuggestionsNative(mNativeDict, mInputCodes, codesSize, mOutputChars, 202 mFrequencies, MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, -1, 203 nextLettersFrequencies, 204 nextLettersFrequencies != null ? nextLettersFrequencies.length : 0); 205 206 for (int j = 0; j < count; j++) { 207 if (mFrequencies[j] < 1) break; 208 int start = j * MAX_WORD_LENGTH; 209 int len = 0; 210 while (mOutputChars[start + len] != 0) { 211 len++; 212 } 213 if (len > 0) { 214 callback.addWord(mOutputChars, start, len, mFrequencies[j], mDicTypeId, 215 DataType.UNIGRAM); 216 } 217 } 218 } 219 220 @Override 221 public boolean isValidWord(CharSequence word) { 222 if (word == null) return false; 223 char[] chars = word.toString().toCharArray(); 224 return isValidWordNative(mNativeDict, chars, chars.length); 225 } 226 227 public int getSize() { 228 return mDictLength; // This value is initialized on the call to openNative() 229 } 230 231 @Override 232 public synchronized void close() { 233 if (mNativeDict != 0) { 234 closeNative(mNativeDict); 235 mNativeDict = 0; 236 } 237 } 238 239 @Override 240 protected void finalize() throws Throwable { 241 close(); 242 super.finalize(); 243 } 244} 245