1923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project/* 2443c360d0afdbab091994244f045f4756feaf2b4Jean-Baptiste Queru * Copyright (C) 2008 The Android Open Source Project 3e90b333017c68e888a5e3d351f07ea29036457d0Ken Wakasa * 48aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * Licensed under the Apache License, Version 2.0 (the "License"); 58aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * you may not use this file except in compliance with the License. 68aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * You may obtain a copy of the License at 7e90b333017c68e888a5e3d351f07ea29036457d0Ken Wakasa * 88aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * http://www.apache.org/licenses/LICENSE-2.0 9e90b333017c68e888a5e3d351f07ea29036457d0Ken Wakasa * 10923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project * Unless required by applicable law or agreed to in writing, software 118aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * distributed under the License is distributed on an "AS IS" BASIS, 128aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 138aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * See the License for the specific language governing permissions and 148aa9963a895f9dd5bb1bc92ab2e4f461e058f87aTadashi G. Takaoka * limitations under the License. 15923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project */ 16923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 17923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Projectpackage com.android.inputmethod.latin; 18923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 194d289d39aeae21064f63d958974816ceee3e9fdeTom Ouyangimport android.text.TextUtils; 20d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagiimport android.util.Log; 213979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataokaimport android.util.SparseArray; 22fa086c90760bc2bedf0b74eacb0fed3bf7ebc2b7Tadashi G. Takaoka 23d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagiimport com.android.inputmethod.annotations.UsedForTesting; 24bda7eaa63aace64f3d40eae3affaf281591ffa66Jean Chalardimport com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 25ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalardimport com.android.inputmethod.latin.common.ComposedData; 269342484e8d573a40f470b6a593df31c602fa4076Ken Wakasaimport com.android.inputmethod.latin.common.Constants; 275b91b551e5ffaf2c2e691dfbd434f21c82293986Jean Chalardimport com.android.inputmethod.latin.common.FileUtils; 2836799b2aa2982ec17341cd2c5ed81e608bcee8c6Jean Chalardimport com.android.inputmethod.latin.common.InputPointers; 294beeb9253a06482299e0c67467531d30436a02fcJean Chalardimport com.android.inputmethod.latin.common.StringUtils; 30927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagiimport com.android.inputmethod.latin.makedict.DictionaryHeader; 31927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagiimport com.android.inputmethod.latin.makedict.FormatSpec; 323ad4af2354e7003ac288dafe3600268fe860d752Keisuke Kuroyanagiimport com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; 33927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagiimport com.android.inputmethod.latin.makedict.UnsupportedFormatException; 345f5feeba13f6f1a907d90365d8037a361d0ff5daKeisuke Kuroyanagiimport com.android.inputmethod.latin.makedict.WordProperty; 35b8a9479b57007edb5cb12c628797f89a8164f596Keisuke Kuroyanagiimport com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 36a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagiimport com.android.inputmethod.latin.utils.BinaryDictionaryUtils; 37e28eba5074664d5716b8e58b8d0a235746b261ebKen Wakasaimport com.android.inputmethod.latin.utils.JniUtils; 3816cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagiimport com.android.inputmethod.latin.utils.WordInputEventForPersonalization; 39ab72a97d7ce44230a0c824797d1675a5ca354a56Tadashi G. Takaoka 402e58670da9687fd1fd28c322e03343957d11568cKeisuke Kuroyanagiimport java.io.File; 41bda7eaa63aace64f3d40eae3affaf281591ffa66Jean Chalardimport java.util.ArrayList; 42923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Projectimport java.util.Arrays; 43927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagiimport java.util.HashMap; 4443ebd8a035af31244a2d54fce5d8000a1fbada4csatokimport java.util.Locale; 453b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagiimport java.util.Map; 46923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 479290d0a4eba454b9b1501830a4e470005cc85332Keisuke Kuroyanagiimport javax.annotation.Nonnull; 489290d0a4eba454b9b1501830a4e470005cc85332Keisuke Kuroyanagi 49923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project/** 50923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project * Implements a static, compacted, binary dictionary of standard words. 51923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project */ 526142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi// TODO: All methods which should be locked need to have a suffix "Locked". 53a28a05e971cc242b338331a3b78276fa95188d19Tadashi G. Takaokapublic final class BinaryDictionary extends Dictionary { 54b02ee3d67a1884b6ff59cc16c29a476845c0694fKen Wakasa private static final String TAG = BinaryDictionary.class.getSimpleName(); 55cba93f50c3d46ada773ec49435689dc3e2094385Jean Chalard 56459cd6f8ef3eaa561e47dd996ce537770ea8b37aJean Chalard // The cutoff returned by native for auto-commit confidence. 57459cd6f8ef3eaa561e47dd996ce537770ea8b37aJean Chalard // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h 58459cd6f8ef3eaa561e47dd996ce537770ea8b37aJean Chalard private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; 59923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 60f4686092232588781910cc4e64406c4958577e86Mohammadinamul Sheik public static final int DICTIONARY_MAX_WORD_LENGTH = 48; 61f4686092232588781910cc4e64406c4958577e86Mohammadinamul Sheik public static final int MAX_PREV_WORD_COUNT_FOR_N_GRAM = 3; 6229500ef4ba8e01f4c467a62399c8249d532ee82cMohammadinamul Sheik 6331097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi @UsedForTesting 6431097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; 6531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi @UsedForTesting 6631097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; 67cfb018ba6db78f2b33b54d4811f0bf166db29792Keisuke Kuroyanagi @UsedForTesting 68cfb018ba6db78f2b33b54d4811f0bf166db29792Keisuke Kuroyanagi public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT"; 69cfb018ba6db78f2b33b54d4811f0bf166db29792Keisuke Kuroyanagi @UsedForTesting 70cfb018ba6db78f2b33b54d4811f0bf166db29792Keisuke Kuroyanagi public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; 7131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi 722fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa public static final int NOT_A_VALID_TIMESTAMP = -1; 732fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 74080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi // Format to get unigram flags from native side via getWordPropertyNative(). 7588fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi private static final int FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT = 5; 76080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; 7705172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu private static final int FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX = 1; 78d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi private static final int FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX = 2; 7912d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; // DEPRECATED 8088fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; 812fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 8294d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi // Format to get probability and historical info from native side via getWordPropertyNative(). 8394d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4; 8494d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0; 8594d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1; 8694d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2; 8794d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3; 882fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 89a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi public static final String DICT_FILE_NAME_SUFFIX_FOR_MIGRATION = ".migrate"; 901d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi public static final String DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION = ".migrating"; 91a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi 925fdcd7d5cd9d39d41568aa1412a4b1a866c05d3aTadashi G. Takaoka private long mNativeDict; 934c2767857a02c9cf18a9579aa0391fd09b3fe411Keisuke Kuroyanagi private final long mDictSize; 94d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi private final String mDictFilePath; 95162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi private final boolean mUseFullEditDistance; 96d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi private final boolean mIsUpdatable; 97fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi private boolean mHasUpdated; 98fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi 99a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); 1008ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka 1018ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka // TODO: There should be a way to remove used DicTraverseSession objects from 1028ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka // {@code mDicTraverseSessions}. 103bc464e2952e102219f0b977fc1e9140ad5bd03e4Tadashi G. Takaoka private DicTraverseSession getTraverseSession(final int traverseSessionId) { 1048ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka synchronized(mDicTraverseSessions) { 1058ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); 1068ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka if (traverseSession == null) { 10709c72207c634ec5e1350aff23f7abe7bb47e5671Keisuke Kuroyanagi traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); 10809c72207c634ec5e1350aff23f7abe7bb47e5671Keisuke Kuroyanagi mDicTraverseSessions.put(traverseSessionId, traverseSession); 1093979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataoka } 1108ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka return traverseSession; 1113979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataoka } 1123979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataoka } 113923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 114923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project /** 1153b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * Constructs binary dictionary using existing dictionary file. 1164250eb27f54f8fedc388fe4825b0646a88778744Jean Chalard * @param filename the name of the file to read through native code. 1174250eb27f54f8fedc388fe4825b0646a88778744Jean Chalard * @param offset the offset of the dictionary data within the file. 1184250eb27f54f8fedc388fe4825b0646a88778744Jean Chalard * @param length the length of the binary data. 11924aee9100e92dc4c06cdb54487a4922420fa8660Jean Chalard * @param useFullEditDistance whether to use the full edit distance in suggestions 12005efe576f976f5fa280f8d523f2935c15cbb9bd1Jean Chalard * @param dictType the dictionary type, as a human-readable string 121981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi * @param isUpdatable whether to open the dictionary file in writable mode. 122923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project */ 123f6870cc82ddf394e94155322fcc7e4e2256bea66Ken Wakasa public BinaryDictionary(final String filename, final long offset, final long length, 124981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi final boolean useFullEditDistance, final Locale locale, final String dictType, 125981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi final boolean isUpdatable) { 126576c96af95d7f1df869224ada78933d968e9a9c3Jean Chalard super(dictType, locale); 1274c2767857a02c9cf18a9579aa0391fd09b3fe411Keisuke Kuroyanagi mDictSize = length; 128d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi mDictFilePath = filename; 129d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi mIsUpdatable = isUpdatable; 130fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi mHasUpdated = false; 131162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi mUseFullEditDistance = useFullEditDistance; 132981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi loadDictionary(filename, offset, length, isUpdatable); 133979f8690967ff5409fe18f5085858ccdb8e0ccf1satok } 134979f8690967ff5409fe18f5085858ccdb8e0ccf1satok 1353b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi /** 1363b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * Constructs binary dictionary on memory. 1373b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * @param filename the name of the file used to flush. 1383b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * @param useFullEditDistance whether to use the full edit distance in suggestions 1393b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * @param dictType the dictionary type, as a human-readable string 1403b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * @param formatVersion the format version of the dictionary 1413b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi * @param attributeMap the attributes of the dictionary 1423b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi */ 1433b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi public BinaryDictionary(final String filename, final boolean useFullEditDistance, 1443b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi final Locale locale, final String dictType, final long formatVersion, 1453b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi final Map<String, String> attributeMap) { 146576c96af95d7f1df869224ada78933d968e9a9c3Jean Chalard super(dictType, locale); 1473b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi mDictSize = 0; 1483b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi mDictFilePath = filename; 1493b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi // On memory dictionary is always updatable. 1503b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi mIsUpdatable = true; 1513b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi mHasUpdated = false; 152162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi mUseFullEditDistance = useFullEditDistance; 1533b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi final String[] keyArray = new String[attributeMap.size()]; 1543b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi final String[] valueArray = new String[attributeMap.size()]; 1553b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi int index = 0; 1563b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi for (final String key : attributeMap.keySet()) { 1573b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi keyArray[index] = key; 1583b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi valueArray[index] = attributeMap.get(key); 1593b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi index++; 1603b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi } 1613b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi mNativeDict = createOnMemoryNative(formatVersion, locale.toString(), keyArray, valueArray); 1623b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi } 1633b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi 1643b7984752c88bff157016a09158dc92d94ed401dKeisuke Kuroyanagi 165eaef1c500703b4ee378821884c7b108815ed2983Ken Wakasa static { 166cc8c8b99bd0463f5977dea82f5e2379ea1dd4e73Tadashi G. Takaoka JniUtils.loadNativeLibrary(); 167eaef1c500703b4ee378821884c7b108815ed2983Ken Wakasa } 168c2bbc6a4499a6da979381fa0e8e6e855a5ac6aa4Jean Chalard 169f3b62900c7bcb0d6434f45ec7b467b7b4bad6f9aKeisuke Kuroynagi private static native long openNative(String sourceDir, long dictOffset, long dictSize, 170f3b62900c7bcb0d6434f45ec7b467b7b4bad6f9aKeisuke Kuroynagi boolean isUpdatable); 17161cb852d336543f120a9fa51fc40dda6b639da39Keisuke Kuroyanagi private static native long createOnMemoryNative(long formatVersion, 17261cb852d336543f120a9fa51fc40dda6b639da39Keisuke Kuroyanagi String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray); 173927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, 174927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, 175927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi ArrayList<int[]> outAttributeValues); 176dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi private static native boolean flushNative(long dict, String filePath); 177c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); 178dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi private static native boolean flushWithGCNative(long dict, String filePath); 1795db594abbad2d9e8d2cf1aa6e417aa50ffc5dfc1Ken Wakasa private static native void closeNative(long dict); 1802fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa private static native int getFormatVersionNative(long dict); 181e0e67373735918c78eaeaf24f127e1d28816aa29Satoshi Kataoka private static native int getProbabilityNative(long dict, int[] word); 182d6b89e17a678d0f7cb8a4a2c1dafa72a4568b744Keisuke Kuroyanagi private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); 18305b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi private static native int getNgramProbabilityNative(long dict, int[][] prevWordCodePointArrays, 18405b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi boolean[] isBeginningOfSentenceArray, int[] word); 185080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi private static native void getWordPropertyNative(long dict, int[] word, 18688fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, 187d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi int[] outProbabilityInfo, ArrayList<int[][]> outNgramPrevWordsArray, 188d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray, 189d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi ArrayList<int[]> outNgramTargets, ArrayList<int[]> outNgramProbabilityInfo, 190d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi ArrayList<int[]> outShortcutTargets, ArrayList<Integer> outShortcutProbabilities); 19188fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi private static native int getNextWordNative(long dict, int token, int[] outCodePoints, 19288fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi boolean[] outIsBeginningOfSentence); 193e137ec0a91cf93b0a99fd1e1556ee835d026f731Keisuke Kuroyanagi private static native void getSuggestionsNative(long dict, long proximityInfo, 1945db594abbad2d9e8d2cf1aa6e417aa50ffc5dfc1Ken Wakasa long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, 1955a0381f7d00abf97f5792958420dbb8535977ab3Keisuke Kuroyanagi int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, 19605b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 197880624838611a69f20f39ae762181ea4639dd071Keisuke Kuroyanagi int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints, 198880624838611a69f20f39ae762181ea4639dd071Keisuke Kuroyanagi int[] outputScores, int[] outputIndices, int[] outputTypes, 1996da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard int[] outputAutoCommitFirstWordConfidence, 2006da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard float[] inOutWeightOfLangModelVsSpatialModel); 20183b6ee25385e1c4fd76ca5842ff025acf945596bKeisuke Kuroyanagi private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, 2021adca93381d261a6070be2721dbf8b8abafbfe01Keisuke Kuroyanagi int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, 20305172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu boolean isNotAWord, boolean isPossiblyOffensive, int timestamp); 20483b6ee25385e1c4fd76ca5842ff025acf945596bKeisuke Kuroyanagi private static native boolean removeUnigramEntryNative(long dict, int[] word); 20505b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi private static native boolean addNgramEntryNative(long dict, 20605b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 20705b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi int[] word, int probability, int timestamp); 20805b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi private static native boolean removeNgramEntryNative(long dict, 20905b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); 210ab4437f4681db3fb8327f752df8f08bd0d8cf967Keisuke Kuroyanagi private static native boolean updateEntriesForWordWithNgramContextNative(long dict, 211ce5fd94b9714f0b8bf3c28eef7176a30b9334bcbKeisuke Kuroyanagi int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, 212ce5fd94b9714f0b8bf3c28eef7176a30b9334bcbKeisuke Kuroyanagi int[] word, boolean isValidWord, int count, int timestamp); 21316cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi private static native int updateEntriesForInputEventsNative(long dict, 21416cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi WordInputEventForPersonalization[] inputEvents, int startIndex); 21531097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi private static native String getPropertyNative(long dict, String query); 216d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi private static native boolean isCorruptedNative(long dict); 2171471fbad66f68a06494ac3efea5d9d16ea46b322Keisuke Kuroyanagi private static native boolean migrateNative(long dict, String dictFilePath, 2181471fbad66f68a06494ac3efea5d9d16ea46b322Keisuke Kuroyanagi long newFormatVersion); 219979f8690967ff5409fe18f5085858ccdb8e0ccf1satok 220e9f3e182e4b3217282831fd8805958270b4dbba3Satoshi Kataoka // TODO: Move native dict into session 22177fd5dbf3e432356dd16eb428c740e446c04373eDan Zivkovic private void loadDictionary(final String path, final long startOffset, 222981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi final long length, final boolean isUpdatable) { 223fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi mHasUpdated = false; 224981717da4c414caee57ba98596f9bc634a97f74fKeisuke Kuroynagi mNativeDict = openNative(path, startOffset, length, isUpdatable); 225979f8690967ff5409fe18f5085858ccdb8e0ccf1satok } 226979f8690967ff5409fe18f5085858ccdb8e0ccf1satok 227d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi // TODO: Check isCorrupted() for main dictionaries. 228d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi public boolean isCorrupted() { 229d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi if (!isValidDictionary()) { 230d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi return false; 231d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi } 232d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi if (!isCorruptedNative(mNativeDict)) { 233d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi return false; 234d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi } 235d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi // TODO: Record the corruption. 236d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted."); 237d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi Log.e(TAG, "locale: " + mLocale); 238d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi Log.e(TAG, "dict size: " + mDictSize); 239d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi Log.e(TAG, "updatable: " + mIsUpdatable); 240d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi return true; 241d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi } 242d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi 243927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi public DictionaryHeader getHeader() throws UnsupportedFormatException { 244927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi if (mNativeDict == 0) { 245927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi return null; 246927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi } 247927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi final int[] outHeaderSize = new int[1]; 248927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi final int[] outFormatVersion = new int[1]; 249a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); 250a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final ArrayList<int[]> outAttributeValues = new ArrayList<>(); 251927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, 252927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi outAttributeValues); 253a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final HashMap<String, String> attributes = new HashMap<>(); 254927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi for (int i = 0; i < outAttributeKeys.size(); i++) { 255927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( 256927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi outAttributeKeys.get(i)); 257927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi final String attributeValue = StringUtils.getStringFromNullTerminatedCodePointArray( 258927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi outAttributeValues.get(i)); 259927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi attributes.put(attributeKey, attributeValue); 260927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi } 261fc9ca59cc1dccc8353528c42a7eb710ad6d90e14Keisuke Kuroyanagi final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( 262fc9ca59cc1dccc8353528c42a7eb710ad6d90e14Keisuke Kuroyanagi attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); 263927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), 264927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); 265927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi } 266927aff72bc9f0e59450ec6278fbcc05ba6442f25Keisuke Kuroyanagi 267979f8690967ff5409fe18f5085858ccdb8e0ccf1satok @Override 268ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard public ArrayList<SuggestedWordInfo> getSuggestions(final ComposedData composedData, 269ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard final NgramContext ngramContext, final long proximityInfoHandle, 270b8a9479b57007edb5cb12c628797f89a8164f596Keisuke Kuroyanagi final SettingsValuesForSuggestion settingsValuesForSuggestion, 2716da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard final int sessionId, final float weightForLocale, 2726da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard final float[] inOutWeightOfLangModelVsSpatialModel) { 2735f430e0189f6c0a1a34edb0e00809bc53a525a7bJean Chalard if (!isValidDictionary()) { 2745f430e0189f6c0a1a34edb0e00809bc53a525a7bJean Chalard return null; 2755f430e0189f6c0a1a34edb0e00809bc53a525a7bJean Chalard } 276162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi final DicTraverseSession session = getTraverseSession(sessionId); 277162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); 278bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi ngramContext.outputToArray(session.mPrevWordCodePointArrays, 279e708b1bc2e11285ad404133b8de21719ce08acb5Keisuke Kuroyanagi session.mIsBeginningOfSentenceArray); 280ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard final InputPointers inputPointers = composedData.mInputPointers; 281ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard final boolean isGesture = composedData.mIsBatchMode; 28245e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard final int inputSize; 28345e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard if (!isGesture) { 284ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard inputSize = 285ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard composedData.copyCodePointsExceptTrailingSingleQuotesAndReturnCodePointCount( 286ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard session.mInputCodePoints); 2875f430e0189f6c0a1a34edb0e00809bc53a525a7bJean Chalard if (inputSize < 0) { 2885f430e0189f6c0a1a34edb0e00809bc53a525a7bJean Chalard return null; 289860a9f85ff7f2753b7e1bed2e00f86de8eca68e1Jean Chalard } 29045e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard } else { 29145e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard inputSize = inputPointers.getPointerSize(); 292860a9f85ff7f2753b7e1bed2e00f86de8eca68e1Jean Chalard } 293162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi session.mNativeSuggestOptions.setUseFullEditDistance(mUseFullEditDistance); 294162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi session.mNativeSuggestOptions.setIsGesture(isGesture); 295b8a9479b57007edb5cb12c628797f89a8164f596Keisuke Kuroyanagi session.mNativeSuggestOptions.setBlockOffensiveWords( 296b8a9479b57007edb5cb12c628797f89a8164f596Keisuke Kuroyanagi settingsValuesForSuggestion.mBlockPotentiallyOffensive); 2977d5e1cb2650d74816767c085ad71d04d6e605559Jean Chalard session.mNativeSuggestOptions.setWeightForLocale(weightForLocale); 2986da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard if (inOutWeightOfLangModelVsSpatialModel != null) { 2996da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 3006da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard inOutWeightOfLangModelVsSpatialModel[0]; 3011de95ceada64e7fd27ca4ee43243930b5d9c1df7Keisuke Kuroyanagi } else { 3026da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard session.mInputOutputWeightOfLangModelVsSpatialModel[0] = 3036da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL; 3041de95ceada64e7fd27ca4ee43243930b5d9c1df7Keisuke Kuroyanagi } 305e708b1bc2e11285ad404133b8de21719ce08acb5Keisuke Kuroyanagi // TOOD: Pass multiple previous words information for n-gram. 306ecab6aff5908bfd5b34670d2e2bb3696627fa47cJean Chalard getSuggestionsNative(mNativeDict, proximityInfoHandle, 30745e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), 30845e34b43744b75a76cfe09d5026026af83f24de0Jean Chalard inputPointers.getYCoordinates(), inputPointers.getTimes(), 309162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, 31005b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays, 311bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi session.mIsBeginningOfSentenceArray, ngramContext.getPrevWordCount(), 312880624838611a69f20f39ae762181ea4639dd071Keisuke Kuroyanagi session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, 313880624838611a69f20f39ae762181ea4639dd071Keisuke Kuroyanagi session.mSpaceIndices, session.mOutputTypes, 3146da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard session.mOutputAutoCommitFirstWordConfidence, 3156da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard session.mInputOutputWeightOfLangModelVsSpatialModel); 3166da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard if (inOutWeightOfLangModelVsSpatialModel != null) { 3176da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard inOutWeightOfLangModelVsSpatialModel[0] = 3186da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard session.mInputOutputWeightOfLangModelVsSpatialModel[0]; 3191de95ceada64e7fd27ca4ee43243930b5d9c1df7Keisuke Kuroyanagi } 320162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi final int count = session.mOutputSuggestionCount[0]; 321a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); 322f5cded1c6cf0f39df13750d4f9f5ba66c1b32964satok for (int j = 0; j < count; ++j) { 32329500ef4ba8e01f4c467a62399c8249d532ee82cMohammadinamul Sheik final int start = j * DICTIONARY_MAX_WORD_LENGTH; 324979f8690967ff5409fe18f5085858ccdb8e0ccf1satok int len = 0; 32529500ef4ba8e01f4c467a62399c8249d532ee82cMohammadinamul Sheik while (len < DICTIONARY_MAX_WORD_LENGTH 326162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi && session.mOutputCodePoints[start + len] != 0) { 327f5cded1c6cf0f39df13750d4f9f5ba66c1b32964satok ++len; 328979f8690967ff5409fe18f5085858ccdb8e0ccf1satok } 329979f8690967ff5409fe18f5085858ccdb8e0ccf1satok if (len > 0) { 330162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi suggestions.add(new SuggestedWordInfo( 331162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi new String(session.mOutputCodePoints, start, len), 332ab5912959435c1901e268bc9766090e604f3523dMohammadinamul Sheik "" /* prevWordsContext */, 333ab5912959435c1901e268bc9766090e604f3523dMohammadinamul Sheik (int)(session.mOutputScores[j] * weightForLocale), 334ab5912959435c1901e268bc9766090e604f3523dMohammadinamul Sheik session.mOutputTypes[j], 3356da9b21191dc7d6049d96945366ec7e605e716e6Jean Chalard this /* sourceDict */, 336162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, 337162f529e9334a3c920e02771a8b53cad1458cf3eKeisuke Kuroyanagi session.mOutputAutoCommitFirstWordConfidence[0])); 338979f8690967ff5409fe18f5085858ccdb8e0ccf1satok } 339979f8690967ff5409fe18f5085858ccdb8e0ccf1satok } 340d82898c5a91f8aa69d5dc594b7a9290b8be1247aJean Chalard return suggestions; 341923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project } 342923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 34315f6d4ae34664ea3d92827a2c3003198c0bac70bTadashi G. Takaoka public boolean isValidDictionary() { 3446f4eba814a7f8426617db61f928a965209ebf359Tadashi G. Takaoka return mNativeDict != 0; 3456f4eba814a7f8426617db61f928a965209ebf359Tadashi G. Takaoka } 3466f4eba814a7f8426617db61f928a965209ebf359Tadashi G. Takaoka 3472fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa public int getFormatVersion() { 3482fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return getFormatVersionNative(mNativeDict); 3492fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 3502fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 351923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project @Override 3521910392eeddf2c9f4c1d34925e64f8d8772e7dc4Keisuke Kuroyanagi public boolean isInDictionary(final String word) { 3534d02a2d44db94985c9f079cdd58c7c51d3e557eeKeisuke Kuroyanagi return getFrequency(word) != NOT_A_PROBABILITY; 354c88f61215c5b9ca6e0cc3f776e3b7da19eec9caeSatoshi Kataoka } 355c88f61215c5b9ca6e0cc3f776e3b7da19eec9caeSatoshi Kataoka 356c88f61215c5b9ca6e0cc3f776e3b7da19eec9caeSatoshi Kataoka @Override 357bc464e2952e102219f0b977fc1e9140ad5bd03e4Tadashi G. Takaoka public int getFrequency(final String word) { 358d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (TextUtils.isEmpty(word)) { 359d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return NOT_A_PROBABILITY; 360d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 361d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka final int[] codePoints = StringUtils.toCodePointArray(word); 362e0e67373735918c78eaeaf24f127e1d28816aa29Satoshi Kataoka return getProbabilityNative(mNativeDict, codePoints); 363923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project } 364c3df2d6fd27f3a5b84040b59aece3367769f0cb6Amith Yamasani 365d9b8602f4862c2c876e1499aad7ca7d77ea66595Keisuke Kuroyanagi @Override 366d6b89e17a678d0f7cb8a4a2c1dafa72a4568b744Keisuke Kuroyanagi public int getMaxFrequencyOfExactMatches(final String word) { 367d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (TextUtils.isEmpty(word)) { 368d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return NOT_A_PROBABILITY; 369d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 370d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka final int[] codePoints = StringUtils.toCodePointArray(word); 371d6b89e17a678d0f7cb8a4a2c1dafa72a4568b744Keisuke Kuroyanagi return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); 372d6b89e17a678d0f7cb8a4a2c1dafa72a4568b744Keisuke Kuroyanagi } 373d6b89e17a678d0f7cb8a4a2c1dafa72a4568b744Keisuke Kuroyanagi 37482fa755bac1effec15bc50517c50d12de99d0fd6Keisuke Kuroyanagi @UsedForTesting 375bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi public boolean isValidNgram(final NgramContext ngramContext, final String word) { 376bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi return getNgramProbability(ngramContext, word) != NOT_A_PROBABILITY; 3774d02a2d44db94985c9f079cdd58c7c51d3e557eeKeisuke Kuroyanagi } 3784d02a2d44db94985c9f079cdd58c7c51d3e557eeKeisuke Kuroyanagi 379bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi public int getNgramProbability(final NgramContext ngramContext, final String word) { 380bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 381e507d92aa3ee4ae43124c5452f20aa8ed0ecef4cKeisuke Kuroyanagi return NOT_A_PROBABILITY; 382e507d92aa3ee4ae43124c5452f20aa8ed0ecef4cKeisuke Kuroyanagi } 383bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 384bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 385bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 386e708b1bc2e11285ad404133b8de21719ce08acb5Keisuke Kuroyanagi final int[] wordCodePoints = StringUtils.toCodePointArray(word); 38705b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays, 38805b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi isBeginningOfSentenceArray, wordCodePoints); 389f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi } 390f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi 39188fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { 39288fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi if (word == null) { 3932fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return null; 3942fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 3952fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa final int[] codePoints = StringUtils.toCodePointArray(word); 39629500ef4ba8e01f4c467a62399c8249d532ee82cMohammadinamul Sheik final int[] outCodePoints = new int[DICTIONARY_MAX_WORD_LENGTH]; 397080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; 39894d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi final int[] outProbabilityInfo = 39994d9a2309fbca6b1e42b6c57b9c9509182fe8a0bKeisuke Kuroyanagi new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; 400d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi final ArrayList<int[][]> outNgramPrevWordsArray = new ArrayList<>(); 401d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi final ArrayList<boolean[]> outNgramPrevWordIsBeginningOfSentenceArray = 402d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi new ArrayList<>(); 403d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi final ArrayList<int[]> outNgramTargets = new ArrayList<>(); 404d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi final ArrayList<int[]> outNgramProbabilityInfo = new ArrayList<>(); 405a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); 406a91561aa58db1c43092c1caecc051a11fa5391c7Tadashi G. Takaoka final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); 40788fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi getWordPropertyNative(mNativeDict, codePoints, isBeginningOfSentence, outCodePoints, 408d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi outFlags, outProbabilityInfo, outNgramPrevWordsArray, 409d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi outNgramPrevWordIsBeginningOfSentenceArray, outNgramTargets, 410d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi outNgramProbabilityInfo, outShortcutTargets, outShortcutProbabilities); 411080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi return new WordProperty(codePoints, 412080a35e959435566e768f2e31afdac784a4dcd00Keisuke Kuroyanagi outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], 41305172bf1a5693c2e108e91436b98ecd35d2dadadAdrian Velicu outFlags[FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX], 414d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi outFlags[FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX], 41588fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, 416d7a51c242bd21aed28b33279add1a2d54cf3bd60Keisuke Kuroyanagi outNgramPrevWordsArray, outNgramPrevWordIsBeginningOfSentenceArray, 41712d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic outNgramTargets, outNgramProbabilityInfo); 4182fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 4192fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 42038f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi public static class GetNextWordPropertyResult { 42138f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi public WordProperty mWordProperty; 42238f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi public int mNextToken; 42338f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi 4240befbf8289539971d2d4dcb1af24283feb5c0b0aKeisuke Kuroyanagi public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { 4250befbf8289539971d2d4dcb1af24283feb5c0b0aKeisuke Kuroyanagi mWordProperty = wordProperty; 42638f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi mNextToken = nextToken; 42738f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi } 42838f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi } 42938f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi 43038f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi /** 43138f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi * Method to iterate all words in the dictionary for makedict. 43238f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi * If token is 0, this method newly starts iterating the dictionary. 43338f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi */ 43438f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi public GetNextWordPropertyResult getNextWordProperty(final int token) { 43529500ef4ba8e01f4c467a62399c8249d532ee82cMohammadinamul Sheik final int[] codePoints = new int[DICTIONARY_MAX_WORD_LENGTH]; 43688fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi final boolean[] isBeginningOfSentence = new boolean[1]; 43788fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi final int nextToken = getNextWordNative(mNativeDict, token, codePoints, 43888fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi isBeginningOfSentence); 43953fb86c447bc322c13212ae8eaa102f6a3e57604Keisuke Kuroyanagi final String word = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); 44088fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi return new GetNextWordPropertyResult( 44188fa47a27d45f6460971d0d223aa558e121b3478Keisuke Kuroyanagi getWordProperty(word, isBeginningOfSentence[0]), nextToken); 44238f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi } 44338f341a2a53a04ce4195a0cb99fcb6e71203dec0Keisuke Kuroyanagi 4442fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa // Add a unigram entry to binary dictionary with unigram attributes in native code. 44512d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic public boolean addUnigramEntry( 44612d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic final String word, final int probability, final boolean isBeginningOfSentence, 44712d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { 4481adca93381d261a6070be2721dbf8b8abafbfe01Keisuke Kuroyanagi if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { 449dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 450f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi } 451f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi final int[] codePoints = StringUtils.toCodePointArray(word); 45212d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic if (!addUnigramEntryNative(mNativeDict, codePoints, probability, 45312d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic null /* shortcutTargetCodePoints */, 0 /* shortcutProbability */, 45412d80ebead6a1d7f704a5a3af3b6fe3313ceab05Dan Zivkovic isBeginningOfSentence, isNotAWord, isPossiblyOffensive, timestamp)) { 455dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 456dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi } 457fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi mHasUpdated = true; 458dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return true; 459f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi } 460f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi 461b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi // Remove a unigram entry from the binary dictionary in native code. 462b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi public boolean removeUnigramEntry(final String word) { 463b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi if (TextUtils.isEmpty(word)) { 464b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi return false; 465b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi } 466b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi final int[] codePoints = StringUtils.toCodePointArray(word); 46783b6ee25385e1c4fd76ca5842ff025acf945596bKeisuke Kuroyanagi if (!removeUnigramEntryNative(mNativeDict, codePoints)) { 468b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi return false; 469b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi } 470b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi mHasUpdated = true; 471b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi return true; 472b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi } 473b4d77eca55fa48eaf29ab036ac3b098ebac5e691Keisuke Kuroyanagi 474e507d92aa3ee4ae43124c5452f20aa8ed0ecef4cKeisuke Kuroyanagi // Add an n-gram entry to the binary dictionary with timestamp in native code. 475bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi public boolean addNgramEntry(final NgramContext ngramContext, final String word, 476dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi final int probability, final int timestamp) { 477bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi if (!ngramContext.isValid() || TextUtils.isEmpty(word)) { 478dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 479f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi } 480bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 481bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 482bb0eca57054758ef17b032d2654c1fc5f6b32101Keisuke Kuroyanagi ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 483e708b1bc2e11285ad404133b8de21719ce08acb5Keisuke Kuroyanagi final int[] wordCodePoints = StringUtils.toCodePointArray(word); 48405b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays, 48505b1e0d42f9f103516103d4d33e61862c0851e9dKeisuke Kuroyanagi isBeginningOfSentenceArray, wordCodePoints, probability, timestamp)) { 486dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 487dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi } 488fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi mHasUpdated = true; 489dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return true; 490f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi } 491f8d2796724b67ffb7e02c033ae15183a0d58febeKeisuke Kuroynagi 4926accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi // Update entries for the word occurrence with the ngramContext. 4939290d0a4eba454b9b1501830a4e470005cc85332Keisuke Kuroyanagi public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext, 4946accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi final String word, final boolean isValidWord, final int count, final int timestamp) { 4956accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi if (TextUtils.isEmpty(word)) { 4966accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi return false; 4976accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi } 4986accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; 4996accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; 5006accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); 5016accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi final int[] wordCodePoints = StringUtils.toCodePointArray(word); 502ab4437f4681db3fb8327f752df8f08bd0d8cf967Keisuke Kuroyanagi if (!updateEntriesForWordWithNgramContextNative(mNativeDict, prevWordCodePointArrays, 5036accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { 5046accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi return false; 5056accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi } 5066accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi mHasUpdated = true; 5076accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi return true; 5086accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi } 5096accf63d0a1132ee582687592a6aa738ad4443f1Keisuke Kuroyanagi 5108c889784e2d20bb3ebc1ad869176a791a755ccc6Adrian Velicu @UsedForTesting 51116cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi public void updateEntriesForInputEvents(final WordInputEventForPersonalization[] inputEvents) { 512d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (!isValidDictionary()) { 513d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return; 514d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 51516cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi int processedEventCount = 0; 51616cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi while (processedEventCount < inputEvents.length) { 5172fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa if (needsToRunGC(true /* mindsBlockByGC */)) { 5182fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa flushWithGC(); 5192fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 52016cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi processedEventCount = updateEntriesForInputEventsNative(mNativeDict, inputEvents, 52116cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi processedEventCount); 522fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi mHasUpdated = true; 52316cc3992d7468ef781753df7b4227330e0834501Keisuke Kuroyanagi if (processedEventCount <= 0) { 5242fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa return; 5252fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 5262fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 5272fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa } 5282fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa 5296142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi private void reopen() { 5306142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi close(); 5312e58670da9687fd1fd28c322e03343957d11568cKeisuke Kuroyanagi final File dictFile = new File(mDictFilePath); 5322fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa // WARNING: Because we pass 0 as the offset and file.length() as the length, this can 5332fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa // only be called for actual files. Right now it's only called by the flush() family of 5342fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa // functions, which require an updatable dictionary, so it's okay. But beware. 5352fa3693c264a4c150ac307d9bb7f6f8f18cc4ffcKen Wakasa loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 536d934fb43ada7876cc3a7986bbdccaa00bbbec302Keisuke Kuroyanagi dictFile.length(), mIsUpdatable); 537d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 538d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 539c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi // Flush to dict file if the dictionary has been updated. 540dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi public boolean flush() { 541d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (!isValidDictionary()) { 542d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return false; 543d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 544fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi if (mHasUpdated) { 545dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi if (!flushNative(mNativeDict, mDictFilePath)) { 546dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 547dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi } 548fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi reopen(); 549fdd2db576dc69aa69ae99ddb2178a955e71b61f0Keisuke Kuroyanagi } 550dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return true; 5516142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi } 5526142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi 553c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi // Run GC and flush to dict file if the dictionary has been updated. 554dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi public boolean flushWithGCIfHasUpdated() { 555c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi if (mHasUpdated) { 556dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return flushWithGC(); 557c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi } 558dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return true; 559c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi } 560c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi 561c6704dbe512c8b7ee88c3fca4e84fae021ff6c83Keisuke Kuroyanagi // Run GC and flush to dict file. 562dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi public boolean flushWithGC() { 563d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (!isValidDictionary()) { 564d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return false; 565d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 566dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi if (!flushWithGCNative(mNativeDict, mDictFilePath)) { 567dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return false; 568dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi } 5696142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi reopen(); 570dfca51726e9dc9a35f462dee39331823eafa07c9Keisuke Kuroyanagi return true; 571d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 572d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 573c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi /** 574c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi * Checks whether GC is needed to run or not. 575c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about 576c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi * the blocking in some situations such as in idle time or just before closing. 577c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi * @return whether GC is needed to run or not. 578c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi */ 579c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi public boolean needsToRunGC(final boolean mindsBlockByGC) { 580d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (!isValidDictionary()) { 581d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return false; 582d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 583c18510049a3422c88ed3ab3bbc64944c94a611fdKeisuke Kuroyanagi return needsToRunGCNative(mNativeDict, mindsBlockByGC); 584d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi } 585d0246277fde27e9c40a270e206f1d106811e847fKeisuke Kuroyanagi 586a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi public boolean migrateTo(final int newFormatVersion) { 587a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi if (!isValidDictionary()) { 588a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi return false; 589a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi } 5901d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi final File isMigratingDir = 5911d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi new File(mDictFilePath + DIR_NAME_SUFFIX_FOR_RECORD_MIGRATION); 5921d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi if (isMigratingDir.exists()) { 5931d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi isMigratingDir.delete(); 5941d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi Log.e(TAG, "Previous migration attempt failed probably due to a crash. " 5951d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi + "Giving up using the old dictionary (" + mDictFilePath + ")."); 5961471fbad66f68a06494ac3efea5d9d16ea46b322Keisuke Kuroyanagi return false; 5971471fbad66f68a06494ac3efea5d9d16ea46b322Keisuke Kuroyanagi } 5981d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi if (!isMigratingDir.mkdir()) { 5991d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi Log.e(TAG, "Cannot create a dir (" + isMigratingDir.getAbsolutePath() 6001d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi + ") to record migration."); 601455dc84cf2c6526329b535f30000ea45b7d4d4d7Keisuke Kuroyanagi return false; 602455dc84cf2c6526329b535f30000ea45b7d4d4d7Keisuke Kuroyanagi } 6031d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi try { 6041d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; 6051d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { 6061d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi return false; 6071d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi } 6081d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi close(); 6091d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi final File dictFile = new File(mDictFilePath); 6101d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi final File tmpDictFile = new File(tmpDictFilePath); 6111d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi if (!FileUtils.deleteRecursively(dictFile)) { 6121d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi return false; 6131d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi } 6141d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { 6151d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi return false; 6161d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi } 6171d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, 6181d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi dictFile.length(), mIsUpdatable); 6191d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi return true; 6201d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi } finally { 6211d6024d2f8051271e3c9abc1285d989114f5f1d7Keisuke Kuroyanagi isMigratingDir.delete(); 622a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi } 623a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi } 624a785fa8edd7f7a1f91d45c5e66562d92cf5698afKeisuke Kuroyanagi 6254d02a2d44db94985c9f079cdd58c7c51d3e557eeKeisuke Kuroyanagi @UsedForTesting 626743a9b4499c9b53ffedc63f76137ce2eaa3301d0Keisuke Kuroyanagi public String getPropertyForGettingStats(final String query) { 627d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka if (!isValidDictionary()) { 628d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka return ""; 629d3a4c5132422b189c8dbb94dbbe84a9b9761b0a8Tadashi G. Takaoka } 63031097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi return getPropertyNative(mNativeDict, query); 63131097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi } 63231097a57cc6f8022abc0ea56f27147399f41b630Keisuke Kuroyanagi 63336fcf25833f7c8876cbc8286e0c159b052dc2626Amith Yamasani @Override 63424aad5a4d545e743fe43953c1a9d8141c022d355Jean Chalard public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { 635459cd6f8ef3eaa561e47dd996ce537770ea8b37aJean Chalard return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; 63624aad5a4d545e743fe43953c1a9d8141c022d355Jean Chalard } 63724aad5a4d545e743fe43953c1a9d8141c022d355Jean Chalard 63824aad5a4d545e743fe43953c1a9d8141c022d355Jean Chalard @Override 6398ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka public void close() { 6408ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka synchronized (mDicTraverseSessions) { 6418ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka final int sessionsSize = mDicTraverseSessions.size(); 6428ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka for (int index = 0; index < sessionsSize; ++index) { 6438ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka final DicTraverseSession traverseSession = mDicTraverseSessions.valueAt(index); 6448ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka if (traverseSession != null) { 6458ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka traverseSession.close(); 6468ce351a8275f0ad73cdd642e8b46a430b072e8efTadashi G. Takaoka } 6473979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataoka } 6486142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi mDicTraverseSessions.clear(); 6493979f060f0650cbc117eee0307d05fb0be78c6f2Satoshi Kataoka } 6506142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi closeInternalLocked(); 651da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa } 652da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa 6536142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi private synchronized void closeInternalLocked() { 654923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project if (mNativeDict != 0) { 655923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project closeNative(mNativeDict); 656923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project mNativeDict = 0; 657923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project } 658923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project } 659923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project 6606142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi // TODO: Manage BinaryDictionary instances without using WeakReference or something. 661923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project @Override 662923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project protected void finalize() throws Throwable { 663da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa try { 6646142068a3311e4f828bb2acb0e4f9469c29a083fKeisuke Kuroyanagi closeInternalLocked(); 665da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa } finally { 666da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa super.finalize(); 667da50e1e98dadc3733c615dfb8d87fe8b4688c782Ken Wakasa } 668923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project } 669923bf41f853a544fd0d71fbf7dc90359ec35981The Android Open Source Project} 670