1d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada/* 2d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Copyright (C) 2012 The Android Open Source Project 3d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * 4d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Licensed under the Apache License, Version 2.0 (the "License"); 5d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * you may not use this file except in compliance with the License. 6d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * You may obtain a copy of the License at 7d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * 8d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * http://www.apache.org/licenses/LICENSE-2.0 9d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * 10d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Unless required by applicable law or agreed to in writing, software 11d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * distributed under the License is distributed on an "AS IS" BASIS, 12d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * See the License for the specific language governing permissions and 14d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * limitations under the License. 15d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada */ 16d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 172a2b5edc21c9bbea0ee172baf4b90053ceff98b5Yuichiro Hanadapackage com.android.inputmethod.latin.makedict; 18d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 192a2b5edc21c9bbea0ee172baf4b90053ceff98b5Yuichiro Hanadaimport com.android.inputmethod.latin.CollectionUtils; 20be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanadaimport com.android.inputmethod.latin.UserHistoryDictIOUtils; 21be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanadaimport com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; 22d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanadaimport com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; 23d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; 24d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport com.android.inputmethod.latin.makedict.FusionDictionary.Node; 256e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanadaimport com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; 26d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 27d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport android.test.AndroidTestCase; 2866597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanadaimport android.test.MoreAsserts; 29d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport android.util.Log; 30d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport android.util.SparseArray; 31d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 32d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.io.File; 33d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.io.FileInputStream; 34be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanadaimport java.io.FileOutputStream; 35d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.io.IOException; 36be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanadaimport java.nio.ByteBuffer; 37be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanadaimport java.nio.channels.FileChannel; 38d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.util.ArrayList; 39d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.util.HashMap; 40b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanadaimport java.util.HashSet; 41d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.util.List; 42b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanadaimport java.util.Map; 43b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanadaimport java.util.Map.Entry; 44d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.util.Random; 45d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.util.Set; 46d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 47d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada/** 48d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Unit tests for BinaryDictInputOutput 49d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada */ 50d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadapublic class BinaryDictIOTests extends AndroidTestCase { 51d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private static final String TAG = BinaryDictIOTests.class.getSimpleName(); 52d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private static final int MAX_UNIGRAMS = 1000; 53d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private static final int UNIGRAM_FREQ = 10; 54d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private static final int BIGRAM_FREQ = 50; 55b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; 56d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 57be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada private static final int USE_BYTE_ARRAY = 1; 58be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada private static final int USE_BYTE_BUFFER = 2; 59be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 6013b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private static final List<String> sWords = CollectionUtils.newArrayList(); 6113b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private static final SparseArray<List<Integer>> sEmptyBigrams = 6213b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada CollectionUtils.newSparseArray(); 6313b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private static final SparseArray<List<Integer>> sStarBigrams = CollectionUtils.newSparseArray(); 6413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private static final SparseArray<List<Integer>> sChainBigrams = 6513b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada CollectionUtils.newSparseArray(); 6613b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 671a347723c5ad4a71076df67f3af3b702db205719Yuichiro Hanada private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); 6882d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = 6982d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); 7082d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = 7182d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); 7283dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada 7313b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private static final String[] CHARACTERS = { 74d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", 75d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" 7613b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada }; 7713b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 7813b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada public BinaryDictIOTests() { 7913b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada super(); 8013b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 8113b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada final Random random = new Random(123456); 8213b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada sWords.clear(); 8313b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada generateWords(MAX_UNIGRAMS, random); 8413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 8513b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada for (int i = 0; i < sWords.size(); ++i) { 8613b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada sChainBigrams.put(i, new ArrayList<Integer>()); 8713b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada if (i > 0) { 887e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada sChainBigrams.get(i - 1).add(i); 8913b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada } 9013b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada } 9113b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 9213b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada sStarBigrams.put(0, new ArrayList<Integer>()); 9313b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada for (int i = 1; i < sWords.size(); ++i) { 9413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada sStarBigrams.get(0).add(i); 9513b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada } 9613b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada } 97d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 98b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // Utilities for test 99be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 100be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada /** 101be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada * Makes new buffer according to BUFFER_TYPE. 102be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada */ 1037e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada private FusionDictionaryBufferInterface getBuffer(final File file, final int bufferType) { 104be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada FileInputStream inStream = null; 105be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada try { 106be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada inStream = new FileInputStream(file); 107be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada if (bufferType == USE_BYTE_ARRAY) { 108be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final byte[] array = new byte[(int)file.length()]; 109be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada inStream.read(array); 110be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada return new UserHistoryDictIOUtils.ByteArrayWrapper(array); 111be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } else if (bufferType == USE_BYTE_BUFFER){ 112be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final ByteBuffer buffer = inStream.getChannel().map( 113be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada FileChannel.MapMode.READ_ONLY, 0, file.length()); 114be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada return new BinaryDictInputOutput.ByteBufferWrapper(buffer); 115be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 116be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } catch (IOException e) { 117be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada Log.e(TAG, "IOException while making buffer: " + e); 118be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } finally { 119be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada if (inStream != null) { 120be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada try { 121be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada inStream.close(); 122be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } catch (IOException e) { 123be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada Log.e(TAG, "IOException while closing stream: " + e); 124be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 125be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 126be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 127be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada return null; 128be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 129be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 130d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada /** 131d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Generates a random word. 132d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada */ 133d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private String generateWord(final int value) { 134d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int lengthOfChars = CHARACTERS.length; 135d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada StringBuilder builder = new StringBuilder("a"); 136d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada long lvalue = Math.abs((long)value); 137d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada while (lvalue > 0) { 138d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); 139d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada lvalue /= lengthOfChars; 140d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 141d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada return builder.toString(); 142d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 143d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 14413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private void generateWords(final int number, final Random random) { 145d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final Set<String> wordSet = CollectionUtils.newHashSet(); 146d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada while (wordSet.size() < number) { 147d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada wordSet.add(generateWord(random.nextInt())); 148d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 14913b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada sWords.addAll(wordSet); 150d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 151d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 152b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada /** 153b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada * Adds unigrams to the dictionary. 154b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada */ 1556e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada private void addUnigrams(final int number, final FusionDictionary dict, 1566e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada final List<String> words, final Map<String, List<String>> shortcutMap) { 157d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada for (int i = 0; i < number; ++i) { 158d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final String word = words.get(i); 1596e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); 1606e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada if (shortcutMap != null && shortcutMap.containsKey(word)) { 1616e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada for (final String shortcut : shortcutMap.get(word)) { 1626e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); 1636e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada } 1646e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada } 1656e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada dict.add(word, UNIGRAM_FREQ, (shortcutMap == null) ? null : shortcuts, 1666e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada false /* isNotAWord */); 167d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 168d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 169d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 170d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private void addBigrams(final FusionDictionary dict, 171d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final List<String> words, 172b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final SparseArray<List<Integer>> bigrams) { 173b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (int i = 0; i < bigrams.size(); ++i) { 174b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final int w1 = bigrams.keyAt(i); 175b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (int w2 : bigrams.valueAt(i)) { 176d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada dict.setBigram(words.get(w1), words.get(w2), BIGRAM_FREQ); 177d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 178d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 179d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 180d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 1817e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada private long timeWritingDictToFile(final File file, final FusionDictionary dict, 1827e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final FormatSpec.FormatOptions formatOptions) { 183d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 184d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada long now = -1, diff = -1; 185d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 186d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada try { 187d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final FileOutputStream out = new FileOutputStream(file); 188d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 189d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada now = System.currentTimeMillis(); 1907e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions); 191d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada diff = System.currentTimeMillis() - now; 192d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 193d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada out.flush(); 194d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada out.close(); 195d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } catch (IOException e) { 196d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada Log.e(TAG, "IO exception while writing file: " + e); 197d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } catch (UnsupportedFormatException e) { 198d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada Log.e(TAG, "UnsupportedFormatException: " + e); 199d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 200d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 201d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada return diff; 202d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 203d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 2046e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada private void checkDictionary(final FusionDictionary dict, final List<String> words, 2056e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap) { 206d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada assertNotNull(dict); 207d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 208d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada // check unigram 209d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada for (final String word : words) { 210d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word); 211d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada assertNotNull(cg); 212d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 213d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 214d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada // check bigram 215d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada for (int i = 0; i < bigrams.size(); ++i) { 216d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int w1 = bigrams.keyAt(i); 217d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada for (final int w2 : bigrams.valueAt(i)) { 218d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, words.get(w1)); 219d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada assertNotNull(words.get(w1) + "," + words.get(w2), cg.getBigram(words.get(w2))); 220d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 221d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 2226e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada 2236e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada // check shortcut 2246e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada if (shortcutMap != null) { 2256e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada for (final Map.Entry<String, List<String>> entry : shortcutMap.entrySet()) { 2266e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, entry.getKey()); 2276e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada for (final String word : entry.getValue()) { 2286e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada assertNotNull("shortcut not found: " + entry.getKey() + ", " + word, 2296e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada group.getShortcut(word)); 2306e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada } 2316e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada } 2326e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada } 233d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 234d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 2357e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada private String outputOptions(final int bufferType, 2367e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final FormatSpec.FormatOptions formatOptions) { 2377e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada String result = " : buffer type = " 2387e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada + ((bufferType == USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); 2397e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada result += " : version = " + formatOptions.mVersion; 24082d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; 2417e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada } 2427e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada 243b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // Tests for readDictionaryBinary and writeDictionaryBinary 244b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 245b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada private long timeReadingAndCheckDict(final File file, final List<String> words, 246be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap, 247be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final int bufferType) { 248d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada long now, diff = -1; 249be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType); 250be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada assertNotNull(buffer); 251d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 252be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada FusionDictionary dict = null; 253d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada try { 254d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada now = System.currentTimeMillis(); 255be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada dict = BinaryDictInputOutput.readDictionaryBinary(buffer, null); 256be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada diff = System.currentTimeMillis() - now; 257d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } catch (IOException e) { 258be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada Log.e(TAG, "IOException while reading dictionary: " + e); 259d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } catch (UnsupportedFormatException e) { 2607e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada Log.e(TAG, "Unsupported format: " + e); 261d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 262d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 263be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada checkDictionary(dict, words, bigrams, shortcutMap); 264d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada return diff; 265d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 266d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 2676e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada // Tests for readDictionaryBinary and writeDictionaryBinary 268d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada private String runReadAndWrite(final List<String> words, 2696e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcuts, 2707e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final int bufferType, final FormatSpec.FormatOptions formatOptions, 2717e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final String message) { 272b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada File file = null; 273b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada try { 274b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada file = File.createTempFile("runReadAndWrite", ".dict"); 275b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } catch (IOException e) { 276b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada Log.e(TAG, "IOException: " + e); 277b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 278b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertNotNull(file); 279d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 28013b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada final FusionDictionary dict = new FusionDictionary(new Node(), 2817e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); 2826e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada addUnigrams(words.size(), dict, words, shortcuts); 283d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada addBigrams(dict, words, bigrams); 2846e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada checkDictionary(dict, words, bigrams, shortcuts); 285d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 2867e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final long write = timeWritingDictToFile(file, dict, formatOptions); 287be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); 288be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 2897e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada return "PROF: read=" + read + "ms, write=" + write + "ms :" + message 2907e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada + " : " + outputOptions(bufferType, formatOptions); 2917e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada } 2927e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada 2937e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada private void runReadAndWriteTests(final List<String> results, final int bufferType, 2947e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final FormatSpec.FormatOptions formatOptions) { 2957e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadAndWrite(sWords, sEmptyBigrams, null /* shortcuts */, bufferType, 2967e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "unigram")); 2977e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadAndWrite(sWords, sChainBigrams, null /* shortcuts */, bufferType, 2987e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "chain")); 2997e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */, bufferType, 3007e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "star")); 301be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 302be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 303be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada public void testReadAndWriteWithByteBuffer() { 304be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final List<String> results = CollectionUtils.newArrayList(); 305d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 3067e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); 30782d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); 30882d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); 309be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 310be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada for (final String result : results) { 311be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada Log.d(TAG, result); 312be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 313d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 314d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 315be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada public void testReadAndWriteWithByteArray() { 316be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final List<String> results = CollectionUtils.newArrayList(); 317d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 3187e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); 31982d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); 32082d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); 321d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 322d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada for (final String result : results) { 323d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada Log.d(TAG, result); 324d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 325d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 326b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 327b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // Tests for readUnigramsAndBigramsBinary 328b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 329b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada private void checkWordMap(final List<String> expectedWords, 330b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final SparseArray<List<Integer>> expectedBigrams, 331b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, String> resultWords, 332b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, Integer> resultFrequencies, 333b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, ArrayList<PendingAttribute>> resultBigrams) { 334b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // check unigrams 335b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Set<String> actualWordsSet = new HashSet<String>(resultWords.values()); 336b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Set<String> expectedWordsSet = new HashSet<String>(expectedWords); 337b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertEquals(actualWordsSet, expectedWordsSet); 338b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 339b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (int freq : resultFrequencies.values()) { 340b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertEquals(freq, UNIGRAM_FREQ); 341b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 342b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 343b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // check bigrams 344b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<String, List<String>> expBigrams = new HashMap<String, List<String>>(); 345b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (int i = 0; i < expectedBigrams.size(); ++i) { 346b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final String word1 = expectedWords.get(expectedBigrams.keyAt(i)); 347b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (int w2 : expectedBigrams.valueAt(i)) { 348b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada if (expBigrams.get(word1) == null) { 349b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada expBigrams.put(word1, new ArrayList<String>()); 350b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 351b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada expBigrams.get(word1).add(expectedWords.get(w2)); 352b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 353b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 354b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 355b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<String, List<String>> actBigrams = new HashMap<String, List<String>>(); 356b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (Entry<Integer, ArrayList<PendingAttribute>> entry : resultBigrams.entrySet()) { 357b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final String word1 = resultWords.get(entry.getKey()); 358b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final int unigramFreq = resultFrequencies.get(entry.getKey()); 359b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada for (PendingAttribute attr : entry.getValue()) { 360b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final String word2 = resultWords.get(attr.mAddress); 361b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada if (actBigrams.get(word1) == null) { 362b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada actBigrams.put(word1, new ArrayList<String>()); 363b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 364b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada actBigrams.get(word1).add(word2); 365b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 366b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency( 367b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada unigramFreq, attr.mFrequency); 368b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); 369b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 370b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 371b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 372b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertEquals(actBigrams, expBigrams); 373b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 374b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 375b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words, 376be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final SparseArray<List<Integer>> bigrams, final int bufferType) { 377b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada FileInputStream inStream = null; 378b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 379b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, String> resultWords = CollectionUtils.newTreeMap(); 380b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, ArrayList<PendingAttribute>> resultBigrams = 381b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada CollectionUtils.newTreeMap(); 382b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap(); 383b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 384b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada long now = -1, diff = -1; 385be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType); 386be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada assertNotNull("Can't get buffer.", buffer); 387b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada try { 388b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada now = System.currentTimeMillis(); 38965feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs, 390b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada resultBigrams); 391b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada diff = System.currentTimeMillis() - now; 392b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } catch (IOException e) { 393b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada Log.e(TAG, "IOException " + e); 394b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } catch (UnsupportedFormatException e) { 395b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada Log.e(TAG, "UnsupportedFormatException: " + e); 396b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } finally { 397b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada if (inStream != null) { 398b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada try { 399b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada inStream.close(); 400b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } catch (IOException e) { 401b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada // do nothing 402b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 403b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 404b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 405b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 406be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams); 407b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada return diff; 408b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 409b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 41013b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada private String runReadUnigramsAndBigramsBinary(final List<String> words, 411be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final SparseArray<List<Integer>> bigrams, final int bufferType, 4127e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final FormatSpec.FormatOptions formatOptions, final String message) { 413b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada File file = null; 414b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada try { 415b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada file = File.createTempFile("runReadUnigrams", ".dict"); 416b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } catch (IOException e) { 417b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada Log.e(TAG, "IOException: " + e); 418b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 419b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada assertNotNull(file); 420b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 42113b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada // making the dictionary from lists of words. 42213b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada final FusionDictionary dict = new FusionDictionary(new Node(), 42313b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada new FusionDictionary.DictionaryOptions( 42413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada new HashMap<String, String>(), false, false)); 4256e422af8817259def2ca13fb9259361c8d11f6bcYuichiro Hanada addUnigrams(words.size(), dict, words, null /* shortcutMap */); 426b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada addBigrams(dict, words, bigrams); 42713b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada 4287e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada timeWritingDictToFile(file, dict, formatOptions); 429b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 430be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType); 431be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, 432be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada bufferType); 433b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 43413b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap 4357e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada + " : " + message + " : " + outputOptions(bufferType, formatOptions); 4367e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada } 4377e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada 4387e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada private void runReadUnigramsAndBigramsTests(final List<String> results, final int bufferType, 4397e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada final FormatSpec.FormatOptions formatOptions) { 4407e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadUnigramsAndBigramsBinary(sWords, sEmptyBigrams, bufferType, 4417e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "unigram")); 4427e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, 4437e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "chain")); 4447e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, 4457e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada formatOptions, "star")); 446be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 447be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 448be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada public void testReadUnigramsAndBigramsBinaryWithByteBuffer() { 449be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final List<String> results = CollectionUtils.newArrayList(); 450be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 4517e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); 45282d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); 45382d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); 454be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada 455be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada for (final String result : results) { 456be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada Log.d(TAG, result); 457be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada } 458b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 459b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 460be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada public void testReadUnigramsAndBigramsBinaryWithByteArray() { 461be5db53a09a705575e3902769b44d687142f6a83Yuichiro Hanada final List<String> results = CollectionUtils.newArrayList(); 462b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 4637e35841053df859880b6f45b62f03e20b5cb7982Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); 46482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); 46582d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); 466b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 46713b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada for (final String result : results) { 46813b85c416716345811fad7804aab7e87dc174bd0Yuichiro Hanada Log.d(TAG, result); 469b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 470b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada } 471d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 472d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // Tests for getTerminalPosition 473d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada private String getWordFromBinary(final FusionDictionaryBufferInterface buffer, 474d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final int address) { 475d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada if (buffer.position() != 0) buffer.position(0); 476d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 477d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada FileHeader header = null; 478d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada try { 479d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada header = BinaryDictInputOutput.readHeader(buffer); 480d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (IOException e) { 481d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada return null; 482d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (UnsupportedFormatException e) { 483d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada return null; 484d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 485d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada if (header == null) return null; 486d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize, 487d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada address - header.mHeaderSize, header.mFormatOptions); 488d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 489d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 490d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada private long runGetTerminalPosition(final FusionDictionaryBufferInterface buffer, 491d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final String word, int index, boolean contained) { 492d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; 493d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada long diff = -1; 494d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada int position = -1; 495d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada try { 496d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final long now = System.nanoTime(); 497d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada position = BinaryDictIOUtils.getTerminalPosition(buffer, word); 498d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada diff = System.nanoTime() - now; 499d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (IOException e) { 500d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada Log.e(TAG, "IOException while getTerminalPosition: " + e); 501d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (UnsupportedFormatException e) { 502d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada Log.e(TAG, "UnsupportedFormatException while getTermianlPosition: " + e); 503d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 504d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 505d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD != position, contained); 506d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada if (contained) assertEquals(getWordFromBinary(buffer, position), word); 507d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada return diff; 508d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 509d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 510d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada public void testGetTerminalPosition() { 511d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada File file = null; 512d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada try { 51366597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada file = File.createTempFile("testGetTerminalPosition", ".dict"); 514d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (IOException e) { 515d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // do nothing 516d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 517d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada assertNotNull(file); 518d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 519d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final FusionDictionary dict = new FusionDictionary(new Node(), 520d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada new FusionDictionary.DictionaryOptions( 521d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada new HashMap<String, String>(), false, false)); 522d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); 52382d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); 524d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 525d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY); 526d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 527d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada try { 528d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // too long word 529d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; 530d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD, 531d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, longWord)); 532d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 533d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // null 534d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD, 535d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, null)); 536d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 537d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // empty string 538d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD, 539d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, "")); 540d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (IOException e) { 541d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } catch (UnsupportedFormatException e) { 542d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 543d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 544d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // Test a word that is contained within the dictionary. 545d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada long sum = 0; 546d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada for (int i = 0; i < sWords.size(); ++i) { 547d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final long time = runGetTerminalPosition(buffer, sWords.get(i), i, true); 548d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada sum += time == -1 ? 0 : time; 549d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 550d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada Log.d(TAG, "per a search : " + (((double)sum) / sWords.size() / 1000000)); 551d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada 552d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada // Test a word that isn't contained within the dictionary. 553d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final Random random = new Random((int)System.currentTimeMillis()); 554d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada for (int i = 0; i < 1000; ++i) { 555d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final String word = generateWord(random.nextInt()); 556d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada if (sWords.indexOf(word) != -1) continue; 557d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada runGetTerminalPosition(buffer, word, i, false); 558d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 559d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada } 56066597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada 56166597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada public void testDeleteWord() { 56266597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada File file = null; 56366597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada try { 56482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada file = File.createTempFile("testDeleteWord", ".dict"); 56566597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } catch (IOException e) { 56666597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada // do nothing 56766597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } 56866597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada assertNotNull(file); 56966597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada 57066597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada final FusionDictionary dict = new FusionDictionary(new Node(), 57166597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada new FusionDictionary.DictionaryOptions( 57266597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada new HashMap<String, String>(), false, false)); 57366597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); 57482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); 57566597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada 57666597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY); 57766597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada 57866597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada try { 57966597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, 58066597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); 58166597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.deleteWord(buffer, sWords.get(0)); 58266597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD, 58366597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); 58466597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada 58566597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, 58666597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); 58766597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.deleteWord(buffer, sWords.get(5)); 58866597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada assertEquals(FormatSpec.NOT_VALID_WORD, 58966597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); 59066597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } catch (IOException e) { 59166597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } catch (UnsupportedFormatException e) { 59266597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } 59366597f5e5f3249f418665c1990fb539d2f5565d5Yuichiro Hanada } 594d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada} 595