UserHistoryDictionaryTests.java revision 5f00fe09e9a611b647592188316e5999465df4d3 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.inputmethod.latin.personalization;
18
19import android.test.AndroidTestCase;
20import android.test.suitebuilder.annotation.LargeTest;
21import android.util.Log;
22
23import com.android.inputmethod.latin.ExpandableBinaryDictionary;
24import com.android.inputmethod.latin.NgramContext;
25import com.android.inputmethod.latin.NgramContext.WordInfo;
26import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
27import com.android.inputmethod.latin.utils.DistracterFilter;
28import com.android.inputmethod.latin.utils.FileUtils;
29
30import java.io.File;
31import java.io.FilenameFilter;
32import java.util.ArrayList;
33import java.util.HashSet;
34import java.util.List;
35import java.util.Locale;
36import java.util.Random;
37import java.util.concurrent.TimeUnit;
38
39/**
40 * Unit tests for UserHistoryDictionary
41 */
42@LargeTest
43public class UserHistoryDictionaryTests extends AndroidTestCase {
44    private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
45    private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
46    private static final String TEST_LOCALE_PREFIX = "test_";
47
48    private static final String[] CHARACTERS = {
49        "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
50        "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
51    };
52
53    private int mCurrentTime = 0;
54
55    private void removeAllTestDictFiles() {
56        final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
57        final String dictName = ExpandableBinaryDictionary.getDictName(
58                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */);
59        final File dictFile = ExpandableBinaryDictionary.getDictFile(
60                mContext, dictName, null /* dictFile */);
61        final FilenameFilter filenameFilter = new FilenameFilter() {
62            @Override
63            public boolean accept(File dir, String filename) {
64                return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
65            }
66        };
67        FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
68    }
69
70    private static void printAllFiles(final File dir) {
71        Log.d(TAG, dir.getAbsolutePath());
72        for (final File file : dir.listFiles()) {
73            Log.d(TAG, "  " + file.getName());
74        }
75    }
76
77    private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
78            final File dictFile) {
79        Log.d(TAG, "waiting for writing ...");
80        dict.waitAllTasksForTests();
81        if (!dictFile.exists()) {
82            try {
83                Log.d(TAG, dictFile + " is not existing. Wait "
84                        + WAIT_FOR_WRITING_FILE_IN_MILLISECONDS + " ms for writing.");
85                printAllFiles(dictFile.getParentFile());
86                Thread.sleep(WAIT_FOR_WRITING_FILE_IN_MILLISECONDS);
87            } catch (final InterruptedException e) {
88                Log.e(TAG, "Interrupted during waiting for writing the dict file.");
89            }
90        }
91        assertTrue("check exisiting of " + dictFile, dictFile.exists());
92        FileUtils.deleteRecursively(dictFile);
93    }
94
95    private static Locale getDummyLocale(final String name) {
96        return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
97    }
98
99    @Override
100    protected void setUp() throws Exception {
101        super.setUp();
102        resetCurrentTimeForTestMode();
103        removeAllTestDictFiles();
104    }
105
106    @Override
107    protected void tearDown() throws Exception {
108        removeAllTestDictFiles();
109        stopTestModeInNativeCode();
110        super.tearDown();
111    }
112
113    private void resetCurrentTimeForTestMode() {
114        mCurrentTime = 0;
115        setCurrentTimeForTestMode(mCurrentTime);
116    }
117
118    private void forcePassingShortTime() {
119        // 3 days.
120        final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(3);
121        mCurrentTime += timeToElapse;
122        setCurrentTimeForTestMode(mCurrentTime);
123    }
124
125    private void forcePassingLongTime() {
126        // 365 days.
127        final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(365);
128        mCurrentTime += timeToElapse;
129        setCurrentTimeForTestMode(mCurrentTime);
130    }
131
132    private static int setCurrentTimeForTestMode(final int currentTime) {
133        return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
134    }
135
136    private static int stopTestModeInNativeCode() {
137        return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
138    }
139
140    /**
141     * Generates a random word.
142     */
143    private static String generateWord(final int value) {
144        final int lengthOfChars = CHARACTERS.length;
145        StringBuilder builder = new StringBuilder();
146        long lvalue = Math.abs((long)value);
147        while (lvalue > 0) {
148            builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
149            lvalue /= lengthOfChars;
150        }
151        return builder.toString();
152    }
153
154    private static List<String> generateWords(final int number, final Random random) {
155        final HashSet<String> wordSet = new HashSet<>();
156        while (wordSet.size() < number) {
157            wordSet.add(generateWord(random.nextInt()));
158        }
159        return new ArrayList<>(wordSet);
160    }
161
162    private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
163            final int timestamp) {
164        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
165        for (String word : words) {
166            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
167                    DistracterFilter.EMPTY_DISTRACTER_FILTER);
168            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
169        }
170    }
171
172    /**
173     * @param checkContents if true, checks whether written words are actually in the dictionary
174     * or not.
175     */
176    private void addAndWriteRandomWords(final UserHistoryDictionary dict,
177            final int numberOfWords, final Random random, final boolean checkContents) {
178        final List<String> words = generateWords(numberOfWords, random);
179        // Add random words to the user history dictionary.
180        addToDict(dict, words, mCurrentTime);
181        if (checkContents) {
182            dict.waitAllTasksForTests();
183            for (int i = 0; i < numberOfWords; ++i) {
184                final String word = words.get(i);
185                assertTrue(dict.isInDictionary(word));
186            }
187        }
188        // write to file.
189        dict.close();
190    }
191
192    /**
193     * Clear all entries in the user history dictionary.
194     * @param dict the user history dictionary.
195     */
196    private static void clearHistory(final UserHistoryDictionary dict) {
197        dict.waitAllTasksForTests();
198        dict.clear();
199        dict.close();
200        dict.waitAllTasksForTests();
201    }
202
203    public void testRandomWords() {
204        Log.d(TAG, "This test can be used for profiling.");
205        Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
206        final Locale dummyLocale = getDummyLocale("random_words");
207        final String dictName = ExpandableBinaryDictionary.getDictName(
208                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */);
209        final File dictFile = ExpandableBinaryDictionary.getDictFile(
210                mContext, dictName, null /* dictFile */);
211        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
212                getContext(), dummyLocale);
213
214        final int numberOfWords = 1000;
215        final Random random = new Random(123456);
216        clearHistory(dict);
217        addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
218        checkExistenceAndRemoveDictFile(dict, dictFile);
219    }
220
221    public void testStressTestForSwitchingLanguagesAndAddingWords() {
222        final int numberOfLanguages = 2;
223        final int numberOfLanguageSwitching = 80;
224        final int numberOfWordsInsertedForEachLanguageSwitch = 100;
225
226        final File dictFiles[] = new File[numberOfLanguages];
227        final UserHistoryDictionary dicts[] = new UserHistoryDictionary[numberOfLanguages];
228
229        try {
230            final Random random = new Random(123456);
231
232            // Create filename suffixes for this test.
233            for (int i = 0; i < numberOfLanguages; i++) {
234                final Locale dummyLocale = getDummyLocale("switching_languages" + i);
235                final String dictName = ExpandableBinaryDictionary.getDictName(
236                        UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */);
237                dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
238                        mContext, dictName, null /* dictFile */);
239                dicts[i] = PersonalizationHelper.getUserHistoryDictionary(getContext(),
240                        dummyLocale);
241                clearHistory(dicts[i]);
242            }
243
244            final long start = System.currentTimeMillis();
245
246            for (int i = 0; i < numberOfLanguageSwitching; i++) {
247                final int index = i % numberOfLanguages;
248                // Switch to dicts[index].
249                addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
250                        random, false /* checksContents */);
251            }
252
253            final long end = System.currentTimeMillis();
254            Log.d(TAG, "testStressTestForSwitchingLanguageAndAddingWords took "
255                    + (end - start) + " ms");
256        } finally {
257            for (int i = 0; i < numberOfLanguages; i++) {
258                checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
259            }
260        }
261    }
262
263    public void testAddManyWords() {
264        final Locale dummyLocale = getDummyLocale("many_random_words");
265        final String dictName = ExpandableBinaryDictionary.getDictName(
266                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */);
267        final File dictFile = ExpandableBinaryDictionary.getDictFile(
268                mContext, dictName, null /* dictFile */);
269        final int numberOfWords = 10000;
270        final Random random = new Random(123456);
271        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
272                getContext(), dummyLocale);
273        clearHistory(dict);
274        try {
275            addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
276        } finally {
277            checkExistenceAndRemoveDictFile(dict, dictFile);
278        }
279    }
280
281    public void testDecaying() {
282        final Locale dummyLocale = getDummyLocale("decaying");
283        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
284                getContext(), dummyLocale);
285        final int numberOfWords = 5000;
286        final Random random = new Random(123456);
287        resetCurrentTimeForTestMode();
288        clearHistory(dict);
289        final List<String> words = generateWords(numberOfWords, random);
290        dict.waitAllTasksForTests();
291        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
292        for (final String word : words) {
293            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
294                    DistracterFilter.EMPTY_DISTRACTER_FILTER);
295            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
296            dict.waitAllTasksForTests();
297            assertTrue(dict.isInDictionary(word));
298        }
299        forcePassingShortTime();
300        dict.runGCIfRequired();
301        dict.waitAllTasksForTests();
302        for (final String word : words) {
303            assertTrue(dict.isInDictionary(word));
304        }
305        forcePassingLongTime();
306        dict.runGCIfRequired();
307        dict.waitAllTasksForTests();
308        for (final String word : words) {
309            assertFalse(dict.isInDictionary(word));
310        }
311    }
312}
313