MainLogBuffer.java revision 9c3860ce461c3791891bf667edc77fe798c8d332
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.inputmethod.research; 18 19import android.util.Log; 20 21import com.android.inputmethod.latin.Dictionary; 22import com.android.inputmethod.latin.Suggest; 23import com.android.inputmethod.latin.define.ProductionFlag; 24 25import java.util.ArrayList; 26import java.util.LinkedList; 27import java.util.Random; 28 29/** 30 * MainLogBuffer is a FixedLogBuffer that tracks the state of LogUnits to make privacy guarantees. 31 * 32 * There are three forms of privacy protection: 1) only words in the main dictionary are allowed to 33 * be logged in enough detail to determine their contents, 2) only a subset of words are logged 34 * in detail, such as 10%, and 3) no numbers are logged. 35 * 36 * This class maintains a list of LogUnits, each corresponding to a word. As the user completes 37 * words, they are added here. But if the user backs up over their current word to edit a word 38 * entered earlier, then it is pulled out of this LogBuffer, changes are then added to the end of 39 * the LogUnit, and it is pushed back in here when the user is done. Because words may be pulled 40 * back out even after they are pushed in, we must not publish the contents of this LogBuffer too 41 * quickly. However, we cannot let the contents pile up either, or it will limit the editing that 42 * a user can perform. 43 * 44 * To balance these requirements (keep history so user can edit, flush history so it does not pile 45 * up), the LogBuffer is considered "complete" when the user has entered enough words to form an 46 * n-gram, followed by enough additional non-detailed words (that are in the 90%, as per above). 47 * Once complete, the n-gram may be published to flash storage (via the ResearchLog class). 48 * However, the additional non-detailed words are retained, in case the user backspaces to edit 49 * them. The MainLogBuffer then continues to add words, publishing individual non-detailed words 50 * as new words arrive. After enough non-detailed words have been pushed out to account for the 51 * 90% between words, the words at the front of the LogBuffer can be published as an n-gram again. 52 * 53 * If the words that would form the valid n-gram are not in the dictionary, then words are pushed 54 * through the LogBuffer one at a time until an n-gram is found that is entirely composed of 55 * dictionary words. 56 * 57 * If the user closes a session, then the entire LogBuffer is flushed, publishing any embedded 58 * n-gram containing dictionary words. 59 */ 60public abstract class MainLogBuffer extends FixedLogBuffer { 61 private static final String TAG = MainLogBuffer.class.getSimpleName(); 62 private static final boolean DEBUG = false 63 && ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS_DEBUG; 64 65 // The size of the n-grams logged. E.g. N_GRAM_SIZE = 2 means to sample bigrams. 66 public static final int N_GRAM_SIZE = 2; 67 68 private Suggest mSuggest; 69 private boolean mIsStopping = false; 70 71 /* package for test */ int mNumWordsBetweenNGrams; 72 73 // Counter for words left to suppress before an n-gram can be sampled. Reset to mMinWordPeriod 74 // after a sample is taken. 75 /* package for test */ int mNumWordsUntilSafeToSample; 76 77 public MainLogBuffer(final int wordsBetweenSamples, final int numInitialWordsToIgnore) { 78 super(N_GRAM_SIZE + wordsBetweenSamples); 79 mNumWordsBetweenNGrams = wordsBetweenSamples; 80 mNumWordsUntilSafeToSample = DEBUG ? 0 : numInitialWordsToIgnore; 81 } 82 83 public void setSuggest(final Suggest suggest) { 84 mSuggest = suggest; 85 } 86 87 private Dictionary getDictionary() { 88 if (mSuggest == null || !mSuggest.hasMainDictionary()) return null; 89 return mSuggest.getMainDictionary(); 90 } 91 92 public void resetWordCounter() { 93 mNumWordsUntilSafeToSample = mNumWordsBetweenNGrams; 94 } 95 96 public void setIsStopping() { 97 mIsStopping = true; 98 } 99 100 /** 101 * Determines whether uploading the n words at the front the MainLogBuffer will not violate 102 * user privacy. 103 * 104 * The size of the MainLogBuffer is just enough to hold one n-gram, its corrections, and any 105 * non-character data that is typed between words. The decision about privacy is made based on 106 * the buffer's entire content. If it is decided that the privacy risks are too great to upload 107 * the contents of this buffer, a censored version of the LogItems may still be uploaded. E.g., 108 * the screen orientation and other characteristics about the device can be uploaded without 109 * revealing much about the user. 110 */ 111 private boolean isSafeNGram(final ArrayList<LogUnit> logUnits, final int minNGramSize) { 112 // Bypass privacy checks when debugging. 113 if (ResearchLogger.IS_LOGGING_EVERYTHING) { 114 if (mIsStopping) { 115 return true; 116 } 117 // Only check that it is the right length. If not, wait for later words to make 118 // complete n-grams. 119 int numWordsInLogUnitList = 0; 120 final int length = logUnits.size(); 121 for (int i = 0; i < length; i++) { 122 final LogUnit logUnit = logUnits.get(i); 123 final String word = logUnit.getWord(); 124 if (word != null) { 125 numWordsInLogUnitList++; 126 } 127 } 128 return numWordsInLogUnitList >= minNGramSize; 129 } 130 131 // Check that we are not sampling too frequently. Having sampled recently might disclose 132 // too much of the user's intended meaning. 133 if (mNumWordsUntilSafeToSample > 0) { 134 return false; 135 } 136 // Reload the dictionary in case it has changed (e.g., because the user has changed 137 // languages). 138 final Dictionary dictionary = getDictionary(); 139 if (dictionary == null) { 140 // Main dictionary is unavailable. Since we cannot check it, we cannot tell if a 141 // word is out-of-vocabulary or not. Therefore, we must judge the entire buffer 142 // contents to potentially pose a privacy risk. 143 return false; 144 } 145 146 // Check each word in the buffer. If any word poses a privacy threat, we cannot upload 147 // the complete buffer contents in detail. 148 int numWordsInLogUnitList = 0; 149 final int length = logUnits.size(); 150 for (int i = 0; i < length; i++) { 151 final LogUnit logUnit = logUnits.get(i); 152 if (!logUnit.hasWord()) { 153 // Digits outside words are a privacy threat. 154 if (logUnit.mayContainDigit()) { 155 return false; 156 } 157 } else { 158 numWordsInLogUnitList++; 159 final String word = logUnit.getWord(); 160 // Words not in the dictionary are a privacy threat. 161 if (ResearchLogger.hasLetters(word) && !(dictionary.isValidWord(word))) { 162 if (DEBUG) { 163 Log.d(TAG, "NOT SAFE!: hasLetters: " + ResearchLogger.hasLetters(word) 164 + ", isValid: " + (dictionary.isValidWord(word))); 165 } 166 return false; 167 } 168 } 169 } 170 171 // Finally, only return true if the minNGramSize is met. 172 return numWordsInLogUnitList >= minNGramSize; 173 } 174 175 public void shiftAndPublishAll() { 176 final LinkedList<LogUnit> logUnits = getLogUnits(); 177 while (!logUnits.isEmpty()) { 178 publishLogUnitsAtFrontOfBuffer(); 179 } 180 } 181 182 @Override 183 protected final void onBufferFull() { 184 publishLogUnitsAtFrontOfBuffer(); 185 } 186 187 protected final void publishLogUnitsAtFrontOfBuffer() { 188 ArrayList<LogUnit> logUnits = peekAtFirstNWords(N_GRAM_SIZE); 189 if (isSafeNGram(logUnits, N_GRAM_SIZE)) { 190 // Good n-gram at the front of the buffer. Publish it, disclosing details. 191 publish(logUnits, true /* canIncludePrivateData */); 192 shiftOutWords(N_GRAM_SIZE); 193 resetWordCounter(); 194 } else { 195 // No good n-gram at front, and buffer is full. Shift out the first word (or if there 196 // is none, the existing logUnits). 197 logUnits = peekAtFirstNWords(1); 198 publish(logUnits, false /* canIncludePrivateData */); 199 shiftOutWords(1); 200 } 201 } 202 203 /** 204 * Called when a list of logUnits should be published. 205 * 206 * It is the subclass's responsibility to implement the publication. 207 * 208 * @param logUnits The list of logUnits to be published. 209 * @param canIncludePrivateData Whether the private data in the logUnits can be included in 210 * publication. 211 */ 212 protected abstract void publish(final ArrayList<LogUnit> logUnits, 213 final boolean canIncludePrivateData); 214 215 @Override 216 protected void shiftOutWords(final int numWords) { 217 final int oldNumActualWords = getNumActualWords(); 218 super.shiftOutWords(numWords); 219 final int numWordsShifted = oldNumActualWords - getNumActualWords(); 220 mNumWordsUntilSafeToSample -= numWordsShifted; 221 if (DEBUG) { 222 Log.d(TAG, "wordsUntilSafeToSample now at " + mNumWordsUntilSafeToSample); 223 } 224 } 225} 226