AlwaysOnHotwordDetector.java revision 8ecaf5f5cfd18e0436db1a27ccf46a063e9aacd7
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.service.voice; 18 19import android.content.Intent; 20import android.hardware.soundtrigger.Keyphrase; 21import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 22import android.hardware.soundtrigger.KeyphraseMetadata; 23import android.hardware.soundtrigger.KeyphraseSoundModel; 24import android.hardware.soundtrigger.SoundTrigger; 25import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 26import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 27import android.hardware.soundtrigger.SoundTriggerHelper; 28import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 29import android.os.RemoteException; 30import android.util.Slog; 31 32import com.android.internal.app.IVoiceInteractionManagerService; 33 34import java.util.List; 35 36/** 37 * A class that lets a VoiceInteractionService implementation interact with 38 * always-on keyphrase detection APIs. 39 */ 40public class AlwaysOnHotwordDetector { 41 //---- States of Keyphrase availability ----// 42 /** 43 * Indicates that the given keyphrase is not available on the system because of the 44 * hardware configuration. 45 */ 46 public static final int KEYPHRASE_HARDWARE_UNAVAILABLE = -2; 47 /** 48 * Indicates that the given keyphrase is not supported. 49 */ 50 public static final int KEYPHRASE_UNSUPPORTED = -1; 51 /** 52 * Indicates that the given keyphrase is not enrolled. 53 */ 54 public static final int KEYPHRASE_UNENROLLED = 1; 55 /** 56 * Indicates that the given keyphrase is currently enrolled but not being actively listened for. 57 */ 58 public static final int KEYPHRASE_ENROLLED = 2; 59 60 // Keyphrase management actions ----// 61 /** Indicates that we need to enroll. */ 62 public static final int MANAGE_ACTION_ENROLL = 0; 63 /** Indicates that we need to re-enroll. */ 64 public static final int MANAGE_ACTION_RE_ENROLL = 1; 65 /** Indicates that we need to un-enroll. */ 66 public static final int MANAGE_ACTION_UN_ENROLL = 2; 67 68 /** 69 * Return codes for {@link #startRecognition()}, {@link #stopRecognition()} 70 */ 71 public static final int STATUS_ERROR = Integer.MIN_VALUE; 72 public static final int STATUS_OK = 1; 73 74 //---- Keyphrase recognition status ----// 75 // TODO: Figure out if they are exclusive or should be flags instead? 76 public static final int RECOGNITION_NOT_AVAILABLE = -3; 77 public static final int RECOGNITION_NOT_REQUESTED = -2; 78 public static final int RECOGNITION_DISABLED_TEMPORARILY = -1; 79 public static final int RECOGNITION_REQUESTED = 1; 80 public static final int RECOGNITION_ACTIVE = 2; 81 static final String TAG = "AlwaysOnHotwordDetector"; 82 83 private final String mText; 84 private final String mLocale; 85 /** 86 * The metadata of the Keyphrase, derived from the enrollment application. 87 * This may be null if this keyphrase isn't supported by the enrollment application. 88 */ 89 private final KeyphraseMetadata mKeyphraseMetadata; 90 /** 91 * The sound model for the keyphrase, derived from the model management service 92 * (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet. 93 */ 94 private final KeyphraseSoundModel mEnrolledSoundModel; 95 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 96 private final SoundTriggerHelper mSoundTriggerHelper; 97 private final SoundTriggerHelper.Listener mListener; 98 private final int mAvailability; 99 private final IVoiceInteractionService mVoiceInteractionService; 100 private final IVoiceInteractionManagerService mModelManagementService; 101 102 private int mRecognitionState; 103 104 /** 105 * Callbacks for always-on hotword detection. 106 */ 107 public interface Callback { 108 /** 109 * Called when the keyphrase is spoken. 110 * TODO: Add more data to the callback. 111 */ 112 void onDetected(); 113 /** 114 * Called when the detection for the associated keyphrase starts. 115 */ 116 void onDetectionStarted(); 117 /** 118 * Called when the detection for the associated keyphrase stops. 119 */ 120 void onDetectionStopped(); 121 } 122 123 /** 124 * @param text The keyphrase text to get the detector for. 125 * @param locale The java locale for the detector. 126 * @param callback A non-null Callback for receiving the recognition events. 127 * @param voiceInteractionService The current voice interaction service. 128 * @param modelManagementService A service that allows management of sound models. 129 * 130 * @hide 131 */ 132 public AlwaysOnHotwordDetector(String text, String locale, Callback callback, 133 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 134 SoundTriggerHelper soundTriggerHelper, 135 IVoiceInteractionService voiceInteractionService, 136 IVoiceInteractionManagerService modelManagementService) { 137 mText = text; 138 mLocale = locale; 139 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 140 mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); 141 mListener = new SoundTriggerListener(callback); 142 mSoundTriggerHelper = soundTriggerHelper; 143 mVoiceInteractionService = voiceInteractionService; 144 mModelManagementService = modelManagementService; 145 if (mKeyphraseMetadata != null) { 146 mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id); 147 } else { 148 mEnrolledSoundModel = null; 149 } 150 mAvailability = internalGetAvailability(); 151 } 152 153 /** 154 * Gets the state of always-on hotword detection for the given keyphrase and locale 155 * on this system. 156 * Availability implies that the hardware on this system is capable of listening for 157 * the given keyphrase or not. 158 * 159 * @return Indicates if always-on hotword detection is available for the given keyphrase. 160 * The return code is one of {@link #KEYPHRASE_HARDWARE_UNAVAILABLE}, 161 * {@link #KEYPHRASE_UNSUPPORTED}, {@link #KEYPHRASE_UNENROLLED} or 162 * {@link #KEYPHRASE_ENROLLED}. 163 */ 164 public int getAvailability() { 165 return mAvailability; 166 } 167 168 /** 169 * Gets the status of the recognition. 170 * @return One of {@link #RECOGNITION_NOT_AVAILABLE}, {@link #RECOGNITION_NOT_REQUESTED}, 171 * {@link #RECOGNITION_DISABLED_TEMPORARILY} or {@link #RECOGNITION_ACTIVE}. 172 * @throws UnsupportedOperationException if the recognition isn't supported. 173 * Callers should check the availability by calling {@link #getAvailability()} 174 * before calling this method to avoid this exception. 175 */ 176 public int getRecognitionStatus() { 177 if (mAvailability != KEYPHRASE_ENROLLED) { 178 throw new UnsupportedOperationException( 179 "Recognition for the given keyphrase is not supported"); 180 } 181 182 return mRecognitionState; 183 } 184 185 /** 186 * Starts recognition for the associated keyphrase. 187 * 188 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 189 * @throws UnsupportedOperationException if the recognition isn't supported. 190 * Callers should check the availability by calling {@link #getAvailability()} 191 * before calling this method to avoid this exception. 192 */ 193 public int startRecognition() { 194 if (mAvailability != KEYPHRASE_ENROLLED) { 195 throw new UnsupportedOperationException( 196 "Recognition for the given keyphrase is not supported"); 197 } 198 199 mRecognitionState = RECOGNITION_REQUESTED; 200 mRecognitionState = RECOGNITION_REQUESTED; 201 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 202 // TODO: Do we need to do something about the confidence level here? 203 // TODO: Read the recognition mode flag from the KeyphraseMetadata. 204 // TODO: Take in captureTriggerAudio as a method param here. 205 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, 206 SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, new ConfidenceLevel[0]); 207 int code = mSoundTriggerHelper.startRecognition(mKeyphraseMetadata.id, 208 mEnrolledSoundModel.convertToSoundTriggerKeyphraseSoundModel(), mListener, 209 new RecognitionConfig(false, recognitionExtra, null /* additional data */)); 210 if (code != SoundTriggerHelper.STATUS_OK) { 211 Slog.w(TAG, "startRecognition() failed with error code " + code); 212 return STATUS_ERROR; 213 } else { 214 return STATUS_OK; 215 } 216 } 217 218 /** 219 * Stops recognition for the associated keyphrase. 220 * 221 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 222 * @throws UnsupportedOperationException if the recognition isn't supported. 223 * Callers should check the availability by calling {@link #getAvailability()} 224 * before calling this method to avoid this exception. 225 */ 226 public int stopRecognition() { 227 if (mAvailability != KEYPHRASE_ENROLLED) { 228 throw new UnsupportedOperationException( 229 "Recognition for the given keyphrase is not supported"); 230 } 231 232 mRecognitionState = RECOGNITION_NOT_REQUESTED; 233 int code = mSoundTriggerHelper.stopRecognition(mKeyphraseMetadata.id, mListener); 234 235 if (code != SoundTriggerHelper.STATUS_OK) { 236 Slog.w(TAG, "stopRecognition() failed with error code " + code); 237 return STATUS_ERROR; 238 } else { 239 return STATUS_OK; 240 } 241 } 242 243 /** 244 * Gets an intent to manage the associated keyphrase. 245 * 246 * @param action The manage action that needs to be performed. 247 * One of {@link #MANAGE_ACTION_ENROLL}, {@link #MANAGE_ACTION_RE_ENROLL} or 248 * {@link #MANAGE_ACTION_UN_ENROLL}. 249 * @return An {@link Intent} to manage the given keyphrase. 250 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 251 * Callers should check the availability by calling {@link #getAvailability()} 252 * before calling this method to avoid this exception. 253 */ 254 public Intent getManageIntent(int action) { 255 if (mAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE 256 || mAvailability == KEYPHRASE_UNSUPPORTED) { 257 throw new UnsupportedOperationException( 258 "Managing the given keyphrase is not supported"); 259 } 260 if (action != MANAGE_ACTION_ENROLL 261 && action != MANAGE_ACTION_RE_ENROLL 262 && action != MANAGE_ACTION_UN_ENROLL) { 263 throw new IllegalArgumentException("Invalid action specified " + action); 264 } 265 266 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 267 } 268 269 private int internalGetAvailability() { 270 // No DSP available 271 if (mSoundTriggerHelper.dspInfo == null) { 272 return KEYPHRASE_HARDWARE_UNAVAILABLE; 273 } 274 // No enrollment application supports this keyphrase/locale 275 if (mKeyphraseMetadata == null) { 276 return KEYPHRASE_UNSUPPORTED; 277 } 278 // This keyphrase hasn't been enrolled. 279 if (mEnrolledSoundModel == null) { 280 return KEYPHRASE_UNENROLLED; 281 } 282 return KEYPHRASE_ENROLLED; 283 } 284 285 /** 286 * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. 287 */ 288 private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) { 289 List<KeyphraseSoundModel> soundModels; 290 try { 291 soundModels = mModelManagementService 292 .listRegisteredKeyphraseSoundModels(mVoiceInteractionService); 293 if (soundModels == null || soundModels.isEmpty()) { 294 Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId); 295 return null; 296 } 297 for (KeyphraseSoundModel soundModel : soundModels) { 298 if (soundModel.keyphrases == null) { 299 continue; 300 } 301 for (Keyphrase keyphrase : soundModel.keyphrases) { 302 // TODO: Check the user handle here to only load a model for the current user. 303 if (keyphrase.id == keyphraseId) { 304 return soundModel; 305 } 306 } 307 } 308 } catch (RemoteException e) { 309 Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!"); 310 } 311 return null; 312 } 313 314 /** @hide */ 315 static final class SoundTriggerListener implements SoundTriggerHelper.Listener { 316 private final Callback mCallback; 317 318 public SoundTriggerListener(Callback callback) { 319 this.mCallback = callback; 320 } 321 322 @Override 323 public void onKeyphraseSpoken() { 324 Slog.i(TAG, "onKeyphraseSpoken"); 325 mCallback.onDetected(); 326 } 327 328 @Override 329 public void onListeningStateChanged(int state) { 330 Slog.i(TAG, "onListeningStateChanged: state=" + state); 331 if (state == SoundTriggerHelper.STATE_STARTED) { 332 mCallback.onDetectionStarted(); 333 } else if (state == SoundTriggerHelper.STATE_STOPPED) { 334 mCallback.onDetectionStopped(); 335 } 336 } 337 } 338} 339