SoundTriggerHelper.java revision cb4e81c7fe1ec843d80f7604a688c71086c23685
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.server.voiceinteraction; 18 19import android.hardware.soundtrigger.IRecognitionStatusCallback; 20import android.hardware.soundtrigger.SoundTrigger; 21import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 22import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 23import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 24import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 25import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 26import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 27import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 28import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; 29import android.hardware.soundtrigger.SoundTriggerModule; 30import android.os.RemoteException; 31import android.telephony.PhoneStateListener; 32import android.telephony.TelephonyManager; 33import android.util.Slog; 34 35import java.util.ArrayList; 36import java.util.UUID; 37 38/** 39 * Helper for {@link SoundTrigger} APIs. 40 * Currently this just acts as an abstraction over all SoundTrigger API calls. 41 * 42 * @hide 43 */ 44public class SoundTriggerHelper implements SoundTrigger.StatusListener { 45 static final String TAG = "SoundTriggerHelper"; 46 // TODO: Set to false. 47 static final boolean DBG = true; 48 49 /** 50 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 51 * IRecognitionStatusCallback, RecognitionConfig)}, 52 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 53 */ 54 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 55 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 56 57 private static final int INVALID_VALUE = Integer.MIN_VALUE; 58 59 /** The {@link ModuleProperties} for the system, or null if none exists. */ 60 final ModuleProperties moduleProperties; 61 62 /** The properties for the DSP module */ 63 private final SoundTriggerModule mModule; 64 private final Object mLock = new Object(); 65 private final TelephonyManager mTelephonyManager; 66 private final PhoneStateListener mPhoneStateListener; 67 68 // TODO: Since many layers currently only deal with one recognition 69 // we simplify things by assuming one listener here too. 70 private IRecognitionStatusCallback mActiveListener; 71 private int mKeyphraseId = INVALID_VALUE; 72 private int mCurrentSoundModelHandle = INVALID_VALUE; 73 private UUID mCurrentSoundModelUuid = null; 74 // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. 75 private RecognitionConfig mRecognitionConfig = null; 76 private boolean mRequested = false; 77 private boolean mCallActive = false; 78 // Indicates if the native sound trigger service is disabled or not. 79 // This is an indirect indication of the microphone being open in some other application. 80 private boolean mServiceDisabled = false; 81 private boolean mStarted = false; 82 83 SoundTriggerHelper(TelephonyManager telephonyManager) { 84 ArrayList <ModuleProperties> modules = new ArrayList<>(); 85 int status = SoundTrigger.listModules(modules); 86 mTelephonyManager = telephonyManager; 87 mPhoneStateListener = new MyCallStateListener(); 88 if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { 89 Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); 90 moduleProperties = null; 91 mModule = null; 92 } else { 93 // TODO: Figure out how to determine which module corresponds to the DSP hardware. 94 moduleProperties = modules.get(0); 95 mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); 96 } 97 } 98 99 /** 100 * Starts recognition for the given keyphraseId. 101 * 102 * @param keyphraseId The identifier of the keyphrase for which 103 * the recognition is to be started. 104 * @param soundModel The sound model to use for recognition. 105 * @param listener The listener for the recognition events related to the given keyphrase. 106 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 107 */ 108 int startRecognition(int keyphraseId, 109 KeyphraseSoundModel soundModel, 110 IRecognitionStatusCallback listener, 111 RecognitionConfig recognitionConfig) { 112 if (soundModel == null || listener == null || recognitionConfig == null) { 113 return STATUS_ERROR; 114 } 115 116 synchronized (mLock) { 117 if (DBG) { 118 Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId 119 + " soundModel=" + soundModel + ", listener=" + listener.asBinder() 120 + ", recognitionConfig=" + recognitionConfig); 121 Slog.d(TAG, "moduleProperties=" + moduleProperties); 122 Slog.d(TAG, "current listener=" 123 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 124 Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); 125 Slog.d(TAG, "current SoundModel UUID=" 126 + (mCurrentSoundModelUuid == null ? null : mCurrentSoundModelUuid)); 127 } 128 129 if (!mStarted) { 130 // Get the current call state synchronously for the first recognition. 131 mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; 132 // Register for call state changes when the first call to start recognition occurs. 133 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); 134 } 135 136 if (moduleProperties == null || mModule == null) { 137 Slog.w(TAG, "Attempting startRecognition without the capability"); 138 return STATUS_ERROR; 139 } 140 141 if (mCurrentSoundModelHandle != INVALID_VALUE 142 && !soundModel.uuid.equals(mCurrentSoundModelUuid)) { 143 Slog.w(TAG, "Unloading previous sound model"); 144 int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 145 if (status != SoundTrigger.STATUS_OK) { 146 Slog.w(TAG, "unloadSoundModel call failed with " + status); 147 } 148 mCurrentSoundModelHandle = INVALID_VALUE; 149 mCurrentSoundModelUuid = null; 150 mStarted = false; 151 } 152 153 // If the previous recognition was by a different listener, 154 // Notify them that it was stopped. 155 if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { 156 Slog.w(TAG, "Canceling previous recognition"); 157 try { 158 mActiveListener.onError(STATUS_ERROR); 159 } catch (RemoteException e) { 160 Slog.w(TAG, "RemoteException in onDetectionStopped"); 161 } 162 mActiveListener = null; 163 } 164 165 // Load the sound model if the current one is null. 166 int soundModelHandle = mCurrentSoundModelHandle; 167 if (mCurrentSoundModelHandle == INVALID_VALUE 168 || mCurrentSoundModelUuid == null) { 169 int[] handle = new int[] { INVALID_VALUE }; 170 int status = mModule.loadSoundModel(soundModel, handle); 171 if (status != SoundTrigger.STATUS_OK) { 172 Slog.w(TAG, "loadSoundModel call failed with " + status); 173 return status; 174 } 175 if (handle[0] == INVALID_VALUE) { 176 Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); 177 return STATUS_ERROR; 178 } 179 soundModelHandle = handle[0]; 180 } else { 181 if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); 182 } 183 184 // Start the recognition. 185 mRequested = true; 186 mKeyphraseId = keyphraseId; 187 mCurrentSoundModelHandle = soundModelHandle; 188 mCurrentSoundModelUuid = soundModel.uuid; 189 mRecognitionConfig = recognitionConfig; 190 // Register the new listener. This replaces the old one. 191 // There can only be a maximum of one active listener at any given time. 192 mActiveListener = listener; 193 194 return updateRecognitionLocked(false /* don't notify for synchronous calls */); 195 } 196 } 197 198 /** 199 * Stops recognition for the given {@link Keyphrase} if a recognition is 200 * currently active. 201 * 202 * @param keyphraseId The identifier of the keyphrase for which 203 * the recognition is to be stopped. 204 * @param listener The listener for the recognition events related to the given keyphrase. 205 * 206 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 207 */ 208 int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 209 if (listener == null) { 210 return STATUS_ERROR; 211 } 212 213 synchronized (mLock) { 214 if (DBG) { 215 Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId 216 + ", listener=" + listener.asBinder()); 217 Slog.d(TAG, "current listener=" 218 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 219 } 220 221 if (moduleProperties == null || mModule == null) { 222 Slog.w(TAG, "Attempting stopRecognition without the capability"); 223 return STATUS_ERROR; 224 } 225 226 if (mActiveListener == null) { 227 // startRecognition hasn't been called or it failed. 228 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 229 return STATUS_ERROR; 230 } 231 if (mActiveListener.asBinder() != listener.asBinder()) { 232 // We don't allow a different listener to stop the recognition than the one 233 // that started it. 234 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 235 return STATUS_ERROR; 236 } 237 238 // Stop recognition if it's the current one, ignore otherwise. 239 mRequested = false; 240 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 241 if (status != SoundTrigger.STATUS_OK) { 242 return status; 243 } 244 245 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 246 if (status != SoundTrigger.STATUS_OK) { 247 Slog.w(TAG, "unloadSoundModel call failed with " + status); 248 } 249 250 // Clear the internal state once the recognition has been stopped. 251 // Unload sound model call may fail in scenarios, and we'd still want 252 // to reload the sound model. 253 internalClearStateLocked(); 254 return status; 255 } 256 } 257 258 /** 259 * Stops all recognitions active currently and clears the internal state. 260 */ 261 void stopAllRecognitions() { 262 synchronized (mLock) { 263 if (moduleProperties == null || mModule == null) { 264 return; 265 } 266 267 if (mCurrentSoundModelHandle == INVALID_VALUE) { 268 return; 269 } 270 271 mRequested = false; 272 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 273 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 274 if (status != SoundTrigger.STATUS_OK) { 275 Slog.w(TAG, "unloadSoundModel call failed with " + status); 276 } 277 278 internalClearStateLocked(); 279 } 280 } 281 282 //---- SoundTrigger.StatusListener methods 283 @Override 284 public void onRecognition(RecognitionEvent event) { 285 if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { 286 Slog.w(TAG, "Invalid recognition event!"); 287 return; 288 } 289 290 if (DBG) Slog.d(TAG, "onRecognition: " + event); 291 synchronized (mLock) { 292 if (mActiveListener == null) { 293 Slog.w(TAG, "received onRecognition event without any listener for it"); 294 return; 295 } 296 switch (event.status) { 297 // Fire aborts/failures to all listeners since it's not tied to a keyphrase. 298 case SoundTrigger.RECOGNITION_STATUS_ABORT: 299 onRecognitionAbortLocked(); 300 break; 301 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 302 onRecognitionFailureLocked(); 303 break; 304 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 305 onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); 306 break; 307 } 308 } 309 } 310 311 @Override 312 public void onSoundModelUpdate(SoundModelEvent event) { 313 if (event == null) { 314 Slog.w(TAG, "Invalid sound model event!"); 315 return; 316 } 317 if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); 318 synchronized (mLock) { 319 onSoundModelUpdatedLocked(event); 320 } 321 } 322 323 @Override 324 public void onServiceStateChange(int state) { 325 if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); 326 synchronized (mLock) { 327 onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); 328 } 329 } 330 331 @Override 332 public void onServiceDied() { 333 Slog.e(TAG, "onServiceDied!!"); 334 synchronized (mLock) { 335 onServiceDiedLocked(); 336 } 337 } 338 339 class MyCallStateListener extends PhoneStateListener { 340 @Override 341 public void onCallStateChanged(int state, String arg1) { 342 if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); 343 synchronized (mLock) { 344 onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); 345 } 346 } 347 } 348 349 private void onCallStateChangedLocked(boolean callActive) { 350 if (mCallActive == callActive) { 351 // We consider multiple call states as being active 352 // so we check if something really changed or not here. 353 return; 354 } 355 mCallActive = callActive; 356 updateRecognitionLocked(true /* notify */); 357 } 358 359 private void onSoundModelUpdatedLocked(SoundModelEvent event) { 360 // TODO: Handle sound model update here. 361 } 362 363 private void onServiceStateChangedLocked(boolean disabled) { 364 if (disabled == mServiceDisabled) { 365 return; 366 } 367 mServiceDisabled = disabled; 368 updateRecognitionLocked(true /* notify */); 369 } 370 371 private void onRecognitionAbortLocked() { 372 Slog.w(TAG, "Recognition aborted"); 373 // No-op 374 // This is handled via service state changes instead. 375 } 376 377 private void onRecognitionFailureLocked() { 378 Slog.w(TAG, "Recognition failure"); 379 try { 380 if (mActiveListener != null) { 381 mActiveListener.onError(STATUS_ERROR); 382 } 383 } catch (RemoteException e) { 384 Slog.w(TAG, "RemoteException in onError", e); 385 } finally { 386 internalClearStateLocked(); 387 } 388 } 389 390 private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { 391 Slog.i(TAG, "Recognition success"); 392 KeyphraseRecognitionExtra[] keyphraseExtras = 393 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 394 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 395 Slog.w(TAG, "Invalid keyphrase recognition event!"); 396 return; 397 } 398 // TODO: Handle more than one keyphrase extras. 399 if (mKeyphraseId != keyphraseExtras[0].id) { 400 Slog.w(TAG, "received onRecognition event for a different keyphrase"); 401 return; 402 } 403 404 try { 405 if (mActiveListener != null) { 406 mActiveListener.onDetected((KeyphraseRecognitionEvent) event); 407 } 408 } catch (RemoteException e) { 409 Slog.w(TAG, "RemoteException in onDetected", e); 410 } 411 412 mStarted = false; 413 mRequested = mRecognitionConfig.allowMultipleTriggers; 414 // TODO: Remove this block if the lower layer supports multiple triggers. 415 if (mRequested) { 416 updateRecognitionLocked(true /* notify */); 417 } 418 } 419 420 private void onServiceDiedLocked() { 421 try { 422 if (mActiveListener != null) { 423 mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); 424 } 425 } catch (RemoteException e) { 426 Slog.w(TAG, "RemoteException in onError", e); 427 } finally { 428 internalClearStateLocked(); 429 } 430 } 431 432 private int updateRecognitionLocked(boolean notify) { 433 if (mModule == null || moduleProperties == null 434 || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { 435 // Nothing to do here. 436 return STATUS_OK; 437 } 438 439 boolean start = mRequested && !mCallActive && !mServiceDisabled; 440 if (start == mStarted) { 441 // No-op. 442 return STATUS_OK; 443 } 444 445 // See if the recognition needs to be started. 446 if (start) { 447 // Start recognition. 448 int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); 449 if (status != SoundTrigger.STATUS_OK) { 450 Slog.w(TAG, "startRecognition failed with " + status); 451 // Notify of error if needed. 452 if (notify) { 453 try { 454 mActiveListener.onError(status); 455 } catch (RemoteException e) { 456 Slog.w(TAG, "RemoteException in onError", e); 457 } 458 } 459 } else { 460 mStarted = true; 461 // Notify of resume if needed. 462 if (notify) { 463 try { 464 mActiveListener.onRecognitionResumed(); 465 } catch (RemoteException e) { 466 Slog.w(TAG, "RemoteException in onRecognitionResumed", e); 467 } 468 } 469 } 470 return status; 471 } else { 472 // Stop recognition. 473 int status = mModule.stopRecognition(mCurrentSoundModelHandle); 474 if (status != SoundTrigger.STATUS_OK) { 475 Slog.w(TAG, "stopRecognition call failed with " + status); 476 if (notify) { 477 try { 478 mActiveListener.onError(status); 479 } catch (RemoteException e) { 480 Slog.w(TAG, "RemoteException in onError", e); 481 } 482 } 483 } else { 484 mStarted = false; 485 // Notify of pause if needed. 486 if (notify) { 487 try { 488 mActiveListener.onRecognitionPaused(); 489 } catch (RemoteException e) { 490 Slog.w(TAG, "RemoteException in onRecognitionPaused", e); 491 } 492 } 493 } 494 return status; 495 } 496 } 497 498 private void internalClearStateLocked() { 499 mStarted = false; 500 mRequested = false; 501 502 mKeyphraseId = INVALID_VALUE; 503 mCurrentSoundModelHandle = INVALID_VALUE; 504 mCurrentSoundModelUuid = null; 505 mRecognitionConfig = null; 506 mActiveListener = null; 507 508 // Unregister from call state changes. 509 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); 510 } 511} 512