SoundTriggerHelper.java revision 8cf8f71644643601fe8c3e9538fd00412b1ae8b1
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.server.voiceinteraction; 18 19import android.content.BroadcastReceiver; 20import android.content.Context; 21import android.content.Intent; 22import android.content.IntentFilter; 23import android.hardware.soundtrigger.IRecognitionStatusCallback; 24import android.hardware.soundtrigger.SoundTrigger; 25import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 26import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 27import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 28import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 29import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 30import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 31import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 32import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; 33import android.hardware.soundtrigger.SoundTriggerModule; 34import android.os.PowerManager; 35import android.os.RemoteException; 36import android.telephony.PhoneStateListener; 37import android.telephony.TelephonyManager; 38import android.util.Slog; 39 40import java.io.FileDescriptor; 41import java.io.PrintWriter; 42import java.util.ArrayList; 43import java.util.UUID; 44 45/** 46 * Helper for {@link SoundTrigger} APIs. 47 * Currently this just acts as an abstraction over all SoundTrigger API calls. 48 * 49 * @hide 50 */ 51public class SoundTriggerHelper implements SoundTrigger.StatusListener { 52 static final String TAG = "SoundTriggerHelper"; 53 static final boolean DBG = false; 54 55 /** 56 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 57 * IRecognitionStatusCallback, RecognitionConfig)}, 58 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 59 */ 60 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 61 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 62 63 private static final int INVALID_VALUE = Integer.MIN_VALUE; 64 65 /** The {@link ModuleProperties} for the system, or null if none exists. */ 66 final ModuleProperties moduleProperties; 67 68 /** The properties for the DSP module */ 69 private SoundTriggerModule mModule; 70 private final Object mLock = new Object(); 71 private final Context mContext; 72 private final TelephonyManager mTelephonyManager; 73 private final PhoneStateListener mPhoneStateListener; 74 private final PowerManager mPowerManager; 75 76 // TODO: Since many layers currently only deal with one recognition 77 // we simplify things by assuming one listener here too. 78 private IRecognitionStatusCallback mActiveListener; 79 private int mKeyphraseId = INVALID_VALUE; 80 private int mCurrentSoundModelHandle = INVALID_VALUE; 81 private UUID mCurrentSoundModelUuid = null; 82 // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. 83 private RecognitionConfig mRecognitionConfig = null; 84 private boolean mRequested = false; 85 private boolean mCallActive = false; 86 private boolean mIsPowerSaveMode = false; 87 // Indicates if the native sound trigger service is disabled or not. 88 // This is an indirect indication of the microphone being open in some other application. 89 private boolean mServiceDisabled = false; 90 private boolean mStarted = false; 91 private PowerSaveModeListener mPowerSaveModeListener; 92 93 SoundTriggerHelper(Context context) { 94 ArrayList <ModuleProperties> modules = new ArrayList<>(); 95 int status = SoundTrigger.listModules(modules); 96 mContext = context; 97 mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); 98 mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); 99 mPhoneStateListener = new MyCallStateListener(); 100 if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { 101 Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); 102 moduleProperties = null; 103 mModule = null; 104 } else { 105 // TODO: Figure out how to determine which module corresponds to the DSP hardware. 106 moduleProperties = modules.get(0); 107 } 108 } 109 110 /** 111 * Starts recognition for the given keyphraseId. 112 * 113 * @param keyphraseId The identifier of the keyphrase for which 114 * the recognition is to be started. 115 * @param soundModel The sound model to use for recognition. 116 * @param listener The listener for the recognition events related to the given keyphrase. 117 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 118 */ 119 int startRecognition(int keyphraseId, 120 KeyphraseSoundModel soundModel, 121 IRecognitionStatusCallback listener, 122 RecognitionConfig recognitionConfig) { 123 if (soundModel == null || listener == null || recognitionConfig == null) { 124 return STATUS_ERROR; 125 } 126 127 synchronized (mLock) { 128 if (DBG) { 129 Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId 130 + " soundModel=" + soundModel + ", listener=" + listener.asBinder() 131 + ", recognitionConfig=" + recognitionConfig); 132 Slog.d(TAG, "moduleProperties=" + moduleProperties); 133 Slog.d(TAG, "current listener=" 134 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 135 Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); 136 Slog.d(TAG, "current SoundModel UUID=" 137 + (mCurrentSoundModelUuid == null ? null : mCurrentSoundModelUuid)); 138 } 139 140 if (!mStarted) { 141 // Get the current call state synchronously for the first recognition. 142 mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; 143 // Register for call state changes when the first call to start recognition occurs. 144 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); 145 146 // Register for power saver mode changes when the first call to start recognition 147 // occurs. 148 if (mPowerSaveModeListener == null) { 149 mPowerSaveModeListener = new PowerSaveModeListener(); 150 mContext.registerReceiver(mPowerSaveModeListener, 151 new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); 152 } 153 mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); 154 } 155 156 if (moduleProperties == null) { 157 Slog.w(TAG, "Attempting startRecognition without the capability"); 158 return STATUS_ERROR; 159 } 160 if (mModule == null) { 161 mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); 162 if (mModule == null) { 163 Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); 164 return STATUS_ERROR; 165 } 166 } 167 168 // Unload the previous model if the current one isn't invalid 169 // and, it's not the same as the new one, or we are already started 170 // if we are already started, we can get multiple calls to start 171 // if the underlying sound model changes, in which case we should unload and reload. 172 // The model reuse helps only in cases when we trigger and stop internally 173 // without a start recognition call. 174 if (mCurrentSoundModelHandle != INVALID_VALUE 175 && (!soundModel.uuid.equals(mCurrentSoundModelUuid) || mStarted)) { 176 Slog.w(TAG, "Unloading previous sound model"); 177 int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 178 if (status != SoundTrigger.STATUS_OK) { 179 Slog.w(TAG, "unloadSoundModel call failed with " + status); 180 } 181 mCurrentSoundModelHandle = INVALID_VALUE; 182 mCurrentSoundModelUuid = null; 183 mStarted = false; 184 } 185 186 // If the previous recognition was by a different listener, 187 // Notify them that it was stopped. 188 if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { 189 Slog.w(TAG, "Canceling previous recognition"); 190 try { 191 mActiveListener.onError(STATUS_ERROR); 192 } catch (RemoteException e) { 193 Slog.w(TAG, "RemoteException in onDetectionStopped", e); 194 } 195 mActiveListener = null; 196 } 197 198 // Load the sound model if the current one is null. 199 int soundModelHandle = mCurrentSoundModelHandle; 200 if (mCurrentSoundModelHandle == INVALID_VALUE 201 || mCurrentSoundModelUuid == null) { 202 int[] handle = new int[] { INVALID_VALUE }; 203 int status = mModule.loadSoundModel(soundModel, handle); 204 if (status != SoundTrigger.STATUS_OK) { 205 Slog.w(TAG, "loadSoundModel call failed with " + status); 206 return status; 207 } 208 if (handle[0] == INVALID_VALUE) { 209 Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); 210 return STATUS_ERROR; 211 } 212 soundModelHandle = handle[0]; 213 } else { 214 if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); 215 } 216 217 // Start the recognition. 218 mRequested = true; 219 mKeyphraseId = keyphraseId; 220 mCurrentSoundModelHandle = soundModelHandle; 221 mCurrentSoundModelUuid = soundModel.uuid; 222 mRecognitionConfig = recognitionConfig; 223 // Register the new listener. This replaces the old one. 224 // There can only be a maximum of one active listener at any given time. 225 mActiveListener = listener; 226 227 return updateRecognitionLocked(false /* don't notify for synchronous calls */); 228 } 229 } 230 231 /** 232 * Stops recognition for the given {@link Keyphrase} if a recognition is 233 * currently active. 234 * 235 * @param keyphraseId The identifier of the keyphrase for which 236 * the recognition is to be stopped. 237 * @param listener The listener for the recognition events related to the given keyphrase. 238 * 239 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 240 */ 241 int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 242 if (listener == null) { 243 return STATUS_ERROR; 244 } 245 246 synchronized (mLock) { 247 if (DBG) { 248 Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId 249 + ", listener=" + listener.asBinder()); 250 Slog.d(TAG, "current listener=" 251 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 252 } 253 254 if (moduleProperties == null || mModule == null) { 255 Slog.w(TAG, "Attempting stopRecognition without the capability"); 256 return STATUS_ERROR; 257 } 258 259 if (mActiveListener == null) { 260 // startRecognition hasn't been called or it failed. 261 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 262 return STATUS_ERROR; 263 } 264 if (mActiveListener.asBinder() != listener.asBinder()) { 265 // We don't allow a different listener to stop the recognition than the one 266 // that started it. 267 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 268 return STATUS_ERROR; 269 } 270 271 // Stop recognition if it's the current one, ignore otherwise. 272 mRequested = false; 273 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 274 if (status != SoundTrigger.STATUS_OK) { 275 return status; 276 } 277 278 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 279 if (status != SoundTrigger.STATUS_OK) { 280 Slog.w(TAG, "unloadSoundModel call failed with " + status); 281 } 282 283 // Clear the internal state once the recognition has been stopped. 284 // Unload sound model call may fail in scenarios, and we'd still want 285 // to reload the sound model. 286 internalClearStateLocked(); 287 return status; 288 } 289 } 290 291 /** 292 * Stops all recognitions active currently and clears the internal state. 293 */ 294 void stopAllRecognitions() { 295 synchronized (mLock) { 296 if (moduleProperties == null || mModule == null) { 297 return; 298 } 299 300 if (mCurrentSoundModelHandle == INVALID_VALUE) { 301 return; 302 } 303 304 mRequested = false; 305 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 306 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 307 if (status != SoundTrigger.STATUS_OK) { 308 Slog.w(TAG, "unloadSoundModel call failed with " + status); 309 } 310 311 internalClearStateLocked(); 312 } 313 } 314 315 //---- SoundTrigger.StatusListener methods 316 @Override 317 public void onRecognition(RecognitionEvent event) { 318 if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { 319 Slog.w(TAG, "Invalid recognition event!"); 320 return; 321 } 322 323 if (DBG) Slog.d(TAG, "onRecognition: " + event); 324 synchronized (mLock) { 325 if (mActiveListener == null) { 326 Slog.w(TAG, "received onRecognition event without any listener for it"); 327 return; 328 } 329 switch (event.status) { 330 // Fire aborts/failures to all listeners since it's not tied to a keyphrase. 331 case SoundTrigger.RECOGNITION_STATUS_ABORT: 332 onRecognitionAbortLocked(); 333 break; 334 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 335 onRecognitionFailureLocked(); 336 break; 337 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 338 onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); 339 break; 340 } 341 } 342 } 343 344 @Override 345 public void onSoundModelUpdate(SoundModelEvent event) { 346 if (event == null) { 347 Slog.w(TAG, "Invalid sound model event!"); 348 return; 349 } 350 if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); 351 synchronized (mLock) { 352 onSoundModelUpdatedLocked(event); 353 } 354 } 355 356 @Override 357 public void onServiceStateChange(int state) { 358 if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); 359 synchronized (mLock) { 360 onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); 361 } 362 } 363 364 @Override 365 public void onServiceDied() { 366 Slog.e(TAG, "onServiceDied!!"); 367 synchronized (mLock) { 368 onServiceDiedLocked(); 369 } 370 } 371 372 private void onCallStateChangedLocked(boolean callActive) { 373 if (mCallActive == callActive) { 374 // We consider multiple call states as being active 375 // so we check if something really changed or not here. 376 return; 377 } 378 mCallActive = callActive; 379 updateRecognitionLocked(true /* notify */); 380 } 381 382 private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) { 383 if (mIsPowerSaveMode == isPowerSaveMode) { 384 return; 385 } 386 mIsPowerSaveMode = isPowerSaveMode; 387 updateRecognitionLocked(true /* notify */); 388 } 389 390 private void onSoundModelUpdatedLocked(SoundModelEvent event) { 391 // TODO: Handle sound model update here. 392 } 393 394 private void onServiceStateChangedLocked(boolean disabled) { 395 if (disabled == mServiceDisabled) { 396 return; 397 } 398 mServiceDisabled = disabled; 399 updateRecognitionLocked(true /* notify */); 400 } 401 402 private void onRecognitionAbortLocked() { 403 Slog.w(TAG, "Recognition aborted"); 404 // No-op 405 // This is handled via service state changes instead. 406 } 407 408 private void onRecognitionFailureLocked() { 409 Slog.w(TAG, "Recognition failure"); 410 try { 411 if (mActiveListener != null) { 412 mActiveListener.onError(STATUS_ERROR); 413 } 414 } catch (RemoteException e) { 415 Slog.w(TAG, "RemoteException in onError", e); 416 } finally { 417 internalClearStateLocked(); 418 } 419 } 420 421 private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { 422 Slog.i(TAG, "Recognition success"); 423 KeyphraseRecognitionExtra[] keyphraseExtras = 424 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 425 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 426 Slog.w(TAG, "Invalid keyphrase recognition event!"); 427 return; 428 } 429 // TODO: Handle more than one keyphrase extras. 430 if (mKeyphraseId != keyphraseExtras[0].id) { 431 Slog.w(TAG, "received onRecognition event for a different keyphrase"); 432 return; 433 } 434 435 try { 436 if (mActiveListener != null) { 437 mActiveListener.onDetected((KeyphraseRecognitionEvent) event); 438 } 439 } catch (RemoteException e) { 440 Slog.w(TAG, "RemoteException in onDetected", e); 441 } 442 443 mStarted = false; 444 mRequested = mRecognitionConfig.allowMultipleTriggers; 445 // TODO: Remove this block if the lower layer supports multiple triggers. 446 if (mRequested) { 447 updateRecognitionLocked(true /* notify */); 448 } 449 } 450 451 private void onServiceDiedLocked() { 452 try { 453 if (mActiveListener != null) { 454 mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); 455 } 456 } catch (RemoteException e) { 457 Slog.w(TAG, "RemoteException in onError", e); 458 } finally { 459 internalClearStateLocked(); 460 if (mModule != null) { 461 mModule.detach(); 462 mModule = null; 463 } 464 } 465 } 466 467 private int updateRecognitionLocked(boolean notify) { 468 if (mModule == null || moduleProperties == null 469 || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { 470 // Nothing to do here. 471 return STATUS_OK; 472 } 473 474 boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; 475 if (start == mStarted) { 476 // No-op. 477 return STATUS_OK; 478 } 479 480 // See if the recognition needs to be started. 481 if (start) { 482 // Start recognition. 483 int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); 484 if (status != SoundTrigger.STATUS_OK) { 485 Slog.w(TAG, "startRecognition failed with " + status); 486 // Notify of error if needed. 487 if (notify) { 488 try { 489 mActiveListener.onError(status); 490 } catch (RemoteException e) { 491 Slog.w(TAG, "RemoteException in onError", e); 492 } 493 } 494 } else { 495 mStarted = true; 496 // Notify of resume if needed. 497 if (notify) { 498 try { 499 mActiveListener.onRecognitionResumed(); 500 } catch (RemoteException e) { 501 Slog.w(TAG, "RemoteException in onRecognitionResumed", e); 502 } 503 } 504 } 505 return status; 506 } else { 507 // Stop recognition. 508 int status = mModule.stopRecognition(mCurrentSoundModelHandle); 509 if (status != SoundTrigger.STATUS_OK) { 510 Slog.w(TAG, "stopRecognition call failed with " + status); 511 if (notify) { 512 try { 513 mActiveListener.onError(status); 514 } catch (RemoteException e) { 515 Slog.w(TAG, "RemoteException in onError", e); 516 } 517 } 518 } else { 519 mStarted = false; 520 // Notify of pause if needed. 521 if (notify) { 522 try { 523 mActiveListener.onRecognitionPaused(); 524 } catch (RemoteException e) { 525 Slog.w(TAG, "RemoteException in onRecognitionPaused", e); 526 } 527 } 528 } 529 return status; 530 } 531 } 532 533 private void internalClearStateLocked() { 534 mStarted = false; 535 mRequested = false; 536 537 mKeyphraseId = INVALID_VALUE; 538 mCurrentSoundModelHandle = INVALID_VALUE; 539 mCurrentSoundModelUuid = null; 540 mRecognitionConfig = null; 541 mActiveListener = null; 542 543 // Unregister from call state changes. 544 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); 545 546 // Unregister from power save mode changes. 547 if (mPowerSaveModeListener != null) { 548 mContext.unregisterReceiver(mPowerSaveModeListener); 549 mPowerSaveModeListener = null; 550 } 551 } 552 553 class MyCallStateListener extends PhoneStateListener { 554 @Override 555 public void onCallStateChanged(int state, String arg1) { 556 if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); 557 synchronized (mLock) { 558 onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); 559 } 560 } 561 } 562 563 class PowerSaveModeListener extends BroadcastReceiver { 564 @Override 565 public void onReceive(Context context, Intent intent) { 566 if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) { 567 return; 568 } 569 boolean active = mPowerManager.isPowerSaveMode(); 570 if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active); 571 synchronized (mLock) { 572 onPowerSaveModeChangedLocked(active); 573 } 574 } 575 } 576 577 void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 578 synchronized (mLock) { 579 pw.print(" module properties="); 580 pw.println(moduleProperties == null ? "null" : moduleProperties); 581 pw.print(" keyphrase ID="); pw.println(mKeyphraseId); 582 pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); 583 pw.print(" sound model UUID="); 584 pw.println(mCurrentSoundModelUuid == null ? "null" : mCurrentSoundModelUuid); 585 pw.print(" current listener="); 586 pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); 587 588 pw.print(" requested="); pw.println(mRequested); 589 pw.print(" started="); pw.println(mStarted); 590 pw.print(" call active="); pw.println(mCallActive); 591 pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); 592 pw.print(" service disabled="); pw.println(mServiceDisabled); 593 } 594 } 595} 596