SoundTriggerHelper.java revision 2475e38c10a02973665752e0b829153a5c493b28
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.server.voiceinteraction; 18 19import android.content.BroadcastReceiver; 20import android.content.Context; 21import android.content.Intent; 22import android.content.IntentFilter; 23import android.hardware.soundtrigger.IRecognitionStatusCallback; 24import android.hardware.soundtrigger.SoundTrigger; 25import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 26import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 27import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 28import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 29import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 30import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 31import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 32import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; 33import android.hardware.soundtrigger.SoundTriggerModule; 34import android.os.PowerManager; 35import android.os.RemoteException; 36import android.telephony.PhoneStateListener; 37import android.telephony.TelephonyManager; 38import android.util.Slog; 39 40import java.io.FileDescriptor; 41import java.io.PrintWriter; 42import java.util.ArrayList; 43import java.util.UUID; 44 45/** 46 * Helper for {@link SoundTrigger} APIs. 47 * Currently this just acts as an abstraction over all SoundTrigger API calls. 48 * 49 * @hide 50 */ 51public class SoundTriggerHelper implements SoundTrigger.StatusListener { 52 static final String TAG = "SoundTriggerHelper"; 53 // TODO: Set to false. 54 static final boolean DBG = true; 55 56 /** 57 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 58 * IRecognitionStatusCallback, RecognitionConfig)}, 59 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 60 */ 61 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 62 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 63 64 private static final int INVALID_VALUE = Integer.MIN_VALUE; 65 66 /** The {@link ModuleProperties} for the system, or null if none exists. */ 67 final ModuleProperties moduleProperties; 68 69 /** The properties for the DSP module */ 70 private SoundTriggerModule mModule; 71 private final Object mLock = new Object(); 72 private final Context mContext; 73 private final TelephonyManager mTelephonyManager; 74 private final PhoneStateListener mPhoneStateListener; 75 private final PowerManager mPowerManager; 76 77 // TODO: Since many layers currently only deal with one recognition 78 // we simplify things by assuming one listener here too. 79 private IRecognitionStatusCallback mActiveListener; 80 private int mKeyphraseId = INVALID_VALUE; 81 private int mCurrentSoundModelHandle = INVALID_VALUE; 82 private UUID mCurrentSoundModelUuid = null; 83 // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. 84 private RecognitionConfig mRecognitionConfig = null; 85 private boolean mRequested = false; 86 private boolean mCallActive = false; 87 private boolean mIsPowerSaveMode = false; 88 // Indicates if the native sound trigger service is disabled or not. 89 // This is an indirect indication of the microphone being open in some other application. 90 private boolean mServiceDisabled = false; 91 private boolean mStarted = false; 92 private PowerSaveModeListener mPowerSaveModeListener; 93 94 SoundTriggerHelper(Context context) { 95 ArrayList <ModuleProperties> modules = new ArrayList<>(); 96 int status = SoundTrigger.listModules(modules); 97 mContext = context; 98 mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); 99 mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); 100 mPhoneStateListener = new MyCallStateListener(); 101 if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { 102 Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); 103 moduleProperties = null; 104 mModule = null; 105 } else { 106 // TODO: Figure out how to determine which module corresponds to the DSP hardware. 107 moduleProperties = modules.get(0); 108 } 109 } 110 111 /** 112 * Starts recognition for the given keyphraseId. 113 * 114 * @param keyphraseId The identifier of the keyphrase for which 115 * the recognition is to be started. 116 * @param soundModel The sound model to use for recognition. 117 * @param listener The listener for the recognition events related to the given keyphrase. 118 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 119 */ 120 int startRecognition(int keyphraseId, 121 KeyphraseSoundModel soundModel, 122 IRecognitionStatusCallback listener, 123 RecognitionConfig recognitionConfig) { 124 if (soundModel == null || listener == null || recognitionConfig == null) { 125 return STATUS_ERROR; 126 } 127 128 synchronized (mLock) { 129 if (DBG) { 130 Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId 131 + " soundModel=" + soundModel + ", listener=" + listener.asBinder() 132 + ", recognitionConfig=" + recognitionConfig); 133 Slog.d(TAG, "moduleProperties=" + moduleProperties); 134 Slog.d(TAG, "current listener=" 135 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 136 Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); 137 Slog.d(TAG, "current SoundModel UUID=" 138 + (mCurrentSoundModelUuid == null ? null : mCurrentSoundModelUuid)); 139 } 140 141 if (!mStarted) { 142 // Get the current call state synchronously for the first recognition. 143 mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; 144 // Register for call state changes when the first call to start recognition occurs. 145 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); 146 147 // Register for power saver mode changes when the first call to start recognition 148 // occurs. 149 if (mPowerSaveModeListener == null) { 150 mPowerSaveModeListener = new PowerSaveModeListener(); 151 mContext.registerReceiver(mPowerSaveModeListener, 152 new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); 153 } 154 mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); 155 } 156 157 if (moduleProperties == null) { 158 Slog.w(TAG, "Attempting startRecognition without the capability"); 159 return STATUS_ERROR; 160 } 161 if (mModule == null) { 162 mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); 163 if (mModule == null) { 164 Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); 165 return STATUS_ERROR; 166 } 167 } 168 169 if (mCurrentSoundModelHandle != INVALID_VALUE 170 && !soundModel.uuid.equals(mCurrentSoundModelUuid)) { 171 Slog.w(TAG, "Unloading previous sound model"); 172 int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 173 if (status != SoundTrigger.STATUS_OK) { 174 Slog.w(TAG, "unloadSoundModel call failed with " + status); 175 } 176 mCurrentSoundModelHandle = INVALID_VALUE; 177 mCurrentSoundModelUuid = null; 178 mStarted = false; 179 } 180 181 // If the previous recognition was by a different listener, 182 // Notify them that it was stopped. 183 if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { 184 Slog.w(TAG, "Canceling previous recognition"); 185 try { 186 mActiveListener.onError(STATUS_ERROR); 187 } catch (RemoteException e) { 188 Slog.w(TAG, "RemoteException in onDetectionStopped", e); 189 } 190 mActiveListener = null; 191 } 192 193 // Load the sound model if the current one is null. 194 int soundModelHandle = mCurrentSoundModelHandle; 195 if (mCurrentSoundModelHandle == INVALID_VALUE 196 || mCurrentSoundModelUuid == null) { 197 int[] handle = new int[] { INVALID_VALUE }; 198 int status = mModule.loadSoundModel(soundModel, handle); 199 if (status != SoundTrigger.STATUS_OK) { 200 Slog.w(TAG, "loadSoundModel call failed with " + status); 201 return status; 202 } 203 if (handle[0] == INVALID_VALUE) { 204 Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); 205 return STATUS_ERROR; 206 } 207 soundModelHandle = handle[0]; 208 } else { 209 if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); 210 } 211 212 // Start the recognition. 213 mRequested = true; 214 mKeyphraseId = keyphraseId; 215 mCurrentSoundModelHandle = soundModelHandle; 216 mCurrentSoundModelUuid = soundModel.uuid; 217 mRecognitionConfig = recognitionConfig; 218 // Register the new listener. This replaces the old one. 219 // There can only be a maximum of one active listener at any given time. 220 mActiveListener = listener; 221 222 return updateRecognitionLocked(false /* don't notify for synchronous calls */); 223 } 224 } 225 226 /** 227 * Stops recognition for the given {@link Keyphrase} if a recognition is 228 * currently active. 229 * 230 * @param keyphraseId The identifier of the keyphrase for which 231 * the recognition is to be stopped. 232 * @param listener The listener for the recognition events related to the given keyphrase. 233 * 234 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 235 */ 236 int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 237 if (listener == null) { 238 return STATUS_ERROR; 239 } 240 241 synchronized (mLock) { 242 if (DBG) { 243 Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId 244 + ", listener=" + listener.asBinder()); 245 Slog.d(TAG, "current listener=" 246 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 247 } 248 249 if (moduleProperties == null || mModule == null) { 250 Slog.w(TAG, "Attempting stopRecognition without the capability"); 251 return STATUS_ERROR; 252 } 253 254 if (mActiveListener == null) { 255 // startRecognition hasn't been called or it failed. 256 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 257 return STATUS_ERROR; 258 } 259 if (mActiveListener.asBinder() != listener.asBinder()) { 260 // We don't allow a different listener to stop the recognition than the one 261 // that started it. 262 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 263 return STATUS_ERROR; 264 } 265 266 // Stop recognition if it's the current one, ignore otherwise. 267 mRequested = false; 268 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 269 if (status != SoundTrigger.STATUS_OK) { 270 return status; 271 } 272 273 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 274 if (status != SoundTrigger.STATUS_OK) { 275 Slog.w(TAG, "unloadSoundModel call failed with " + status); 276 } 277 278 // Clear the internal state once the recognition has been stopped. 279 // Unload sound model call may fail in scenarios, and we'd still want 280 // to reload the sound model. 281 internalClearStateLocked(); 282 return status; 283 } 284 } 285 286 /** 287 * Stops all recognitions active currently and clears the internal state. 288 */ 289 void stopAllRecognitions() { 290 synchronized (mLock) { 291 if (moduleProperties == null || mModule == null) { 292 return; 293 } 294 295 if (mCurrentSoundModelHandle == INVALID_VALUE) { 296 return; 297 } 298 299 mRequested = false; 300 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 301 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 302 if (status != SoundTrigger.STATUS_OK) { 303 Slog.w(TAG, "unloadSoundModel call failed with " + status); 304 } 305 306 internalClearStateLocked(); 307 } 308 } 309 310 //---- SoundTrigger.StatusListener methods 311 @Override 312 public void onRecognition(RecognitionEvent event) { 313 if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { 314 Slog.w(TAG, "Invalid recognition event!"); 315 return; 316 } 317 318 if (DBG) Slog.d(TAG, "onRecognition: " + event); 319 synchronized (mLock) { 320 if (mActiveListener == null) { 321 Slog.w(TAG, "received onRecognition event without any listener for it"); 322 return; 323 } 324 switch (event.status) { 325 // Fire aborts/failures to all listeners since it's not tied to a keyphrase. 326 case SoundTrigger.RECOGNITION_STATUS_ABORT: 327 onRecognitionAbortLocked(); 328 break; 329 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 330 onRecognitionFailureLocked(); 331 break; 332 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 333 onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); 334 break; 335 } 336 } 337 } 338 339 @Override 340 public void onSoundModelUpdate(SoundModelEvent event) { 341 if (event == null) { 342 Slog.w(TAG, "Invalid sound model event!"); 343 return; 344 } 345 if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); 346 synchronized (mLock) { 347 onSoundModelUpdatedLocked(event); 348 } 349 } 350 351 @Override 352 public void onServiceStateChange(int state) { 353 if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); 354 synchronized (mLock) { 355 onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); 356 } 357 } 358 359 @Override 360 public void onServiceDied() { 361 Slog.e(TAG, "onServiceDied!!"); 362 synchronized (mLock) { 363 onServiceDiedLocked(); 364 } 365 } 366 367 private void onCallStateChangedLocked(boolean callActive) { 368 if (mCallActive == callActive) { 369 // We consider multiple call states as being active 370 // so we check if something really changed or not here. 371 return; 372 } 373 mCallActive = callActive; 374 updateRecognitionLocked(true /* notify */); 375 } 376 377 private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) { 378 if (mIsPowerSaveMode == isPowerSaveMode) { 379 return; 380 } 381 mIsPowerSaveMode = isPowerSaveMode; 382 updateRecognitionLocked(true /* notify */); 383 } 384 385 private void onSoundModelUpdatedLocked(SoundModelEvent event) { 386 // TODO: Handle sound model update here. 387 } 388 389 private void onServiceStateChangedLocked(boolean disabled) { 390 if (disabled == mServiceDisabled) { 391 return; 392 } 393 mServiceDisabled = disabled; 394 updateRecognitionLocked(true /* notify */); 395 } 396 397 private void onRecognitionAbortLocked() { 398 Slog.w(TAG, "Recognition aborted"); 399 // No-op 400 // This is handled via service state changes instead. 401 } 402 403 private void onRecognitionFailureLocked() { 404 Slog.w(TAG, "Recognition failure"); 405 try { 406 if (mActiveListener != null) { 407 mActiveListener.onError(STATUS_ERROR); 408 } 409 } catch (RemoteException e) { 410 Slog.w(TAG, "RemoteException in onError", e); 411 } finally { 412 internalClearStateLocked(); 413 } 414 } 415 416 private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { 417 Slog.i(TAG, "Recognition success"); 418 KeyphraseRecognitionExtra[] keyphraseExtras = 419 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 420 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 421 Slog.w(TAG, "Invalid keyphrase recognition event!"); 422 return; 423 } 424 // TODO: Handle more than one keyphrase extras. 425 if (mKeyphraseId != keyphraseExtras[0].id) { 426 Slog.w(TAG, "received onRecognition event for a different keyphrase"); 427 return; 428 } 429 430 try { 431 if (mActiveListener != null) { 432 mActiveListener.onDetected((KeyphraseRecognitionEvent) event); 433 } 434 } catch (RemoteException e) { 435 Slog.w(TAG, "RemoteException in onDetected", e); 436 } 437 438 mStarted = false; 439 mRequested = mRecognitionConfig.allowMultipleTriggers; 440 // TODO: Remove this block if the lower layer supports multiple triggers. 441 if (mRequested) { 442 updateRecognitionLocked(true /* notify */); 443 } 444 } 445 446 private void onServiceDiedLocked() { 447 try { 448 if (mActiveListener != null) { 449 mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); 450 } 451 } catch (RemoteException e) { 452 Slog.w(TAG, "RemoteException in onError", e); 453 } finally { 454 internalClearStateLocked(); 455 if (mModule != null) { 456 mModule.detach(); 457 mModule = null; 458 } 459 } 460 } 461 462 private int updateRecognitionLocked(boolean notify) { 463 if (mModule == null || moduleProperties == null 464 || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { 465 // Nothing to do here. 466 return STATUS_OK; 467 } 468 469 boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; 470 if (start == mStarted) { 471 // No-op. 472 return STATUS_OK; 473 } 474 475 // See if the recognition needs to be started. 476 if (start) { 477 // Start recognition. 478 int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); 479 if (status != SoundTrigger.STATUS_OK) { 480 Slog.w(TAG, "startRecognition failed with " + status); 481 // Notify of error if needed. 482 if (notify) { 483 try { 484 mActiveListener.onError(status); 485 } catch (RemoteException e) { 486 Slog.w(TAG, "RemoteException in onError", e); 487 } 488 } 489 } else { 490 mStarted = true; 491 // Notify of resume if needed. 492 if (notify) { 493 try { 494 mActiveListener.onRecognitionResumed(); 495 } catch (RemoteException e) { 496 Slog.w(TAG, "RemoteException in onRecognitionResumed", e); 497 } 498 } 499 } 500 return status; 501 } else { 502 // Stop recognition. 503 int status = mModule.stopRecognition(mCurrentSoundModelHandle); 504 if (status != SoundTrigger.STATUS_OK) { 505 Slog.w(TAG, "stopRecognition call failed with " + status); 506 if (notify) { 507 try { 508 mActiveListener.onError(status); 509 } catch (RemoteException e) { 510 Slog.w(TAG, "RemoteException in onError", e); 511 } 512 } 513 } else { 514 mStarted = false; 515 // Notify of pause if needed. 516 if (notify) { 517 try { 518 mActiveListener.onRecognitionPaused(); 519 } catch (RemoteException e) { 520 Slog.w(TAG, "RemoteException in onRecognitionPaused", e); 521 } 522 } 523 } 524 return status; 525 } 526 } 527 528 private void internalClearStateLocked() { 529 mStarted = false; 530 mRequested = false; 531 532 mKeyphraseId = INVALID_VALUE; 533 mCurrentSoundModelHandle = INVALID_VALUE; 534 mCurrentSoundModelUuid = null; 535 mRecognitionConfig = null; 536 mActiveListener = null; 537 538 // Unregister from call state changes. 539 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); 540 541 // Unregister from power save mode changes. 542 if (mPowerSaveModeListener != null) { 543 mContext.unregisterReceiver(mPowerSaveModeListener); 544 mPowerSaveModeListener = null; 545 } 546 } 547 548 class MyCallStateListener extends PhoneStateListener { 549 @Override 550 public void onCallStateChanged(int state, String arg1) { 551 if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); 552 synchronized (mLock) { 553 onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); 554 } 555 } 556 } 557 558 class PowerSaveModeListener extends BroadcastReceiver { 559 @Override 560 public void onReceive(Context context, Intent intent) { 561 if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) { 562 return; 563 } 564 boolean active = mPowerManager.isPowerSaveMode(); 565 if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active); 566 synchronized (mLock) { 567 onPowerSaveModeChangedLocked(active); 568 } 569 } 570 } 571 572 void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 573 synchronized (mLock) { 574 pw.print(" module properties="); 575 pw.println(moduleProperties == null ? "null" : moduleProperties); 576 pw.print(" keyphrase ID="); pw.println(mKeyphraseId); 577 pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); 578 pw.print(" sound model UUID="); 579 pw.println(mCurrentSoundModelUuid == null ? "null" : mCurrentSoundModelUuid); 580 pw.print(" current listener="); 581 pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); 582 583 pw.print(" requested="); pw.println(mRequested); 584 pw.print(" started="); pw.println(mStarted); 585 pw.print(" call active="); pw.println(mCallActive); 586 pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); 587 pw.print(" service disabled="); pw.println(mServiceDisabled); 588 } 589 } 590} 591