SoundTriggerHelper.java revision efe0f9c7f2bdc10cfd28c186e68676e27b6944a1
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.server.voiceinteraction; 18 19import android.content.BroadcastReceiver; 20import android.content.Context; 21import android.content.Intent; 22import android.content.IntentFilter; 23import android.hardware.soundtrigger.IRecognitionStatusCallback; 24import android.hardware.soundtrigger.SoundTrigger; 25import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 26import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 27import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 28import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 29import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 30import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 31import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 32import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; 33import android.hardware.soundtrigger.SoundTriggerModule; 34import android.os.PowerManager; 35import android.os.RemoteException; 36import android.telephony.PhoneStateListener; 37import android.telephony.TelephonyManager; 38import android.util.Slog; 39 40import java.io.FileDescriptor; 41import java.io.PrintWriter; 42import java.util.ArrayList; 43import java.util.UUID; 44 45/** 46 * Helper for {@link SoundTrigger} APIs. 47 * Currently this just acts as an abstraction over all SoundTrigger API calls. 48 * 49 * @hide 50 */ 51public class SoundTriggerHelper implements SoundTrigger.StatusListener { 52 static final String TAG = "SoundTriggerHelper"; 53 // TODO: Set to false. 54 static final boolean DBG = true; 55 56 /** 57 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 58 * IRecognitionStatusCallback, RecognitionConfig)}, 59 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 60 */ 61 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 62 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 63 64 private static final int INVALID_VALUE = Integer.MIN_VALUE; 65 66 /** The {@link ModuleProperties} for the system, or null if none exists. */ 67 final ModuleProperties moduleProperties; 68 69 /** The properties for the DSP module */ 70 private final SoundTriggerModule mModule; 71 private final Object mLock = new Object(); 72 private final Context mContext; 73 private final TelephonyManager mTelephonyManager; 74 private final PhoneStateListener mPhoneStateListener; 75 private final PowerManager mPowerManager; 76 77 // TODO: Since many layers currently only deal with one recognition 78 // we simplify things by assuming one listener here too. 79 private IRecognitionStatusCallback mActiveListener; 80 private int mKeyphraseId = INVALID_VALUE; 81 private int mCurrentSoundModelHandle = INVALID_VALUE; 82 private UUID mCurrentSoundModelUuid = null; 83 // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. 84 private RecognitionConfig mRecognitionConfig = null; 85 private boolean mRequested = false; 86 private boolean mCallActive = false; 87 private boolean mIsPowerSaveMode = false; 88 // Indicates if the native sound trigger service is disabled or not. 89 // This is an indirect indication of the microphone being open in some other application. 90 private boolean mServiceDisabled = false; 91 private boolean mStarted = false; 92 private PowerSaveModeListener mPowerSaveModeListener; 93 94 SoundTriggerHelper(Context context) { 95 ArrayList <ModuleProperties> modules = new ArrayList<>(); 96 int status = SoundTrigger.listModules(modules); 97 mContext = context; 98 mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); 99 mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); 100 mPhoneStateListener = new MyCallStateListener(); 101 if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { 102 Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); 103 moduleProperties = null; 104 mModule = null; 105 } else { 106 // TODO: Figure out how to determine which module corresponds to the DSP hardware. 107 moduleProperties = modules.get(0); 108 mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); 109 } 110 } 111 112 /** 113 * Starts recognition for the given keyphraseId. 114 * 115 * @param keyphraseId The identifier of the keyphrase for which 116 * the recognition is to be started. 117 * @param soundModel The sound model to use for recognition. 118 * @param listener The listener for the recognition events related to the given keyphrase. 119 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 120 */ 121 int startRecognition(int keyphraseId, 122 KeyphraseSoundModel soundModel, 123 IRecognitionStatusCallback listener, 124 RecognitionConfig recognitionConfig) { 125 if (soundModel == null || listener == null || recognitionConfig == null) { 126 return STATUS_ERROR; 127 } 128 129 synchronized (mLock) { 130 if (DBG) { 131 Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId 132 + " soundModel=" + soundModel + ", listener=" + listener.asBinder() 133 + ", recognitionConfig=" + recognitionConfig); 134 Slog.d(TAG, "moduleProperties=" + moduleProperties); 135 Slog.d(TAG, "current listener=" 136 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 137 Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); 138 Slog.d(TAG, "current SoundModel UUID=" 139 + (mCurrentSoundModelUuid == null ? null : mCurrentSoundModelUuid)); 140 } 141 142 if (!mStarted) { 143 // Get the current call state synchronously for the first recognition. 144 mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; 145 // Register for call state changes when the first call to start recognition occurs. 146 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); 147 148 // Register for power saver mode changes when the first call to start recognition 149 // occurs. 150 if (mPowerSaveModeListener == null) { 151 mPowerSaveModeListener = new PowerSaveModeListener(); 152 mContext.registerReceiver(mPowerSaveModeListener, 153 new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); 154 } 155 mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); 156 } 157 158 if (moduleProperties == null || mModule == null) { 159 Slog.w(TAG, "Attempting startRecognition without the capability"); 160 return STATUS_ERROR; 161 } 162 163 if (mCurrentSoundModelHandle != INVALID_VALUE 164 && !soundModel.uuid.equals(mCurrentSoundModelUuid)) { 165 Slog.w(TAG, "Unloading previous sound model"); 166 int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 167 if (status != SoundTrigger.STATUS_OK) { 168 Slog.w(TAG, "unloadSoundModel call failed with " + status); 169 } 170 mCurrentSoundModelHandle = INVALID_VALUE; 171 mCurrentSoundModelUuid = null; 172 mStarted = false; 173 } 174 175 // If the previous recognition was by a different listener, 176 // Notify them that it was stopped. 177 if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { 178 Slog.w(TAG, "Canceling previous recognition"); 179 try { 180 mActiveListener.onError(STATUS_ERROR); 181 } catch (RemoteException e) { 182 Slog.w(TAG, "RemoteException in onDetectionStopped", e); 183 } 184 mActiveListener = null; 185 } 186 187 // Load the sound model if the current one is null. 188 int soundModelHandle = mCurrentSoundModelHandle; 189 if (mCurrentSoundModelHandle == INVALID_VALUE 190 || mCurrentSoundModelUuid == null) { 191 int[] handle = new int[] { INVALID_VALUE }; 192 int status = mModule.loadSoundModel(soundModel, handle); 193 if (status != SoundTrigger.STATUS_OK) { 194 Slog.w(TAG, "loadSoundModel call failed with " + status); 195 return status; 196 } 197 if (handle[0] == INVALID_VALUE) { 198 Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); 199 return STATUS_ERROR; 200 } 201 soundModelHandle = handle[0]; 202 } else { 203 if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); 204 } 205 206 // Start the recognition. 207 mRequested = true; 208 mKeyphraseId = keyphraseId; 209 mCurrentSoundModelHandle = soundModelHandle; 210 mCurrentSoundModelUuid = soundModel.uuid; 211 mRecognitionConfig = recognitionConfig; 212 // Register the new listener. This replaces the old one. 213 // There can only be a maximum of one active listener at any given time. 214 mActiveListener = listener; 215 216 return updateRecognitionLocked(false /* don't notify for synchronous calls */); 217 } 218 } 219 220 /** 221 * Stops recognition for the given {@link Keyphrase} if a recognition is 222 * currently active. 223 * 224 * @param keyphraseId The identifier of the keyphrase for which 225 * the recognition is to be stopped. 226 * @param listener The listener for the recognition events related to the given keyphrase. 227 * 228 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 229 */ 230 int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 231 if (listener == null) { 232 return STATUS_ERROR; 233 } 234 235 synchronized (mLock) { 236 if (DBG) { 237 Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId 238 + ", listener=" + listener.asBinder()); 239 Slog.d(TAG, "current listener=" 240 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 241 } 242 243 if (moduleProperties == null || mModule == null) { 244 Slog.w(TAG, "Attempting stopRecognition without the capability"); 245 return STATUS_ERROR; 246 } 247 248 if (mActiveListener == null) { 249 // startRecognition hasn't been called or it failed. 250 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 251 return STATUS_ERROR; 252 } 253 if (mActiveListener.asBinder() != listener.asBinder()) { 254 // We don't allow a different listener to stop the recognition than the one 255 // that started it. 256 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 257 return STATUS_ERROR; 258 } 259 260 // Stop recognition if it's the current one, ignore otherwise. 261 mRequested = false; 262 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 263 if (status != SoundTrigger.STATUS_OK) { 264 return status; 265 } 266 267 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 268 if (status != SoundTrigger.STATUS_OK) { 269 Slog.w(TAG, "unloadSoundModel call failed with " + status); 270 } 271 272 // Clear the internal state once the recognition has been stopped. 273 // Unload sound model call may fail in scenarios, and we'd still want 274 // to reload the sound model. 275 internalClearStateLocked(); 276 return status; 277 } 278 } 279 280 /** 281 * Stops all recognitions active currently and clears the internal state. 282 */ 283 void stopAllRecognitions() { 284 synchronized (mLock) { 285 if (moduleProperties == null || mModule == null) { 286 return; 287 } 288 289 if (mCurrentSoundModelHandle == INVALID_VALUE) { 290 return; 291 } 292 293 mRequested = false; 294 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 295 status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 296 if (status != SoundTrigger.STATUS_OK) { 297 Slog.w(TAG, "unloadSoundModel call failed with " + status); 298 } 299 300 internalClearStateLocked(); 301 } 302 } 303 304 //---- SoundTrigger.StatusListener methods 305 @Override 306 public void onRecognition(RecognitionEvent event) { 307 if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { 308 Slog.w(TAG, "Invalid recognition event!"); 309 return; 310 } 311 312 if (DBG) Slog.d(TAG, "onRecognition: " + event); 313 synchronized (mLock) { 314 if (mActiveListener == null) { 315 Slog.w(TAG, "received onRecognition event without any listener for it"); 316 return; 317 } 318 switch (event.status) { 319 // Fire aborts/failures to all listeners since it's not tied to a keyphrase. 320 case SoundTrigger.RECOGNITION_STATUS_ABORT: 321 onRecognitionAbortLocked(); 322 break; 323 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 324 onRecognitionFailureLocked(); 325 break; 326 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 327 onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); 328 break; 329 } 330 } 331 } 332 333 @Override 334 public void onSoundModelUpdate(SoundModelEvent event) { 335 if (event == null) { 336 Slog.w(TAG, "Invalid sound model event!"); 337 return; 338 } 339 if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); 340 synchronized (mLock) { 341 onSoundModelUpdatedLocked(event); 342 } 343 } 344 345 @Override 346 public void onServiceStateChange(int state) { 347 if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); 348 synchronized (mLock) { 349 onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); 350 } 351 } 352 353 @Override 354 public void onServiceDied() { 355 Slog.e(TAG, "onServiceDied!!"); 356 synchronized (mLock) { 357 onServiceDiedLocked(); 358 } 359 } 360 361 private void onCallStateChangedLocked(boolean callActive) { 362 if (mCallActive == callActive) { 363 // We consider multiple call states as being active 364 // so we check if something really changed or not here. 365 return; 366 } 367 mCallActive = callActive; 368 updateRecognitionLocked(true /* notify */); 369 } 370 371 private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) { 372 if (mIsPowerSaveMode == isPowerSaveMode) { 373 return; 374 } 375 mIsPowerSaveMode = isPowerSaveMode; 376 updateRecognitionLocked(true /* notify */); 377 } 378 379 private void onSoundModelUpdatedLocked(SoundModelEvent event) { 380 // TODO: Handle sound model update here. 381 } 382 383 private void onServiceStateChangedLocked(boolean disabled) { 384 if (disabled == mServiceDisabled) { 385 return; 386 } 387 mServiceDisabled = disabled; 388 updateRecognitionLocked(true /* notify */); 389 } 390 391 private void onRecognitionAbortLocked() { 392 Slog.w(TAG, "Recognition aborted"); 393 // No-op 394 // This is handled via service state changes instead. 395 } 396 397 private void onRecognitionFailureLocked() { 398 Slog.w(TAG, "Recognition failure"); 399 try { 400 if (mActiveListener != null) { 401 mActiveListener.onError(STATUS_ERROR); 402 } 403 } catch (RemoteException e) { 404 Slog.w(TAG, "RemoteException in onError", e); 405 } finally { 406 internalClearStateLocked(); 407 } 408 } 409 410 private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { 411 Slog.i(TAG, "Recognition success"); 412 KeyphraseRecognitionExtra[] keyphraseExtras = 413 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 414 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 415 Slog.w(TAG, "Invalid keyphrase recognition event!"); 416 return; 417 } 418 // TODO: Handle more than one keyphrase extras. 419 if (mKeyphraseId != keyphraseExtras[0].id) { 420 Slog.w(TAG, "received onRecognition event for a different keyphrase"); 421 return; 422 } 423 424 try { 425 if (mActiveListener != null) { 426 mActiveListener.onDetected((KeyphraseRecognitionEvent) event); 427 } 428 } catch (RemoteException e) { 429 Slog.w(TAG, "RemoteException in onDetected", e); 430 } 431 432 mStarted = false; 433 mRequested = mRecognitionConfig.allowMultipleTriggers; 434 // TODO: Remove this block if the lower layer supports multiple triggers. 435 if (mRequested) { 436 updateRecognitionLocked(true /* notify */); 437 } 438 } 439 440 private void onServiceDiedLocked() { 441 try { 442 if (mActiveListener != null) { 443 mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); 444 } 445 } catch (RemoteException e) { 446 Slog.w(TAG, "RemoteException in onError", e); 447 } finally { 448 internalClearStateLocked(); 449 } 450 } 451 452 private int updateRecognitionLocked(boolean notify) { 453 if (mModule == null || moduleProperties == null 454 || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { 455 // Nothing to do here. 456 return STATUS_OK; 457 } 458 459 boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; 460 if (start == mStarted) { 461 // No-op. 462 return STATUS_OK; 463 } 464 465 // See if the recognition needs to be started. 466 if (start) { 467 // Start recognition. 468 int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); 469 if (status != SoundTrigger.STATUS_OK) { 470 Slog.w(TAG, "startRecognition failed with " + status); 471 // Notify of error if needed. 472 if (notify) { 473 try { 474 mActiveListener.onError(status); 475 } catch (RemoteException e) { 476 Slog.w(TAG, "RemoteException in onError", e); 477 } 478 } 479 } else { 480 mStarted = true; 481 // Notify of resume if needed. 482 if (notify) { 483 try { 484 mActiveListener.onRecognitionResumed(); 485 } catch (RemoteException e) { 486 Slog.w(TAG, "RemoteException in onRecognitionResumed", e); 487 } 488 } 489 } 490 return status; 491 } else { 492 // Stop recognition. 493 int status = mModule.stopRecognition(mCurrentSoundModelHandle); 494 if (status != SoundTrigger.STATUS_OK) { 495 Slog.w(TAG, "stopRecognition call failed with " + status); 496 if (notify) { 497 try { 498 mActiveListener.onError(status); 499 } catch (RemoteException e) { 500 Slog.w(TAG, "RemoteException in onError", e); 501 } 502 } 503 } else { 504 mStarted = false; 505 // Notify of pause if needed. 506 if (notify) { 507 try { 508 mActiveListener.onRecognitionPaused(); 509 } catch (RemoteException e) { 510 Slog.w(TAG, "RemoteException in onRecognitionPaused", e); 511 } 512 } 513 } 514 return status; 515 } 516 } 517 518 private void internalClearStateLocked() { 519 mStarted = false; 520 mRequested = false; 521 522 mKeyphraseId = INVALID_VALUE; 523 mCurrentSoundModelHandle = INVALID_VALUE; 524 mCurrentSoundModelUuid = null; 525 mRecognitionConfig = null; 526 mActiveListener = null; 527 528 // Unregister from call state changes. 529 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); 530 531 // Unregister from power save mode changes. 532 if (mPowerSaveModeListener != null) { 533 mContext.unregisterReceiver(mPowerSaveModeListener); 534 mPowerSaveModeListener = null; 535 } 536 } 537 538 class MyCallStateListener extends PhoneStateListener { 539 @Override 540 public void onCallStateChanged(int state, String arg1) { 541 if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); 542 synchronized (mLock) { 543 onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); 544 } 545 } 546 } 547 548 class PowerSaveModeListener extends BroadcastReceiver { 549 @Override 550 public void onReceive(Context context, Intent intent) { 551 if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) { 552 return; 553 } 554 boolean active = mPowerManager.isPowerSaveMode(); 555 if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active); 556 synchronized (mLock) { 557 onPowerSaveModeChangedLocked(active); 558 } 559 } 560 } 561 562 void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 563 synchronized (mLock) { 564 pw.print(" module properties="); 565 pw.println(moduleProperties == null ? "null" : moduleProperties); 566 pw.print(" keyphrase ID="); pw.println(mKeyphraseId); 567 pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); 568 pw.print(" sound model UUID="); 569 pw.println(mCurrentSoundModelUuid == null ? "null" : mCurrentSoundModelUuid); 570 pw.print(" current listener="); 571 pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); 572 573 pw.print(" requested="); pw.println(mRequested); 574 pw.print(" started="); pw.println(mStarted); 575 pw.print(" call active="); pw.println(mCallActive); 576 pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); 577 pw.print(" service disabled="); pw.println(mServiceDisabled); 578 } 579 } 580} 581