SoundTriggerHelper.java revision 45c00b5877e908f44853783b42deb437cfd30d94
1/** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.server.voiceinteraction; 18 19import android.content.BroadcastReceiver; 20import android.content.Context; 21import android.content.Intent; 22import android.content.IntentFilter; 23import android.hardware.soundtrigger.IRecognitionStatusCallback; 24import android.hardware.soundtrigger.SoundTrigger; 25import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 26import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 27import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 28import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 29import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 30import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 31import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 32import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; 33import android.hardware.soundtrigger.SoundTriggerModule; 34import android.os.PowerManager; 35import android.os.RemoteException; 36import android.telephony.PhoneStateListener; 37import android.telephony.TelephonyManager; 38import android.util.Slog; 39 40import java.io.FileDescriptor; 41import java.io.PrintWriter; 42import java.util.ArrayList; 43 44/** 45 * Helper for {@link SoundTrigger} APIs. 46 * Currently this just acts as an abstraction over all SoundTrigger API calls. 47 * 48 * @hide 49 */ 50public class SoundTriggerHelper implements SoundTrigger.StatusListener { 51 static final String TAG = "SoundTriggerHelper"; 52 static final boolean DBG = false; 53 54 /** 55 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 56 * IRecognitionStatusCallback, RecognitionConfig)}, 57 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 58 */ 59 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 60 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 61 62 private static final int INVALID_VALUE = Integer.MIN_VALUE; 63 64 /** The {@link ModuleProperties} for the system, or null if none exists. */ 65 final ModuleProperties moduleProperties; 66 67 /** The properties for the DSP module */ 68 private SoundTriggerModule mModule; 69 private final Object mLock = new Object(); 70 private final Context mContext; 71 private final TelephonyManager mTelephonyManager; 72 private final PhoneStateListener mPhoneStateListener; 73 private final PowerManager mPowerManager; 74 75 // TODO: Since many layers currently only deal with one recognition 76 // we simplify things by assuming one listener here too. 77 private IRecognitionStatusCallback mActiveListener; 78 private int mKeyphraseId = INVALID_VALUE; 79 private int mCurrentSoundModelHandle = INVALID_VALUE; 80 private KeyphraseSoundModel mCurrentSoundModel = null; 81 // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. 82 private RecognitionConfig mRecognitionConfig = null; 83 private boolean mRequested = false; 84 private boolean mCallActive = false; 85 private boolean mIsPowerSaveMode = false; 86 // Indicates if the native sound trigger service is disabled or not. 87 // This is an indirect indication of the microphone being open in some other application. 88 private boolean mServiceDisabled = false; 89 private boolean mStarted = false; 90 private PowerSaveModeListener mPowerSaveModeListener; 91 92 SoundTriggerHelper(Context context) { 93 ArrayList <ModuleProperties> modules = new ArrayList<>(); 94 int status = SoundTrigger.listModules(modules); 95 mContext = context; 96 mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); 97 mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); 98 mPhoneStateListener = new MyCallStateListener(); 99 if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { 100 Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); 101 moduleProperties = null; 102 mModule = null; 103 } else { 104 // TODO: Figure out how to determine which module corresponds to the DSP hardware. 105 moduleProperties = modules.get(0); 106 } 107 } 108 109 /** 110 * Starts recognition for the given keyphraseId. 111 * 112 * @param keyphraseId The identifier of the keyphrase for which 113 * the recognition is to be started. 114 * @param soundModel The sound model to use for recognition. 115 * @param listener The listener for the recognition events related to the given keyphrase. 116 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 117 */ 118 int startRecognition(int keyphraseId, 119 KeyphraseSoundModel soundModel, 120 IRecognitionStatusCallback listener, 121 RecognitionConfig recognitionConfig) { 122 if (soundModel == null || listener == null || recognitionConfig == null) { 123 return STATUS_ERROR; 124 } 125 126 synchronized (mLock) { 127 if (DBG) { 128 Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId 129 + " soundModel=" + soundModel + ", listener=" + listener.asBinder() 130 + ", recognitionConfig=" + recognitionConfig); 131 Slog.d(TAG, "moduleProperties=" + moduleProperties); 132 Slog.d(TAG, "current listener=" 133 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 134 Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); 135 Slog.d(TAG, "current SoundModel UUID=" 136 + (mCurrentSoundModel == null ? null : mCurrentSoundModel.uuid)); 137 } 138 139 if (!mStarted) { 140 // Get the current call state synchronously for the first recognition. 141 mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; 142 // Register for call state changes when the first call to start recognition occurs. 143 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); 144 145 // Register for power saver mode changes when the first call to start recognition 146 // occurs. 147 if (mPowerSaveModeListener == null) { 148 mPowerSaveModeListener = new PowerSaveModeListener(); 149 mContext.registerReceiver(mPowerSaveModeListener, 150 new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); 151 } 152 mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); 153 } 154 155 if (moduleProperties == null) { 156 Slog.w(TAG, "Attempting startRecognition without the capability"); 157 return STATUS_ERROR; 158 } 159 if (mModule == null) { 160 mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); 161 if (mModule == null) { 162 Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); 163 return STATUS_ERROR; 164 } 165 } 166 167 // Unload the previous model if the current one isn't invalid 168 // and, it's not the same as the new one. 169 // This helps use cache and reuse the model and just start/stop it when necessary. 170 if (mCurrentSoundModelHandle != INVALID_VALUE 171 && !soundModel.equals(mCurrentSoundModel)) { 172 Slog.w(TAG, "Unloading previous sound model"); 173 int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); 174 if (status != SoundTrigger.STATUS_OK) { 175 Slog.w(TAG, "unloadSoundModel call failed with " + status); 176 } 177 internalClearSoundModelLocked(); 178 mStarted = false; 179 } 180 181 // If the previous recognition was by a different listener, 182 // Notify them that it was stopped. 183 if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { 184 Slog.w(TAG, "Canceling previous recognition"); 185 try { 186 mActiveListener.onError(STATUS_ERROR); 187 } catch (RemoteException e) { 188 Slog.w(TAG, "RemoteException in onDetectionStopped", e); 189 } 190 mActiveListener = null; 191 } 192 193 // Load the sound model if the current one is null. 194 int soundModelHandle = mCurrentSoundModelHandle; 195 if (mCurrentSoundModelHandle == INVALID_VALUE 196 || mCurrentSoundModel == null) { 197 int[] handle = new int[] { INVALID_VALUE }; 198 int status = mModule.loadSoundModel(soundModel, handle); 199 if (status != SoundTrigger.STATUS_OK) { 200 Slog.w(TAG, "loadSoundModel call failed with " + status); 201 return status; 202 } 203 if (handle[0] == INVALID_VALUE) { 204 Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); 205 return STATUS_ERROR; 206 } 207 soundModelHandle = handle[0]; 208 } else { 209 if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); 210 } 211 212 // Start the recognition. 213 mRequested = true; 214 mKeyphraseId = keyphraseId; 215 mCurrentSoundModelHandle = soundModelHandle; 216 mCurrentSoundModel = soundModel; 217 mRecognitionConfig = recognitionConfig; 218 // Register the new listener. This replaces the old one. 219 // There can only be a maximum of one active listener at any given time. 220 mActiveListener = listener; 221 222 return updateRecognitionLocked(false /* don't notify for synchronous calls */); 223 } 224 } 225 226 /** 227 * Stops recognition for the given {@link Keyphrase} if a recognition is 228 * currently active. 229 * 230 * @param keyphraseId The identifier of the keyphrase for which 231 * the recognition is to be stopped. 232 * @param listener The listener for the recognition events related to the given keyphrase. 233 * 234 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 235 */ 236 int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 237 if (listener == null) { 238 return STATUS_ERROR; 239 } 240 241 synchronized (mLock) { 242 if (DBG) { 243 Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId 244 + ", listener=" + listener.asBinder()); 245 Slog.d(TAG, "current listener=" 246 + (mActiveListener == null ? "null" : mActiveListener.asBinder())); 247 } 248 249 if (moduleProperties == null || mModule == null) { 250 Slog.w(TAG, "Attempting stopRecognition without the capability"); 251 return STATUS_ERROR; 252 } 253 254 if (mActiveListener == null) { 255 // startRecognition hasn't been called or it failed. 256 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 257 return STATUS_ERROR; 258 } 259 if (mActiveListener.asBinder() != listener.asBinder()) { 260 // We don't allow a different listener to stop the recognition than the one 261 // that started it. 262 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 263 return STATUS_ERROR; 264 } 265 266 // Stop recognition if it's the current one, ignore otherwise. 267 mRequested = false; 268 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 269 if (status != SoundTrigger.STATUS_OK) { 270 return status; 271 } 272 273 // We leave the sound model loaded but not started, this helps us when we start 274 // back. 275 // Also clear the internal state once the recognition has been stopped. 276 internalClearStateLocked(); 277 return status; 278 } 279 } 280 281 /** 282 * Stops all recognitions active currently and clears the internal state. 283 */ 284 void stopAllRecognitions() { 285 synchronized (mLock) { 286 if (moduleProperties == null || mModule == null) { 287 return; 288 } 289 290 if (mCurrentSoundModelHandle == INVALID_VALUE) { 291 return; 292 } 293 294 mRequested = false; 295 int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); 296 internalClearStateLocked(); 297 } 298 } 299 300 //---- SoundTrigger.StatusListener methods 301 @Override 302 public void onRecognition(RecognitionEvent event) { 303 if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { 304 Slog.w(TAG, "Invalid recognition event!"); 305 return; 306 } 307 308 if (DBG) Slog.d(TAG, "onRecognition: " + event); 309 synchronized (mLock) { 310 if (mActiveListener == null) { 311 Slog.w(TAG, "received onRecognition event without any listener for it"); 312 return; 313 } 314 switch (event.status) { 315 // Fire aborts/failures to all listeners since it's not tied to a keyphrase. 316 case SoundTrigger.RECOGNITION_STATUS_ABORT: 317 onRecognitionAbortLocked(); 318 break; 319 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 320 onRecognitionFailureLocked(); 321 break; 322 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 323 onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); 324 break; 325 } 326 } 327 } 328 329 @Override 330 public void onSoundModelUpdate(SoundModelEvent event) { 331 if (event == null) { 332 Slog.w(TAG, "Invalid sound model event!"); 333 return; 334 } 335 if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); 336 synchronized (mLock) { 337 onSoundModelUpdatedLocked(event); 338 } 339 } 340 341 @Override 342 public void onServiceStateChange(int state) { 343 if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); 344 synchronized (mLock) { 345 onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); 346 } 347 } 348 349 @Override 350 public void onServiceDied() { 351 Slog.e(TAG, "onServiceDied!!"); 352 synchronized (mLock) { 353 onServiceDiedLocked(); 354 } 355 } 356 357 private void onCallStateChangedLocked(boolean callActive) { 358 if (mCallActive == callActive) { 359 // We consider multiple call states as being active 360 // so we check if something really changed or not here. 361 return; 362 } 363 mCallActive = callActive; 364 updateRecognitionLocked(true /* notify */); 365 } 366 367 private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) { 368 if (mIsPowerSaveMode == isPowerSaveMode) { 369 return; 370 } 371 mIsPowerSaveMode = isPowerSaveMode; 372 updateRecognitionLocked(true /* notify */); 373 } 374 375 private void onSoundModelUpdatedLocked(SoundModelEvent event) { 376 // TODO: Handle sound model update here. 377 } 378 379 private void onServiceStateChangedLocked(boolean disabled) { 380 if (disabled == mServiceDisabled) { 381 return; 382 } 383 mServiceDisabled = disabled; 384 updateRecognitionLocked(true /* notify */); 385 } 386 387 private void onRecognitionAbortLocked() { 388 Slog.w(TAG, "Recognition aborted"); 389 // No-op 390 // This is handled via service state changes instead. 391 } 392 393 private void onRecognitionFailureLocked() { 394 Slog.w(TAG, "Recognition failure"); 395 try { 396 if (mActiveListener != null) { 397 mActiveListener.onError(STATUS_ERROR); 398 } 399 } catch (RemoteException e) { 400 Slog.w(TAG, "RemoteException in onError", e); 401 } finally { 402 internalClearStateLocked(); 403 } 404 } 405 406 private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { 407 Slog.i(TAG, "Recognition success"); 408 KeyphraseRecognitionExtra[] keyphraseExtras = 409 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 410 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 411 Slog.w(TAG, "Invalid keyphrase recognition event!"); 412 return; 413 } 414 // TODO: Handle more than one keyphrase extras. 415 if (mKeyphraseId != keyphraseExtras[0].id) { 416 Slog.w(TAG, "received onRecognition event for a different keyphrase"); 417 return; 418 } 419 420 try { 421 if (mActiveListener != null) { 422 mActiveListener.onDetected((KeyphraseRecognitionEvent) event); 423 } 424 } catch (RemoteException e) { 425 Slog.w(TAG, "RemoteException in onDetected", e); 426 } 427 428 mStarted = false; 429 mRequested = mRecognitionConfig.allowMultipleTriggers; 430 // TODO: Remove this block if the lower layer supports multiple triggers. 431 if (mRequested) { 432 updateRecognitionLocked(true /* notify */); 433 } 434 } 435 436 private void onServiceDiedLocked() { 437 try { 438 if (mActiveListener != null) { 439 mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); 440 } 441 } catch (RemoteException e) { 442 Slog.w(TAG, "RemoteException in onError", e); 443 } finally { 444 internalClearSoundModelLocked(); 445 internalClearStateLocked(); 446 if (mModule != null) { 447 mModule.detach(); 448 mModule = null; 449 } 450 } 451 } 452 453 private int updateRecognitionLocked(boolean notify) { 454 if (mModule == null || moduleProperties == null 455 || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { 456 // Nothing to do here. 457 return STATUS_OK; 458 } 459 460 boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; 461 if (start == mStarted) { 462 // No-op. 463 return STATUS_OK; 464 } 465 466 // See if the recognition needs to be started. 467 if (start) { 468 // Start recognition. 469 int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); 470 if (status != SoundTrigger.STATUS_OK) { 471 Slog.w(TAG, "startRecognition failed with " + status); 472 // Notify of error if needed. 473 if (notify) { 474 try { 475 mActiveListener.onError(status); 476 } catch (RemoteException e) { 477 Slog.w(TAG, "RemoteException in onError", e); 478 } 479 } 480 } else { 481 mStarted = true; 482 // Notify of resume if needed. 483 if (notify) { 484 try { 485 mActiveListener.onRecognitionResumed(); 486 } catch (RemoteException e) { 487 Slog.w(TAG, "RemoteException in onRecognitionResumed", e); 488 } 489 } 490 } 491 return status; 492 } else { 493 // Stop recognition. 494 int status = mModule.stopRecognition(mCurrentSoundModelHandle); 495 if (status != SoundTrigger.STATUS_OK) { 496 Slog.w(TAG, "stopRecognition call failed with " + status); 497 if (notify) { 498 try { 499 mActiveListener.onError(status); 500 } catch (RemoteException e) { 501 Slog.w(TAG, "RemoteException in onError", e); 502 } 503 } 504 } else { 505 mStarted = false; 506 // Notify of pause if needed. 507 if (notify) { 508 try { 509 mActiveListener.onRecognitionPaused(); 510 } catch (RemoteException e) { 511 Slog.w(TAG, "RemoteException in onRecognitionPaused", e); 512 } 513 } 514 } 515 return status; 516 } 517 } 518 519 private void internalClearStateLocked() { 520 mStarted = false; 521 mRequested = false; 522 523 mKeyphraseId = INVALID_VALUE; 524 mRecognitionConfig = null; 525 mActiveListener = null; 526 527 // Unregister from call state changes. 528 mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); 529 530 // Unregister from power save mode changes. 531 if (mPowerSaveModeListener != null) { 532 mContext.unregisterReceiver(mPowerSaveModeListener); 533 mPowerSaveModeListener = null; 534 } 535 } 536 537 private void internalClearSoundModelLocked() { 538 mCurrentSoundModelHandle = INVALID_VALUE; 539 mCurrentSoundModel = null; 540 } 541 542 class MyCallStateListener extends PhoneStateListener { 543 @Override 544 public void onCallStateChanged(int state, String arg1) { 545 if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); 546 synchronized (mLock) { 547 onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); 548 } 549 } 550 } 551 552 class PowerSaveModeListener extends BroadcastReceiver { 553 @Override 554 public void onReceive(Context context, Intent intent) { 555 if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) { 556 return; 557 } 558 boolean active = mPowerManager.isPowerSaveMode(); 559 if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active); 560 synchronized (mLock) { 561 onPowerSaveModeChangedLocked(active); 562 } 563 } 564 } 565 566 void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 567 synchronized (mLock) { 568 pw.print(" module properties="); 569 pw.println(moduleProperties == null ? "null" : moduleProperties); 570 pw.print(" keyphrase ID="); pw.println(mKeyphraseId); 571 pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); 572 pw.print(" sound model UUID="); 573 pw.println(mCurrentSoundModel == null ? "null" : mCurrentSoundModel.uuid); 574 pw.print(" current listener="); 575 pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); 576 577 pw.print(" requested="); pw.println(mRequested); 578 pw.print(" started="); pw.println(mStarted); 579 pw.print(" call active="); pw.println(mCallActive); 580 pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); 581 pw.print(" service disabled="); pw.println(mServiceDisabled); 582 } 583 } 584} 585