/** * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.service.voice; import android.content.Intent; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; import android.hardware.soundtrigger.KeyphraseMetadata; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; import android.hardware.soundtrigger.SoundTrigger.Keyphrase; import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; import android.os.AsyncTask; import android.os.Handler; import android.os.Message; import android.os.RemoteException; import android.util.Slog; import com.android.internal.app.IVoiceInteractionManagerService; import java.util.List; /** * A class that lets a VoiceInteractionService implementation interact with * always-on keyphrase detection APIs. */ public class AlwaysOnHotwordDetector { //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// /** * Indicates that this hotword detector is no longer valid for any recognition * and should not be used anymore. */ public static final int STATE_INVALID = -3; /** * Indicates that recognition for the given keyphrase is not available on the system * because of the hardware configuration. */ public static final int STATE_HARDWARE_UNAVAILABLE = -2; /** * Indicates that recognition for the given keyphrase is not supported. */ public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; /** * Indicates that the given keyphrase is not enrolled. */ public static final int STATE_KEYPHRASE_UNENROLLED = 1; /** * Indicates that the given keyphrase is currently enrolled and it's possible to start * recognition for it. */ public static final int STATE_KEYPHRASE_ENROLLED = 2; /** * Indicates that the detector isn't ready currently. */ private static final int STATE_NOT_READY = 0; // Keyphrase management actions. Used in getManageIntent() ----// /** Indicates that we need to enroll. */ public static final int MANAGE_ACTION_ENROLL = 0; /** Indicates that we need to re-enroll. */ public static final int MANAGE_ACTION_RE_ENROLL = 1; /** Indicates that we need to un-enroll. */ public static final int MANAGE_ACTION_UN_ENROLL = 2; /** * Return codes for {@link #startRecognition(int)}, {@link #stopRecognition()} */ public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; public static final int STATUS_OK = SoundTrigger.STATUS_OK; //-- Flags for startRecogntion ----// /** Empty flag for {@link #startRecognition(int)}. */ public static final int RECOGNITION_FLAG_NONE = 0; /** * Recognition flag for {@link #startRecognition(int)} that indicates * whether the trigger audio for hotword needs to be captured. */ public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. /** * Simple recognition of the key phrase. Returned by {@link #getSupportedRecognitionModes()} */ public static final int RECOGNITION_MODE_VOICE_TRIGGER = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; /** * Trigger only if one user is identified. Returned by {@link #getSupportedRecognitionModes()} */ public static final int RECOGNITION_MODE_USER_IDENTIFICATION = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; static final String TAG = "AlwaysOnHotwordDetector"; // TODO: Set to false. static final boolean DBG = true; private static final int MSG_STATE_CHANGED = 1; private static final int MSG_HOTWORD_DETECTED = 2; private static final int MSG_DETECTION_STOPPED = 3; private final String mText; private final String mLocale; /** * The metadata of the Keyphrase, derived from the enrollment application. * This may be null if this keyphrase isn't supported by the enrollment application. */ private final KeyphraseMetadata mKeyphraseMetadata; private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; private final IVoiceInteractionService mVoiceInteractionService; private final IVoiceInteractionManagerService mModelManagementService; private final SoundTriggerListener mInternalCallback; private final Callback mExternalCallback; private final Object mLock = new Object(); private final Handler mHandler; private int mAvailability = STATE_NOT_READY; /** * Callbacks for always-on hotword detection. */ public interface Callback { /** * Called when the hotword availability changes. * This indicates a change in the availability of recognition for the given keyphrase. * It's called at least once with the initial availability.

* * Availability implies whether the hardware on this system is capable of listening for * the given keyphrase or not.

* If the return code is one of {@link #STATE_HARDWARE_UNAVAILABLE} or * {@link #STATE_KEYPHRASE_UNSUPPORTED}, * detection is not possible and no further interaction should be * performed with this detector.
* If it is {@link #STATE_KEYPHRASE_UNENROLLED} the caller may choose to begin * an enrollment flow for the keyphrase.
* and for {@link #STATE_KEYPHRASE_ENROLLED} a recognition can be started as desired.

* * If the return code is {@link #STATE_INVALID}, this detector is stale. * A new detector should be obtained for use in the future. */ void onAvailabilityChanged(int status); /** * Called when the keyphrase is spoken. * * @param data Optional trigger audio data, if it was requested during * {@link AlwaysOnHotwordDetector#startRecognition(int)}. */ void onDetected(byte[] data); /** * Called when the detection for the associated keyphrase stops. */ void onDetectionStopped(); } /** * @param text The keyphrase text to get the detector for. * @param locale The java locale for the detector. * @param callback A non-null Callback for receiving the recognition events. * @param voiceInteractionService The current voice interaction service. * @param modelManagementService A service that allows management of sound models. * * @hide */ public AlwaysOnHotwordDetector(String text, String locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionService voiceInteractionService, IVoiceInteractionManagerService modelManagementService) { mText = text; mLocale = locale; mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); mExternalCallback = callback; mHandler = new MyHandler(); mInternalCallback = new SoundTriggerListener(mHandler); mVoiceInteractionService = voiceInteractionService; mModelManagementService = modelManagementService; new RefreshAvailabiltyTask().execute(); } /** * Gets the recognition modes supported by the associated keyphrase. * * @throws UnsupportedOperationException if the keyphrase itself isn't supported. * Callers should only call this method after a supported state callback on * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. */ public int getSupportedRecognitionModes() { synchronized (mLock) { return getSupportedRecognitionModesLocked(); } } private int getSupportedRecognitionModesLocked() { // This method only makes sense if we can actually support a recognition. if (mAvailability != STATE_KEYPHRASE_ENROLLED && mAvailability != STATE_KEYPHRASE_UNENROLLED) { throw new UnsupportedOperationException( "Getting supported recognition modes for the keyphrase is not supported"); } return mKeyphraseMetadata.recognitionModeFlags; } /** * Starts recognition for the associated keyphrase. * * @param recognitionFlags The flags to control the recognition properties. * The allowed flags are {@link #RECOGNITION_FLAG_NONE} and * {@link #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO}. * @return {@link #STATUS_OK} if the call succeeds, an error code otherwise. * @throws UnsupportedOperationException if the recognition isn't supported. * Callers should only call this method after a supported state callback on * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. */ public int startRecognition(int recognitionFlags) { synchronized (mLock) { return startRecognitionLocked(recognitionFlags); } } private int startRecognitionLocked(int recognitionFlags) { // This method only makes sense if we can start a recognition. if (mAvailability != STATE_KEYPHRASE_ENROLLED) { throw new UnsupportedOperationException( "Recognition for the given keyphrase is not supported"); } KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; // TODO: Do we need to do something about the confidence level here? recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, mKeyphraseMetadata.recognitionModeFlags, new ConfidenceLevel[0]); boolean captureTriggerAudio = (recognitionFlags & RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; int code = STATUS_ERROR; try { code = mModelManagementService.startRecognition(mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback, new RecognitionConfig( captureTriggerAudio, recognitionExtra, null /* additional data */)); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in startRecognition!"); } if (code != STATUS_OK) { Slog.w(TAG, "startRecognition() failed with error code " + code); } return code; } /** * Stops recognition for the associated keyphrase. * * @return {@link #STATUS_OK} if the call succeeds, an error code otherwise. * @throws UnsupportedOperationException if the recognition isn't supported. * Callers should only call this method after a supported state callback on * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. */ public int stopRecognition() { synchronized (mLock) { return stopRecognitionLocked(); } } private int stopRecognitionLocked() { // This method only makes sense if we can start a recognition. if (mAvailability != STATE_KEYPHRASE_ENROLLED) { throw new UnsupportedOperationException( "Recognition for the given keyphrase is not supported"); } int code = STATUS_ERROR; try { code = mModelManagementService.stopRecognition( mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in stopRecognition!"); } if (code != STATUS_OK) { Slog.w(TAG, "stopRecognition() failed with error code " + code); } return code; } /** * Gets an intent to manage the associated keyphrase. * * @param action The manage action that needs to be performed. * One of {@link #MANAGE_ACTION_ENROLL}, {@link #MANAGE_ACTION_RE_ENROLL} or * {@link #MANAGE_ACTION_UN_ENROLL}. * @return An {@link Intent} to manage the given keyphrase. * @throws UnsupportedOperationException if managing they keyphrase isn't supported. * Callers should only call this method after a supported state callback on * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. */ public Intent getManageIntent(int action) { // This method only makes sense if we can actually support a recognition. if (mAvailability != STATE_KEYPHRASE_ENROLLED && mAvailability != STATE_KEYPHRASE_UNENROLLED) { throw new UnsupportedOperationException( "Managing the given keyphrase is not supported"); } if (action != MANAGE_ACTION_ENROLL && action != MANAGE_ACTION_RE_ENROLL && action != MANAGE_ACTION_UN_ENROLL) { throw new IllegalArgumentException("Invalid action specified " + action); } return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); } /** * Invalidates this hotword detector so that any future calls to this result * in an IllegalStateException. * * @hide */ void invalidate() { synchronized (mLock) { mAvailability = STATE_INVALID; notifyStateChangedLocked(); } } /** * Reloads the sound models from the service. * * @hide */ void onSoundModelsChanged() { synchronized (mLock) { // TODO: This should stop the recognition if it was using an enrolled sound model // that's no longer available. if (mAvailability == STATE_INVALID || mAvailability == STATE_HARDWARE_UNAVAILABLE || mAvailability == STATE_KEYPHRASE_UNSUPPORTED) { Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config"); return; } // Execute a refresh availability task - which should then notify of a change. new RefreshAvailabiltyTask().execute(); } } private void notifyStateChangedLocked() { Message message = Message.obtain(mHandler, MSG_STATE_CHANGED); message.arg1 = mAvailability; message.sendToTarget(); } /** @hide */ static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub { private final Handler mHandler; public SoundTriggerListener(Handler handler) { mHandler = handler; } @Override public void onDetected(KeyphraseRecognitionEvent event) { Slog.i(TAG, "onDetected"); Message message = Message.obtain(mHandler, MSG_HOTWORD_DETECTED); message.obj = event.data; message.sendToTarget(); } @Override public void onDetectionStopped() { Slog.i(TAG, "onDetectionStopped"); mHandler.sendEmptyMessage(MSG_DETECTION_STOPPED); } } class MyHandler extends Handler { @Override public void handleMessage(Message msg) { switch (msg.what) { case MSG_STATE_CHANGED: mExternalCallback.onAvailabilityChanged(msg.arg1); break; case MSG_HOTWORD_DETECTED: mExternalCallback.onDetected((byte[]) msg.obj); break; case MSG_DETECTION_STOPPED: mExternalCallback.onDetectionStopped(); default: super.handleMessage(msg); } } } class RefreshAvailabiltyTask extends AsyncTask { @Override public Void doInBackground(Void... params) { int availability = internalGetInitialAvailability(); boolean enrolled = false; // Fetch the sound model if the availability is one of the supported ones. if (availability == STATE_NOT_READY || availability == STATE_KEYPHRASE_UNENROLLED || availability == STATE_KEYPHRASE_ENROLLED) { enrolled = internalGetIsEnrolled(mKeyphraseMetadata.id); if (!enrolled) { availability = STATE_KEYPHRASE_UNENROLLED; } else { availability = STATE_KEYPHRASE_ENROLLED; } } synchronized (mLock) { if (DBG) { Slog.d(TAG, "Hotword availability changed from " + mAvailability + " -> " + availability); } mAvailability = availability; notifyStateChangedLocked(); } return null; } /** * @return The initial availability without checking the enrollment status. */ private int internalGetInitialAvailability() { synchronized (mLock) { // This detector has already been invalidated. if (mAvailability == STATE_INVALID) { return STATE_INVALID; } } ModuleProperties dspModuleProperties = null; try { dspModuleProperties = mModelManagementService.getDspModuleProperties(mVoiceInteractionService); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in getDspProperties!"); } // No DSP available if (dspModuleProperties == null) { return STATE_HARDWARE_UNAVAILABLE; } // No enrollment application supports this keyphrase/locale if (mKeyphraseMetadata == null) { return STATE_KEYPHRASE_UNSUPPORTED; } return STATE_NOT_READY; } /** * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. */ private boolean internalGetIsEnrolled(int keyphraseId) { try { return mModelManagementService.isEnrolledForKeyphrase( mVoiceInteractionService, keyphraseId); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!"); } return false; } } }