xref: /frameworks/base/services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java

/**
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.server.voiceinteraction;

import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.soundtrigger.IRecognitionStatusCallback;
import android.hardware.soundtrigger.SoundTrigger;
import android.hardware.soundtrigger.SoundTrigger.Keyphrase;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent;
import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent;
import android.hardware.soundtrigger.SoundTriggerModule;
import android.os.PowerManager;
import android.os.RemoteException;
import android.telephony.PhoneStateListener;
import android.telephony.TelephonyManager;
import android.util.Slog;

import java.io.FileDescriptor;
import java.io.PrintWriter;
import java.util.ArrayList;

/**
 * Helper for {@link SoundTrigger} APIs.
 * Currently this just acts as an abstraction over all SoundTrigger API calls.
 *
 * @hide
 */
public class SoundTriggerHelper implements SoundTrigger.StatusListener {
    static final String TAG = "SoundTriggerHelper";
    static final boolean DBG = false;

    /**
     * Return codes for {@link #startRecognition(int, KeyphraseSoundModel,
     *      IRecognitionStatusCallback, RecognitionConfig)},
     * {@link #stopRecognition(int, IRecognitionStatusCallback)}
     */
    public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR;
    public static final int STATUS_OK = SoundTrigger.STATUS_OK;

    private static final int INVALID_VALUE = Integer.MIN_VALUE;

    /** The {@link ModuleProperties} for the system, or null if none exists. */
    final ModuleProperties moduleProperties;

    /** The properties for the DSP module */
    private SoundTriggerModule mModule;
    private final Object mLock = new Object();
    private final Context mContext;
    private final TelephonyManager mTelephonyManager;
    private final PhoneStateListener mPhoneStateListener;
    private final PowerManager mPowerManager;

    // TODO: Since many layers currently only deal with one recognition
    // we simplify things by assuming one listener here too.
    private IRecognitionStatusCallback mActiveListener;
    private int mKeyphraseId = INVALID_VALUE;
    private int mCurrentSoundModelHandle = INVALID_VALUE;
    private KeyphraseSoundModel mCurrentSoundModel = null;
    // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer.
    private RecognitionConfig mRecognitionConfig = null;
    private boolean mRequested = false;
    private boolean mCallActive = false;
    private boolean mIsPowerSaveMode = false;
    // Indicates if the native sound trigger service is disabled or not.
    // This is an indirect indication of the microphone being open in some other application.
    private boolean mServiceDisabled = false;
    private boolean mStarted = false;
    private PowerSaveModeListener mPowerSaveModeListener;

    SoundTriggerHelper(Context context) {
        ArrayList <ModuleProperties> modules = new ArrayList<>();
        int status = SoundTrigger.listModules(modules);
        mContext = context;
        mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE);
        mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE);
        mPhoneStateListener = new MyCallStateListener();
        if (status != SoundTrigger.STATUS_OK || modules.size() == 0) {
            Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size());
            moduleProperties = null;
            mModule = null;
        } else {
            // TODO: Figure out how to determine which module corresponds to the DSP hardware.
            moduleProperties = modules.get(0);
        }
    }

    /**
     * Starts recognition for the given keyphraseId.
     *
     * @param keyphraseId The identifier of the keyphrase for which
     *        the recognition is to be started.
     * @param soundModel The sound model to use for recognition.
     * @param listener The listener for the recognition events related to the given keyphrase.
     * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
     */
    int startRecognition(int keyphraseId,
            KeyphraseSoundModel soundModel,
            IRecognitionStatusCallback listener,
            RecognitionConfig recognitionConfig) {
        if (soundModel == null || listener == null || recognitionConfig == null) {
            return STATUS_ERROR;
        }

        synchronized (mLock) {
            if (DBG) {
                Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId
                        + " soundModel=" + soundModel + ", listener=" + listener.asBinder()
                        + ", recognitionConfig=" + recognitionConfig);
                Slog.d(TAG, "moduleProperties=" + moduleProperties);
                Slog.d(TAG, "current listener="
                        + (mActiveListener == null ? "null" : mActiveListener.asBinder()));
                Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle);
                Slog.d(TAG, "current SoundModel UUID="
                        + (mCurrentSoundModel == null ? null : mCurrentSoundModel.uuid));
            }

            if (!mStarted) {
                // Get the current call state synchronously for the first recognition.
                mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE;
                // Register for call state changes when the first call to start recognition occurs.
                mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE);

                // Register for power saver mode changes when the first call to start recognition
                // occurs.
                if (mPowerSaveModeListener == null) {
                    mPowerSaveModeListener = new PowerSaveModeListener();
                    mContext.registerReceiver(mPowerSaveModeListener,
                            new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED));
                }
                mIsPowerSaveMode = mPowerManager.isPowerSaveMode();
            }

            if (moduleProperties == null) {
                Slog.w(TAG, "Attempting startRecognition without the capability");
                return STATUS_ERROR;
            }
            if (mModule == null) {
                mModule = SoundTrigger.attachModule(moduleProperties.id, this, null);
                if (mModule == null) {
                    Slog.w(TAG, "startRecognition cannot attach to sound trigger module");
                    return STATUS_ERROR;
                }
            }

            // Unload the previous model if the current one isn't invalid
            // and, it's not the same as the new one.
            // This helps use cache and reuse the model and just start/stop it when necessary.
            if (mCurrentSoundModelHandle != INVALID_VALUE
                    && !soundModel.equals(mCurrentSoundModel)) {
                Slog.w(TAG, "Unloading previous sound model");
                int status = mModule.unloadSoundModel(mCurrentSoundModelHandle);
                if (status != SoundTrigger.STATUS_OK) {
                    Slog.w(TAG, "unloadSoundModel call failed with " + status);
                }
                internalClearSoundModelLocked();
                mStarted = false;
            }

            // If the previous recognition was by a different listener,
            // Notify them that it was stopped.
            if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) {
                Slog.w(TAG, "Canceling previous recognition");
                try {
                    mActiveListener.onError(STATUS_ERROR);
                } catch (RemoteException e) {
                    Slog.w(TAG, "RemoteException in onDetectionStopped", e);
                }
                mActiveListener = null;
            }

            // Load the sound model if the current one is null.
            int soundModelHandle = mCurrentSoundModelHandle;
            if (mCurrentSoundModelHandle == INVALID_VALUE
                    || mCurrentSoundModel == null) {
                int[] handle = new int[] { INVALID_VALUE };
                int status = mModule.loadSoundModel(soundModel, handle);
                if (status != SoundTrigger.STATUS_OK) {
                    Slog.w(TAG, "loadSoundModel call failed with " + status);
                    return status;
                }
                if (handle[0] == INVALID_VALUE) {
                    Slog.w(TAG, "loadSoundModel call returned invalid sound model handle");
                    return STATUS_ERROR;
                }
                soundModelHandle = handle[0];
            } else {
                if (DBG) Slog.d(TAG, "Reusing previously loaded sound model");
            }

            // Start the recognition.
            mRequested = true;
            mKeyphraseId = keyphraseId;
            mCurrentSoundModelHandle = soundModelHandle;
            mCurrentSoundModel = soundModel;
            mRecognitionConfig = recognitionConfig;
            // Register the new listener. This replaces the old one.
            // There can only be a maximum of one active listener at any given time.
            mActiveListener = listener;

            return updateRecognitionLocked(false /* don't notify for synchronous calls */);
        }
    }

    /**
     * Stops recognition for the given {@link Keyphrase} if a recognition is
     * currently active.
     *
     * @param keyphraseId The identifier of the keyphrase for which
     *        the recognition is to be stopped.
     * @param listener The listener for the recognition events related to the given keyphrase.
     *
     * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
     */
    int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) {
        if (listener == null) {
            return STATUS_ERROR;
        }

        synchronized (mLock) {
            if (DBG) {
                Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId
                        + ", listener=" + listener.asBinder());
                Slog.d(TAG, "current listener="
                        + (mActiveListener == null ? "null" : mActiveListener.asBinder()));
            }

            if (moduleProperties == null || mModule == null) {
                Slog.w(TAG, "Attempting stopRecognition without the capability");
                return STATUS_ERROR;
            }

            if (mActiveListener == null) {
                // startRecognition hasn't been called or it failed.
                Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition");
                return STATUS_ERROR;
            }
            if (mActiveListener.asBinder() != listener.asBinder()) {
                // We don't allow a different listener to stop the recognition than the one
                // that started it.
                Slog.w(TAG, "Attempting stopRecognition for another recognition");
                return STATUS_ERROR;
            }

            // Stop recognition if it's the current one, ignore otherwise.
            mRequested = false;
            int status = updateRecognitionLocked(false /* don't notify for synchronous calls */);
            if (status != SoundTrigger.STATUS_OK) {
                return status;
            }

            // We leave the sound model loaded but not started, this helps us when we start
            // back.
            // Also clear the internal state once the recognition has been stopped.
            internalClearStateLocked();
            return status;
        }
    }

    /**
     * Stops all recognitions active currently and clears the internal state.
     */
    void stopAllRecognitions() {
        synchronized (mLock) {
            if (moduleProperties == null || mModule == null) {
                return;
            }

            if (mCurrentSoundModelHandle == INVALID_VALUE) {
                return;
            }

            mRequested = false;
            int status = updateRecognitionLocked(false /* don't notify for synchronous calls */);
            internalClearStateLocked();
        }
    }

    //---- SoundTrigger.StatusListener methods
    @Override
    public void onRecognition(RecognitionEvent event) {
        if (event == null || !(event instanceof KeyphraseRecognitionEvent)) {
            Slog.w(TAG, "Invalid recognition event!");
            return;
        }

        if (DBG) Slog.d(TAG, "onRecognition: " + event);
        synchronized (mLock) {
            if (mActiveListener == null) {
                Slog.w(TAG, "received onRecognition event without any listener for it");
                return;
            }
            switch (event.status) {
                // Fire aborts/failures to all listeners since it's not tied to a keyphrase.
                case SoundTrigger.RECOGNITION_STATUS_ABORT:
                    onRecognitionAbortLocked();
                    break;
                case SoundTrigger.RECOGNITION_STATUS_FAILURE:
                    onRecognitionFailureLocked();
                    break;
                case SoundTrigger.RECOGNITION_STATUS_SUCCESS:
                    onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event);
                    break;
            }
        }
    }

    @Override
    public void onSoundModelUpdate(SoundModelEvent event) {
        if (event == null) {
            Slog.w(TAG, "Invalid sound model event!");
            return;
        }
        if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event);
        synchronized (mLock) {
            onSoundModelUpdatedLocked(event);
        }
    }

    @Override
    public void onServiceStateChange(int state) {
        if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state);
        synchronized (mLock) {
            onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state);
        }
    }

    @Override
    public void onServiceDied() {
        Slog.e(TAG, "onServiceDied!!");
        synchronized (mLock) {
            onServiceDiedLocked();
        }
    }

    private void onCallStateChangedLocked(boolean callActive) {
        if (mCallActive == callActive) {
            // We consider multiple call states as being active
            // so we check if something really changed or not here.
            return;
        }
        mCallActive = callActive;
        updateRecognitionLocked(true /* notify */);
    }

    private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) {
        if (mIsPowerSaveMode == isPowerSaveMode) {
            return;
        }
        mIsPowerSaveMode = isPowerSaveMode;
        updateRecognitionLocked(true /* notify */);
    }

    private void onSoundModelUpdatedLocked(SoundModelEvent event) {
        // TODO: Handle sound model update here.
    }

    private void onServiceStateChangedLocked(boolean disabled) {
        if (disabled == mServiceDisabled) {
            return;
        }
        mServiceDisabled = disabled;
        updateRecognitionLocked(true /* notify */);
    }

    private void onRecognitionAbortLocked() {
        Slog.w(TAG, "Recognition aborted");
        // No-op
        // This is handled via service state changes instead.
    }

    private void onRecognitionFailureLocked() {
        Slog.w(TAG, "Recognition failure");
        try {
            if (mActiveListener != null) {
                mActiveListener.onError(STATUS_ERROR);
            }
        } catch (RemoteException e) {
            Slog.w(TAG, "RemoteException in onError", e);
        } finally {
            internalClearStateLocked();
        }
    }

    private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) {
        Slog.i(TAG, "Recognition success");
        KeyphraseRecognitionExtra[] keyphraseExtras =
                ((KeyphraseRecognitionEvent) event).keyphraseExtras;
        if (keyphraseExtras == null || keyphraseExtras.length == 0) {
            Slog.w(TAG, "Invalid keyphrase recognition event!");
            return;
        }
        // TODO: Handle more than one keyphrase extras.
        if (mKeyphraseId != keyphraseExtras[0].id) {
            Slog.w(TAG, "received onRecognition event for a different keyphrase");
            return;
        }

        try {
            if (mActiveListener != null) {
                mActiveListener.onDetected((KeyphraseRecognitionEvent) event);
            }
        } catch (RemoteException e) {
            Slog.w(TAG, "RemoteException in onDetected", e);
        }

        mStarted = false;
        mRequested = mRecognitionConfig.allowMultipleTriggers;
        // TODO: Remove this block if the lower layer supports multiple triggers.
        if (mRequested) {
            updateRecognitionLocked(true /* notify */);
        }
    }

    private void onServiceDiedLocked() {
        try {
            if (mActiveListener != null) {
                mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT);
            }
        } catch (RemoteException e) {
            Slog.w(TAG, "RemoteException in onError", e);
        } finally {
            internalClearSoundModelLocked();
            internalClearStateLocked();
            if (mModule != null) {
                mModule.detach();
                mModule = null;
            }
        }
    }

    private int updateRecognitionLocked(boolean notify) {
        if (mModule == null || moduleProperties == null
                || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) {
            // Nothing to do here.
            return STATUS_OK;
        }

        boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode;
        if (start == mStarted) {
            // No-op.
            return STATUS_OK;
        }

        // See if the recognition needs to be started.
        if (start) {
            // Start recognition.
            int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig);
            if (status != SoundTrigger.STATUS_OK) {
                Slog.w(TAG, "startRecognition failed with " + status);
                // Notify of error if needed.
                if (notify) {
                    try {
                        mActiveListener.onError(status);
                    } catch (RemoteException e) {
                        Slog.w(TAG, "RemoteException in onError", e);
                    }
                }
            } else {
                mStarted = true;
                // Notify of resume if needed.
                if (notify) {
                    try {
                        mActiveListener.onRecognitionResumed();
                    } catch (RemoteException e) {
                        Slog.w(TAG, "RemoteException in onRecognitionResumed", e);
                    }
                }
            }
            return status;
        } else {
            // Stop recognition.
            int status = mModule.stopRecognition(mCurrentSoundModelHandle);
            if (status != SoundTrigger.STATUS_OK) {
                Slog.w(TAG, "stopRecognition call failed with " + status);
                if (notify) {
                    try {
                        mActiveListener.onError(status);
                    } catch (RemoteException e) {
                        Slog.w(TAG, "RemoteException in onError", e);
                    }
                }
            } else {
                mStarted = false;
                // Notify of pause if needed.
                if (notify) {
                    try {
                        mActiveListener.onRecognitionPaused();
                    } catch (RemoteException e) {
                        Slog.w(TAG, "RemoteException in onRecognitionPaused", e);
                    }
                }
            }
            return status;
        }
    }

    private void internalClearStateLocked() {
        mStarted = false;
        mRequested = false;

        mKeyphraseId = INVALID_VALUE;
        mRecognitionConfig = null;
        mActiveListener = null;

        // Unregister from call state changes.
        mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE);

        // Unregister from power save mode changes.
        if (mPowerSaveModeListener != null) {
            mContext.unregisterReceiver(mPowerSaveModeListener);
            mPowerSaveModeListener = null;
        }
    }

    private void internalClearSoundModelLocked() {
        mCurrentSoundModelHandle = INVALID_VALUE;
        mCurrentSoundModel = null;
    }

    class MyCallStateListener extends PhoneStateListener {
        @Override
        public void onCallStateChanged(int state, String arg1) {
            if (DBG) Slog.d(TAG, "onCallStateChanged: " + state);
            synchronized (mLock) {
                onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state);
            }
        }
    }

    class PowerSaveModeListener extends BroadcastReceiver {
        @Override
        public void onReceive(Context context, Intent intent) {
            if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) {
                return;
            }
            boolean active = mPowerManager.isPowerSaveMode();
            if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active);
            synchronized (mLock) {
                onPowerSaveModeChangedLocked(active);
            }
        }
    }

    void dump(FileDescriptor fd, PrintWriter pw, String[] args) {
        synchronized (mLock) {
            pw.print("  module properties=");
            pw.println(moduleProperties == null ? "null" : moduleProperties);
            pw.print("  keyphrase ID="); pw.println(mKeyphraseId);
            pw.print("  sound model handle="); pw.println(mCurrentSoundModelHandle);
            pw.print("  sound model UUID=");
            pw.println(mCurrentSoundModel == null ? "null" : mCurrentSoundModel.uuid);
            pw.print("  current listener=");
            pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder());

            pw.print("  requested="); pw.println(mRequested);
            pw.print("  started="); pw.println(mStarted);
            pw.print("  call active="); pw.println(mCallActive);
            pw.print("  power save mode active="); pw.println(mIsPowerSaveMode);
            pw.print("  service disabled="); pw.println(mServiceDisabled);
        }
    }
}