/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.app; import android.annotation.NonNull; import android.annotation.Nullable; import android.content.Context; import android.os.Bundle; import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.os.Parcel; import android.os.Parcelable; import android.os.RemoteException; import android.util.ArrayMap; import android.util.DebugUtils; import android.util.Log; import com.android.internal.app.IVoiceInteractor; import com.android.internal.app.IVoiceInteractorCallback; import com.android.internal.app.IVoiceInteractorRequest; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; /** * Interface for an {@link Activity} to interact with the user through voice. Use * {@link android.app.Activity#getVoiceInteractor() Activity.getVoiceInteractor} * to retrieve the interface, if the activity is currently involved in a voice interaction. * *

The voice interactor revolves around submitting voice interaction requests to the * back-end voice interaction service that is working with the user. These requests are * submitted with {@link #submitRequest}, providing a new instance of a * {@link Request} subclass describing the type of operation to perform -- currently the * possible requests are {@link ConfirmationRequest} and {@link CommandRequest}. * *

Once a request is submitted, the voice system will process it and eventually deliver * the result to the request object. The application can cancel a pending request at any * time. * *

The VoiceInteractor is integrated with Activity's state saving mechanism, so that * if an activity is being restarted with retained state, it will retain the current * VoiceInteractor and any outstanding requests. Because of this, you should always use * {@link Request#getActivity() Request.getActivity} to get back to the activity of a * request, rather than holding on to the activity instance yourself, either explicitly * or implicitly through a non-static inner class. */ public final class VoiceInteractor { static final String TAG = "VoiceInteractor"; static final boolean DEBUG = false; static final Request[] NO_REQUESTS = new Request[0]; final IVoiceInteractor mInteractor; Context mContext; Activity mActivity; boolean mRetaining; final HandlerCaller mHandlerCaller; final HandlerCaller.Callback mHandlerCallerCallback = new HandlerCaller.Callback() { @Override public void executeMessage(Message msg) { SomeArgs args = (SomeArgs)msg.obj; Request request; boolean complete; switch (msg.what) { case MSG_CONFIRMATION_RESULT: request = pullRequest((IVoiceInteractorRequest)args.arg1, true); if (DEBUG) Log.d(TAG, "onConfirmResult: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request + " confirmed=" + msg.arg1 + " result=" + args.arg2); if (request != null) { ((ConfirmationRequest)request).onConfirmationResult(msg.arg1 != 0, (Bundle) args.arg2); request.clear(); } break; case MSG_PICK_OPTION_RESULT: complete = msg.arg1 != 0; request = pullRequest((IVoiceInteractorRequest)args.arg1, complete); if (DEBUG) Log.d(TAG, "onPickOptionResult: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request + " finished=" + complete + " selection=" + args.arg2 + " result=" + args.arg3); if (request != null) { ((PickOptionRequest)request).onPickOptionResult(complete, (PickOptionRequest.Option[]) args.arg2, (Bundle) args.arg3); if (complete) { request.clear(); } } break; case MSG_COMPLETE_VOICE_RESULT: request = pullRequest((IVoiceInteractorRequest)args.arg1, true); if (DEBUG) Log.d(TAG, "onCompleteVoice: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request + " result=" + args.arg2); if (request != null) { ((CompleteVoiceRequest)request).onCompleteResult((Bundle) args.arg2); request.clear(); } break; case MSG_ABORT_VOICE_RESULT: request = pullRequest((IVoiceInteractorRequest)args.arg1, true); if (DEBUG) Log.d(TAG, "onAbortVoice: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request + " result=" + args.arg2); if (request != null) { ((AbortVoiceRequest)request).onAbortResult((Bundle) args.arg2); request.clear(); } break; case MSG_COMMAND_RESULT: complete = msg.arg1 != 0; request = pullRequest((IVoiceInteractorRequest)args.arg1, complete); if (DEBUG) Log.d(TAG, "onCommandResult: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request + " completed=" + msg.arg1 + " result=" + args.arg2); if (request != null) { ((CommandRequest)request).onCommandResult(msg.arg1 != 0, (Bundle) args.arg2); if (complete) { request.clear(); } } break; case MSG_CANCEL_RESULT: request = pullRequest((IVoiceInteractorRequest)args.arg1, true); if (DEBUG) Log.d(TAG, "onCancelResult: req=" + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request); if (request != null) { request.onCancel(); request.clear(); } break; } } }; final IVoiceInteractorCallback.Stub mCallback = new IVoiceInteractorCallback.Stub() { @Override public void deliverConfirmationResult(IVoiceInteractorRequest request, boolean finished, Bundle result) { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOO( MSG_CONFIRMATION_RESULT, finished ? 1 : 0, request, result)); } @Override public void deliverPickOptionResult(IVoiceInteractorRequest request, boolean finished, PickOptionRequest.Option[] options, Bundle result) { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOOO( MSG_PICK_OPTION_RESULT, finished ? 1 : 0, request, options, result)); } @Override public void deliverCompleteVoiceResult(IVoiceInteractorRequest request, Bundle result) { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( MSG_COMPLETE_VOICE_RESULT, request, result)); } @Override public void deliverAbortVoiceResult(IVoiceInteractorRequest request, Bundle result) { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( MSG_ABORT_VOICE_RESULT, request, result)); } @Override public void deliverCommandResult(IVoiceInteractorRequest request, boolean complete, Bundle result) { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOO( MSG_COMMAND_RESULT, complete ? 1 : 0, request, result)); } @Override public void deliverCancel(IVoiceInteractorRequest request) throws RemoteException { mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( MSG_CANCEL_RESULT, request, null)); } }; final ArrayMap mActiveRequests = new ArrayMap<>(); static final int MSG_CONFIRMATION_RESULT = 1; static final int MSG_PICK_OPTION_RESULT = 2; static final int MSG_COMPLETE_VOICE_RESULT = 3; static final int MSG_ABORT_VOICE_RESULT = 4; static final int MSG_COMMAND_RESULT = 5; static final int MSG_CANCEL_RESULT = 6; /** * Base class for voice interaction requests that can be submitted to the interactor. * Do not instantiate this directly -- instead, use the appropriate subclass. */ public static abstract class Request { IVoiceInteractorRequest mRequestInterface; Context mContext; Activity mActivity; String mName; Request() { } /** * Return the name this request was submitted through * {@link #submitRequest(android.app.VoiceInteractor.Request, String)}. */ public String getName() { return mName; } /** * Cancel this active request. */ public void cancel() { if (mRequestInterface == null) { throw new IllegalStateException("Request " + this + " is no longer active"); } try { mRequestInterface.cancel(); } catch (RemoteException e) { Log.w(TAG, "Voice interactor has died", e); } } /** * Return the current {@link Context} this request is associated with. May change * if the activity hosting it goes through a configuration change. */ public Context getContext() { return mContext; } /** * Return the current {@link Activity} this request is associated with. Will change * if the activity is restarted such as through a configuration change. */ public Activity getActivity() { return mActivity; } /** * Report from voice interaction service: this operation has been canceled, typically * as a completion of a previous call to {@link #cancel} or when the user explicitly * cancelled. */ public void onCancel() { } /** * The request is now attached to an activity, or being re-attached to a new activity * after a configuration change. */ public void onAttached(Activity activity) { } /** * The request is being detached from an activity. */ public void onDetached() { } @Override public String toString() { StringBuilder sb = new StringBuilder(128); DebugUtils.buildShortClassTag(this, sb); sb.append(" "); sb.append(getRequestTypeName()); sb.append(" name="); sb.append(mName); sb.append('}'); return sb.toString(); } void dump(String prefix, FileDescriptor fd, PrintWriter writer, String[] args) { writer.print(prefix); writer.print("mRequestInterface="); writer.println(mRequestInterface.asBinder()); writer.print(prefix); writer.print("mActivity="); writer.println(mActivity); writer.print(prefix); writer.print("mName="); writer.println(mName); } String getRequestTypeName() { return "Request"; } void clear() { mRequestInterface = null; mContext = null; mActivity = null; mName = null; } abstract IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, IVoiceInteractorCallback callback) throws RemoteException; } /** * Confirms an operation with the user via the trusted system * VoiceInteractionService. This allows an Activity to complete an unsafe operation that * would require the user to touch the screen when voice interaction mode is not enabled. * The result of the confirmation will be returned through an asynchronous call to * either {@link #onConfirmationResult(boolean, android.os.Bundle)} or * {@link #onCancel()} - these methods should be overridden to define the application specific * behavior. * *

In some cases this may be a simple yes / no confirmation or the confirmation could * include context information about how the action will be completed * (e.g. booking a cab might include details about how long until the cab arrives) * so the user can give a confirmation. */ public static class ConfirmationRequest extends Request { final Prompt mPrompt; final Bundle mExtras; /** * Create a new confirmation request. * @param prompt Optional confirmation to speak to the user or null if nothing * should be spoken. * @param extras Additional optional information or null. */ public ConfirmationRequest(@Nullable Prompt prompt, @Nullable Bundle extras) { mPrompt = prompt; mExtras = extras; } /** * Create a new confirmation request. * @param prompt Optional confirmation to speak to the user or null if nothing * should be spoken. * @param extras Additional optional information or null. * @hide */ public ConfirmationRequest(CharSequence prompt, Bundle extras) { mPrompt = (prompt != null ? new Prompt(prompt) : null); mExtras = extras; } /** * Handle the confirmation result. Override this method to define * the behavior when the user confirms or rejects the operation. * @param confirmed Whether the user confirmed or rejected the operation. * @param result Additional result information or null. */ public void onConfirmationResult(boolean confirmed, Bundle result) { } void dump(String prefix, FileDescriptor fd, PrintWriter writer, String[] args) { super.dump(prefix, fd, writer, args); writer.print(prefix); writer.print("mPrompt="); writer.println(mPrompt); if (mExtras != null) { writer.print(prefix); writer.print("mExtras="); writer.println(mExtras); } } String getRequestTypeName() { return "Confirmation"; } IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, IVoiceInteractorCallback callback) throws RemoteException { return interactor.startConfirmation(packageName, callback, mPrompt, mExtras); } } /** * Select a single option from multiple potential options with the user via the trusted system * VoiceInteractionService. Typically, the application would present this visually as * a list view to allow selecting the option by touch. * The result of the confirmation will be returned through an asynchronous call to * either {@link #onPickOptionResult} or {@link #onCancel()} - these methods should * be overridden to define the application specific behavior. */ public static class PickOptionRequest extends Request { final Prompt mPrompt; final Option[] mOptions; final Bundle mExtras; /** * Represents a single option that the user may select using their voice. The * {@link #getIndex()} method should be used as a unique ID to identify the option * when it is returned from the voice interactor. */ public static final class Option implements Parcelable { final CharSequence mLabel; final int mIndex; ArrayList mSynonyms; Bundle mExtras; /** * Creates an option that a user can select with their voice by matching the label * or one of several synonyms. * @param label The label that will both be matched against what the user speaks * and displayed visually. * @hide */ public Option(CharSequence label) { mLabel = label; mIndex = -1; } /** * Creates an option that a user can select with their voice by matching the label * or one of several synonyms. * @param label The label that will both be matched against what the user speaks * and displayed visually. * @param index The location of this option within the overall set of options. * Can be used to help identify the option when it is returned from the * voice interactor. */ public Option(CharSequence label, int index) { mLabel = label; mIndex = index; } /** * Add a synonym term to the option to indicate an alternative way the content * may be matched. * @param synonym The synonym that will be matched against what the user speaks, * but not displayed. */ public Option addSynonym(CharSequence synonym) { if (mSynonyms == null) { mSynonyms = new ArrayList<>(); } mSynonyms.add(synonym); return this; } public CharSequence getLabel() { return mLabel; } /** * Return the index that was supplied in the constructor. * If the option was constructed without an index, -1 is returned. */ public int getIndex() { return mIndex; } public int countSynonyms() { return mSynonyms != null ? mSynonyms.size() : 0; } public CharSequence getSynonymAt(int index) { return mSynonyms != null ? mSynonyms.get(index) : null; } /** * Set optional extra information associated with this option. Note that this * method takes ownership of the supplied extras Bundle. */ public void setExtras(Bundle extras) { mExtras = extras; } /** * Return any optional extras information associated with this option, or null * if there is none. Note that this method returns a reference to the actual * extras Bundle in the option, so modifications to it will directly modify the * extras in the option. */ public Bundle getExtras() { return mExtras; } Option(Parcel in) { mLabel = in.readCharSequence(); mIndex = in.readInt(); mSynonyms = in.readCharSequenceList(); mExtras = in.readBundle(); } @Override public int describeContents() { return 0; } @Override public void writeToParcel(Parcel dest, int flags) { dest.writeCharSequence(mLabel); dest.writeInt(mIndex); dest.writeCharSequenceList(mSynonyms); dest.writeBundle(mExtras); } public static final Parcelable.Creator