VoiceInteractor.java revision 4870e9d5eba59fb257a87f97f1adf0b734cf48d3
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.app; 18 19import android.content.Context; 20import android.os.Bundle; 21import android.os.IBinder; 22import android.os.Looper; 23import android.os.Message; 24import android.os.Parcel; 25import android.os.Parcelable; 26import android.os.RemoteException; 27import android.util.ArrayMap; 28import android.util.Log; 29import com.android.internal.app.IVoiceInteractor; 30import com.android.internal.app.IVoiceInteractorCallback; 31import com.android.internal.app.IVoiceInteractorRequest; 32import com.android.internal.os.HandlerCaller; 33import com.android.internal.os.SomeArgs; 34 35import java.util.ArrayList; 36 37/** 38 * Interface for an {@link Activity} to interact with the user through voice. Use 39 * {@link android.app.Activity#getVoiceInteractor() Activity.getVoiceInteractor} 40 * to retrieve the interface, if the activity is currently involved in a voice interaction. 41 * 42 * <p>The voice interactor revolves around submitting voice interaction requests to the 43 * back-end voice interaction service that is working with the user. These requests are 44 * submitted with {@link #submitRequest}, providing a new instance of a 45 * {@link Request} subclass describing the type of operation to perform -- currently the 46 * possible requests are {@link ConfirmationRequest} and {@link CommandRequest}. 47 * 48 * <p>Once a request is submitted, the voice system will process it and eventually deliver 49 * the result to the request object. The application can cancel a pending request at any 50 * time. 51 * 52 * <p>The VoiceInteractor is integrated with Activity's state saving mechanism, so that 53 * if an activity is being restarted with retained state, it will retain the current 54 * VoiceInteractor and any outstanding requests. Because of this, you should always use 55 * {@link Request#getActivity() Request.getActivity} to get back to the activity of a 56 * request, rather than holding on to the activity instance yourself, either explicitly 57 * or implicitly through a non-static inner class. 58 */ 59public class VoiceInteractor { 60 static final String TAG = "VoiceInteractor"; 61 static final boolean DEBUG = true; 62 63 final IVoiceInteractor mInteractor; 64 65 Context mContext; 66 Activity mActivity; 67 68 final HandlerCaller mHandlerCaller; 69 final HandlerCaller.Callback mHandlerCallerCallback = new HandlerCaller.Callback() { 70 @Override 71 public void executeMessage(Message msg) { 72 SomeArgs args = (SomeArgs)msg.obj; 73 Request request; 74 boolean complete; 75 switch (msg.what) { 76 case MSG_CONFIRMATION_RESULT: 77 request = pullRequest((IVoiceInteractorRequest)args.arg1, true); 78 if (DEBUG) Log.d(TAG, "onConfirmResult: req=" 79 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request 80 + " confirmed=" + msg.arg1 + " result=" + args.arg2); 81 if (request != null) { 82 ((ConfirmationRequest)request).onConfirmationResult(msg.arg1 != 0, 83 (Bundle) args.arg2); 84 request.clear(); 85 } 86 break; 87 case MSG_PICK_OPTION_RESULT: 88 complete = msg.arg1 != 0; 89 request = pullRequest((IVoiceInteractorRequest)args.arg1, complete); 90 if (DEBUG) Log.d(TAG, "onPickOptionResult: req=" 91 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request 92 + " finished=" + complete + " selection=" + args.arg2 93 + " result=" + args.arg3); 94 if (request != null) { 95 ((PickOptionRequest)request).onPickOptionResult(complete, 96 (PickOptionRequest.Option[]) args.arg2, (Bundle) args.arg3); 97 if (complete) { 98 request.clear(); 99 } 100 } 101 break; 102 case MSG_COMPLETE_VOICE_RESULT: 103 request = pullRequest((IVoiceInteractorRequest)args.arg1, true); 104 if (DEBUG) Log.d(TAG, "onCompleteVoice: req=" 105 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request 106 + " result=" + args.arg2); 107 if (request != null) { 108 ((CompleteVoiceRequest)request).onCompleteResult((Bundle) args.arg2); 109 request.clear(); 110 } 111 break; 112 case MSG_ABORT_VOICE_RESULT: 113 request = pullRequest((IVoiceInteractorRequest)args.arg1, true); 114 if (DEBUG) Log.d(TAG, "onAbortVoice: req=" 115 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request 116 + " result=" + args.arg2); 117 if (request != null) { 118 ((AbortVoiceRequest)request).onAbortResult((Bundle) args.arg2); 119 request.clear(); 120 } 121 break; 122 case MSG_COMMAND_RESULT: 123 complete = msg.arg1 != 0; 124 request = pullRequest((IVoiceInteractorRequest)args.arg1, complete); 125 if (DEBUG) Log.d(TAG, "onCommandResult: req=" 126 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request 127 + " completed=" + msg.arg1 + " result=" + args.arg2); 128 if (request != null) { 129 ((CommandRequest)request).onCommandResult(msg.arg1 != 0, 130 (Bundle) args.arg2); 131 if (complete) { 132 request.clear(); 133 } 134 } 135 break; 136 case MSG_CANCEL_RESULT: 137 request = pullRequest((IVoiceInteractorRequest)args.arg1, true); 138 if (DEBUG) Log.d(TAG, "onCancelResult: req=" 139 + ((IVoiceInteractorRequest)args.arg1).asBinder() + "/" + request); 140 if (request != null) { 141 request.onCancel(); 142 request.clear(); 143 } 144 break; 145 } 146 } 147 }; 148 149 final IVoiceInteractorCallback.Stub mCallback = new IVoiceInteractorCallback.Stub() { 150 @Override 151 public void deliverConfirmationResult(IVoiceInteractorRequest request, boolean finished, 152 Bundle result) { 153 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOO( 154 MSG_CONFIRMATION_RESULT, finished ? 1 : 0, request, result)); 155 } 156 157 @Override 158 public void deliverPickOptionResult(IVoiceInteractorRequest request, 159 boolean finished, PickOptionRequest.Option[] options, Bundle result) { 160 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOOO( 161 MSG_PICK_OPTION_RESULT, finished ? 1 : 0, request, options, result)); 162 } 163 164 @Override 165 public void deliverCompleteVoiceResult(IVoiceInteractorRequest request, Bundle result) { 166 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( 167 MSG_COMPLETE_VOICE_RESULT, request, result)); 168 } 169 170 @Override 171 public void deliverAbortVoiceResult(IVoiceInteractorRequest request, Bundle result) { 172 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( 173 MSG_ABORT_VOICE_RESULT, request, result)); 174 } 175 176 @Override 177 public void deliverCommandResult(IVoiceInteractorRequest request, boolean complete, 178 Bundle result) { 179 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOO( 180 MSG_COMMAND_RESULT, complete ? 1 : 0, request, result)); 181 } 182 183 @Override 184 public void deliverCancel(IVoiceInteractorRequest request) throws RemoteException { 185 mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO( 186 MSG_CANCEL_RESULT, request, null)); 187 } 188 }; 189 190 final ArrayMap<IBinder, Request> mActiveRequests = new ArrayMap<IBinder, Request>(); 191 192 static final int MSG_CONFIRMATION_RESULT = 1; 193 static final int MSG_PICK_OPTION_RESULT = 2; 194 static final int MSG_COMPLETE_VOICE_RESULT = 3; 195 static final int MSG_ABORT_VOICE_RESULT = 4; 196 static final int MSG_COMMAND_RESULT = 5; 197 static final int MSG_CANCEL_RESULT = 6; 198 199 /** 200 * Base class for voice interaction requests that can be submitted to the interactor. 201 * Do not instantiate this directly -- instead, use the appropriate subclass. 202 */ 203 public static abstract class Request { 204 IVoiceInteractorRequest mRequestInterface; 205 Context mContext; 206 Activity mActivity; 207 208 Request() { 209 } 210 211 public void cancel() { 212 try { 213 mRequestInterface.cancel(); 214 } catch (RemoteException e) { 215 Log.w(TAG, "Voice interactor has died", e); 216 } 217 } 218 219 public Context getContext() { 220 return mContext; 221 } 222 223 public Activity getActivity() { 224 return mActivity; 225 } 226 227 public void onCancel() { 228 } 229 230 public void onAttached(Activity activity) { 231 } 232 233 public void onDetached() { 234 } 235 236 void clear() { 237 mRequestInterface = null; 238 mContext = null; 239 mActivity = null; 240 } 241 242 abstract IVoiceInteractorRequest submit(IVoiceInteractor interactor, 243 String packageName, IVoiceInteractorCallback callback) throws RemoteException; 244 } 245 246 /** 247 * Confirms an operation with the user via the trusted system 248 * VoiceInteractionService. This allows an Activity to complete an unsafe operation that 249 * would require the user to touch the screen when voice interaction mode is not enabled. 250 * The result of the confirmation will be returned through an asynchronous call to 251 * either {@link #onConfirmationResult(boolean, android.os.Bundle)} or 252 * {@link #onCancel()}. 253 * 254 * <p>In some cases this may be a simple yes / no confirmation or the confirmation could 255 * include context information about how the action will be completed 256 * (e.g. booking a cab might include details about how long until the cab arrives) 257 * so the user can give a confirmation. 258 */ 259 public static class ConfirmationRequest extends Request { 260 final CharSequence mPrompt; 261 final Bundle mExtras; 262 263 /** 264 * Create a new confirmation request. 265 * @param prompt Optional confirmation text to read to the user as the action being 266 * confirmed. 267 * @param extras Additional optional information. 268 */ 269 public ConfirmationRequest(CharSequence prompt, Bundle extras) { 270 mPrompt = prompt; 271 mExtras = extras; 272 } 273 274 public void onConfirmationResult(boolean confirmed, Bundle result) { 275 } 276 277 IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, 278 IVoiceInteractorCallback callback) throws RemoteException { 279 return interactor.startConfirmation(packageName, callback, mPrompt, mExtras); 280 } 281 } 282 283 /** 284 * Select a single option from multiple potential options with the user via the trusted system 285 * VoiceInteractionService. Typically, the application would present this visually as 286 * a list view to allow selecting the option by touch. 287 * The result of the confirmation will be returned through an asynchronous call to 288 * either {@link #onPickOptionResult} or {@link #onCancel()}. 289 */ 290 public static class PickOptionRequest extends Request { 291 final CharSequence mPrompt; 292 final Option[] mOptions; 293 final Bundle mExtras; 294 295 /** 296 * Represents a single option that the user may select using their voice. 297 */ 298 public static final class Option implements Parcelable { 299 final CharSequence mLabel; 300 final int mIndex; 301 ArrayList<CharSequence> mSynonyms; 302 Bundle mExtras; 303 304 /** 305 * Creates an option that a user can select with their voice by matching the label 306 * or one of several synonyms. 307 * @param label The label that will both be matched against what the user speaks 308 * and displayed visually. 309 */ 310 public Option(CharSequence label) { 311 mLabel = label; 312 mIndex = -1; 313 } 314 315 /** 316 * Creates an option that a user can select with their voice by matching the label 317 * or one of several synonyms. 318 * @param label The label that will both be matched against what the user speaks 319 * and displayed visually. 320 * @param index The location of this option within the overall set of options. 321 * Can be used to help identify the option when it is returned from the 322 * voice interactor. 323 */ 324 public Option(CharSequence label, int index) { 325 mLabel = label; 326 mIndex = index; 327 } 328 329 /** 330 * Add a synonym term to the option to indicate an alternative way the content 331 * may be matched. 332 * @param synonym The synonym that will be matched against what the user speaks, 333 * but not displayed. 334 */ 335 public Option addSynonym(CharSequence synonym) { 336 if (mSynonyms == null) { 337 mSynonyms = new ArrayList<>(); 338 } 339 mSynonyms.add(synonym); 340 return this; 341 } 342 343 public CharSequence getLabel() { 344 return mLabel; 345 } 346 347 /** 348 * Return the index that was supplied in the constructor. 349 * If the option was constructed without an index, -1 is returned. 350 */ 351 public int getIndex() { 352 return mIndex; 353 } 354 355 public int countSynonyms() { 356 return mSynonyms != null ? mSynonyms.size() : 0; 357 } 358 359 public CharSequence getSynonymAt(int index) { 360 return mSynonyms != null ? mSynonyms.get(index) : null; 361 } 362 363 /** 364 * Set optional extra information associated with this option. Note that this 365 * method takes ownership of the supplied extras Bundle. 366 */ 367 public void setExtras(Bundle extras) { 368 mExtras = extras; 369 } 370 371 /** 372 * Return any optional extras information associated with this option, or null 373 * if there is none. Note that this method returns a reference to the actual 374 * extras Bundle in the option, so modifications to it will directly modify the 375 * extras in the option. 376 */ 377 public Bundle getExtras() { 378 return mExtras; 379 } 380 381 Option(Parcel in) { 382 mLabel = in.readCharSequence(); 383 mIndex = in.readInt(); 384 mSynonyms = in.readCharSequenceList(); 385 mExtras = in.readBundle(); 386 } 387 388 @Override 389 public int describeContents() { 390 return 0; 391 } 392 393 @Override 394 public void writeToParcel(Parcel dest, int flags) { 395 dest.writeCharSequence(mLabel); 396 dest.writeInt(mIndex); 397 dest.writeCharSequenceList(mSynonyms); 398 dest.writeBundle(mExtras); 399 } 400 401 public static final Parcelable.Creator<Option> CREATOR 402 = new Parcelable.Creator<Option>() { 403 public Option createFromParcel(Parcel in) { 404 return new Option(in); 405 } 406 407 public Option[] newArray(int size) { 408 return new Option[size]; 409 } 410 }; 411 }; 412 413 /** 414 * Create a new pick option request. 415 * @param prompt Optional question to be spoken to the user via text to speech. 416 * @param options The set of {@link Option}s the user is selecting from. 417 * @param extras Additional optional information. 418 */ 419 public PickOptionRequest(CharSequence prompt, Option[] options, Bundle extras) { 420 mPrompt = prompt; 421 mOptions = options; 422 mExtras = extras; 423 } 424 425 /** 426 * Called when a single option is confirmed or narrowed to one of several options. 427 * @param finished True if the voice interaction has finished making a selection, in 428 * which case {@code selections} contains the final result. If false, this request is 429 * still active and you will continue to get calls on it. 430 * @param selections Either a single {@link Option} or one of several {@link Option}s the 431 * user has narrowed the choices down to. 432 * @param result Additional optional information. 433 */ 434 public void onPickOptionResult(boolean finished, Option[] selections, Bundle result) { 435 } 436 437 IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, 438 IVoiceInteractorCallback callback) throws RemoteException { 439 return interactor.startPickOption(packageName, callback, mPrompt, mOptions, mExtras); 440 } 441 } 442 443 /** 444 * Reports that the current interaction was successfully completed with voice, so the 445 * application can report the final status to the user. When the response comes back, the 446 * voice system has handled the request and is ready to switch; at that point the 447 * application can start a new non-voice activity or finish. Be sure when starting the new 448 * activity to use {@link android.content.Intent#FLAG_ACTIVITY_NEW_TASK 449 * Intent.FLAG_ACTIVITY_NEW_TASK} to keep the new activity out of the current voice 450 * interaction task. 451 */ 452 public static class CompleteVoiceRequest extends Request { 453 final CharSequence mMessage; 454 final Bundle mExtras; 455 456 /** 457 * Create a new completed voice interaction request. 458 * @param message Optional message to tell user about the completion status of the task. 459 * @param extras Additional optional information. 460 */ 461 public CompleteVoiceRequest(CharSequence message, Bundle extras) { 462 mMessage = message; 463 mExtras = extras; 464 } 465 466 public void onCompleteResult(Bundle result) { 467 } 468 469 IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, 470 IVoiceInteractorCallback callback) throws RemoteException { 471 return interactor.startCompleteVoice(packageName, callback, mMessage, mExtras); 472 } 473 } 474 475 /** 476 * Reports that the current interaction can not be complete with voice, so the 477 * application will need to switch to a traditional input UI. Applications should 478 * only use this when they need to completely bail out of the voice interaction 479 * and switch to a traditional UI. When the response comes back, the voice 480 * system has handled the request and is ready to switch; at that point the application 481 * can start a new non-voice activity. Be sure when starting the new activity 482 * to use {@link android.content.Intent#FLAG_ACTIVITY_NEW_TASK 483 * Intent.FLAG_ACTIVITY_NEW_TASK} to keep the new activity out of the current voice 484 * interaction task. 485 */ 486 public static class AbortVoiceRequest extends Request { 487 final CharSequence mMessage; 488 final Bundle mExtras; 489 490 /** 491 * Create a new voice abort request. 492 * @param message Optional message to tell user about not being able to complete 493 * the interaction with voice. 494 * @param extras Additional optional information. 495 */ 496 public AbortVoiceRequest(CharSequence message, Bundle extras) { 497 mMessage = message; 498 mExtras = extras; 499 } 500 501 public void onAbortResult(Bundle result) { 502 } 503 504 IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, 505 IVoiceInteractorCallback callback) throws RemoteException { 506 return interactor.startAbortVoice(packageName, callback, mMessage, mExtras); 507 } 508 } 509 510 /** 511 * Execute an extended command using the trusted system VoiceInteractionService. 512 * This allows an Activity to request additional information from the user needed to 513 * complete an action (e.g. booking a table might have several possible times that the 514 * user could select from or an app might need the user to agree to a terms of service). 515 * The result of the confirmation will be returned through an asynchronous call to 516 * either {@link #onCommandResult(boolean, android.os.Bundle)} or 517 * {@link #onCancel()}. 518 * 519 * <p>The command is a string that describes the generic operation to be performed. 520 * The command will determine how the properties in extras are interpreted and the set of 521 * available commands is expected to grow over time. An example might be 522 * "com.google.voice.commands.REQUEST_NUMBER_BAGS" to request the number of bags as part of 523 * airline check-in. (This is not an actual working example.) 524 */ 525 public static class CommandRequest extends Request { 526 final String mCommand; 527 final Bundle mArgs; 528 529 /** 530 * Create a new generic command request. 531 * @param command The desired command to perform. 532 * @param args Additional arguments to control execution of the command. 533 */ 534 public CommandRequest(String command, Bundle args) { 535 mCommand = command; 536 mArgs = args; 537 } 538 539 /** 540 * Results for CommandRequest can be returned in partial chunks. 541 * The isCompleted is set to true iff all results have been returned, indicating the 542 * CommandRequest has completed. 543 */ 544 public void onCommandResult(boolean isCompleted, Bundle result) { 545 } 546 547 IVoiceInteractorRequest submit(IVoiceInteractor interactor, String packageName, 548 IVoiceInteractorCallback callback) throws RemoteException { 549 return interactor.startCommand(packageName, callback, mCommand, mArgs); 550 } 551 } 552 553 VoiceInteractor(IVoiceInteractor interactor, Context context, Activity activity, 554 Looper looper) { 555 mInteractor = interactor; 556 mContext = context; 557 mActivity = activity; 558 mHandlerCaller = new HandlerCaller(context, looper, mHandlerCallerCallback, true); 559 } 560 561 Request pullRequest(IVoiceInteractorRequest request, boolean complete) { 562 synchronized (mActiveRequests) { 563 Request req = mActiveRequests.get(request.asBinder()); 564 if (req != null && complete) { 565 mActiveRequests.remove(request.asBinder()); 566 } 567 return req; 568 } 569 } 570 571 private ArrayList<Request> makeRequestList() { 572 final int N = mActiveRequests.size(); 573 if (N < 1) { 574 return null; 575 } 576 ArrayList<Request> list = new ArrayList<Request>(N); 577 for (int i=0; i<N; i++) { 578 list.add(mActiveRequests.valueAt(i)); 579 } 580 return list; 581 } 582 583 void attachActivity(Activity activity) { 584 if (mActivity == activity) { 585 return; 586 } 587 mContext = activity; 588 mActivity = activity; 589 ArrayList<Request> reqs = makeRequestList(); 590 if (reqs != null) { 591 for (int i=0; i<reqs.size(); i++) { 592 Request req = reqs.get(i); 593 req.mContext = activity; 594 req.mActivity = activity; 595 req.onAttached(activity); 596 } 597 } 598 } 599 600 void detachActivity() { 601 ArrayList<Request> reqs = makeRequestList(); 602 if (reqs != null) { 603 for (int i=0; i<reqs.size(); i++) { 604 Request req = reqs.get(i); 605 req.onDetached(); 606 req.mActivity = null; 607 req.mContext = null; 608 } 609 } 610 mContext = null; 611 mActivity = null; 612 } 613 614 public boolean submitRequest(Request request) { 615 try { 616 IVoiceInteractorRequest ireq = request.submit(mInteractor, 617 mContext.getOpPackageName(), mCallback); 618 request.mRequestInterface = ireq; 619 request.mContext = mContext; 620 request.mActivity = mActivity; 621 synchronized (mActiveRequests) { 622 mActiveRequests.put(ireq.asBinder(), request); 623 } 624 return true; 625 } catch (RemoteException e) { 626 Log.w(TAG, "Remove voice interactor service died", e); 627 return false; 628 } 629 } 630 631 /** 632 * Queries the supported commands available from the VoiceinteractionService. 633 * The command is a string that describes the generic operation to be performed. 634 * An example might be "com.google.voice.commands.REQUEST_NUMBER_BAGS" to request the number 635 * of bags as part of airline check-in. (This is not an actual working example.) 636 * 637 * @param commands 638 */ 639 public boolean[] supportsCommands(String[] commands) { 640 try { 641 boolean[] res = mInteractor.supportsCommands(mContext.getOpPackageName(), commands); 642 if (DEBUG) Log.d(TAG, "supportsCommands: cmds=" + commands + " res=" + res); 643 return res; 644 } catch (RemoteException e) { 645 throw new RuntimeException("Voice interactor has died", e); 646 } 647 } 648} 649