TextClassification.java revision 20d346eafec9404fb6f5b8eeb9a18ad794b4ca9a
1/* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.view.textclassifier; 18 19import android.annotation.FloatRange; 20import android.annotation.IntDef; 21import android.annotation.IntRange; 22import android.annotation.NonNull; 23import android.annotation.Nullable; 24import android.app.PendingIntent; 25import android.app.RemoteAction; 26import android.content.Context; 27import android.content.Intent; 28import android.content.pm.PackageManager; 29import android.content.pm.ResolveInfo; 30import android.content.res.Resources; 31import android.graphics.Bitmap; 32import android.graphics.Canvas; 33import android.graphics.drawable.BitmapDrawable; 34import android.graphics.drawable.Drawable; 35import android.os.LocaleList; 36import android.os.Parcel; 37import android.os.Parcelable; 38import android.util.ArrayMap; 39import android.view.View.OnClickListener; 40import android.view.textclassifier.TextClassifier.EntityType; 41 42import com.android.internal.util.Preconditions; 43 44import java.lang.annotation.Retention; 45import java.lang.annotation.RetentionPolicy; 46import java.util.ArrayList; 47import java.util.Calendar; 48import java.util.Collections; 49import java.util.List; 50import java.util.Locale; 51import java.util.Map; 52 53/** 54 * Information for generating a widget to handle classified text. 55 * 56 * <p>A TextClassification object contains icons, labels, onClickListeners and intents that may 57 * be used to build a widget that can be used to act on classified text. There is the concept of a 58 * <i>primary action</i> and other <i>secondary actions</i>. 59 * 60 * <p>e.g. building a view that, when clicked, shares the classified text with the preferred app: 61 * 62 * <pre>{@code 63 * // Called preferably outside the UiThread. 64 * TextClassification classification = textClassifier.classifyText(allText, 10, 25); 65 * 66 * // Called on the UiThread. 67 * Button button = new Button(context); 68 * button.setCompoundDrawablesWithIntrinsicBounds(classification.getIcon(), null, null, null); 69 * button.setText(classification.getLabel()); 70 * button.setOnClickListener(v -> context.startActivity(classification.getIntent())); 71 * }</pre> 72 * 73 * <p>e.g. starting an action mode with menu items that can handle the classified text: 74 * 75 * <pre>{@code 76 * // Called preferably outside the UiThread. 77 * final TextClassification classification = textClassifier.classifyText(allText, 10, 25); 78 * 79 * // Called on the UiThread. 80 * view.startActionMode(new ActionMode.Callback() { 81 * 82 * public boolean onCreateActionMode(ActionMode mode, Menu menu) { 83 * for (int i = 0; i < classification.getActions().size(); ++i) { 84 * RemoteAction action = classification.getActions().get(i); 85 * menu.add(Menu.NONE, i, 20, action.getTitle()) 86 * .setIcon(action.getIcon()); 87 * } 88 * return true; 89 * } 90 * 91 * public boolean onActionItemClicked(ActionMode mode, MenuItem item) { 92 * classification.getActions().get(item.getItemId()).getActionIntent().send(); 93 * return true; 94 * } 95 * 96 * ... 97 * }); 98 * }</pre> 99 */ 100public final class TextClassification implements Parcelable { 101 102 /** 103 * @hide 104 */ 105 static final TextClassification EMPTY = new TextClassification.Builder().build(); 106 107 private static final String LOG_TAG = "TextClassification"; 108 // TODO(toki): investigate a way to derive this based on device properties. 109 private static final int MAX_LEGACY_ICON_SIZE = 192; 110 111 @Retention(RetentionPolicy.SOURCE) 112 @IntDef(value = {IntentType.UNSUPPORTED, IntentType.ACTIVITY, IntentType.SERVICE}) 113 private @interface IntentType { 114 int UNSUPPORTED = -1; 115 int ACTIVITY = 0; 116 int SERVICE = 1; 117 } 118 119 @NonNull private final String mText; 120 @Nullable private final Drawable mLegacyIcon; 121 @Nullable private final String mLegacyLabel; 122 @Nullable private final Intent mLegacyIntent; 123 @Nullable private final OnClickListener mLegacyOnClickListener; 124 @NonNull private final List<RemoteAction> mActions; 125 @NonNull private final EntityConfidence mEntityConfidence; 126 @NonNull private final String mSignature; 127 128 private TextClassification( 129 @Nullable String text, 130 @Nullable Drawable legacyIcon, 131 @Nullable String legacyLabel, 132 @Nullable Intent legacyIntent, 133 @Nullable OnClickListener legacyOnClickListener, 134 @NonNull List<RemoteAction> actions, 135 @NonNull Map<String, Float> entityConfidence, 136 @NonNull String signature) { 137 mText = text; 138 mLegacyIcon = legacyIcon; 139 mLegacyLabel = legacyLabel; 140 mLegacyIntent = legacyIntent; 141 mLegacyOnClickListener = legacyOnClickListener; 142 mActions = Collections.unmodifiableList(actions); 143 mEntityConfidence = new EntityConfidence(entityConfidence); 144 mSignature = signature; 145 } 146 147 /** 148 * Gets the classified text. 149 */ 150 @Nullable 151 public String getText() { 152 return mText; 153 } 154 155 /** 156 * Returns the number of entities found in the classified text. 157 */ 158 @IntRange(from = 0) 159 public int getEntityCount() { 160 return mEntityConfidence.getEntities().size(); 161 } 162 163 /** 164 * Returns the entity at the specified index. Entities are ordered from high confidence 165 * to low confidence. 166 * 167 * @throws IndexOutOfBoundsException if the specified index is out of range. 168 * @see #getEntityCount() for the number of entities available. 169 */ 170 @NonNull 171 public @EntityType String getEntity(int index) { 172 return mEntityConfidence.getEntities().get(index); 173 } 174 175 /** 176 * Returns the confidence score for the specified entity. The value ranges from 177 * 0 (low confidence) to 1 (high confidence). 0 indicates that the entity was not found for the 178 * classified text. 179 */ 180 @FloatRange(from = 0.0, to = 1.0) 181 public float getConfidenceScore(@EntityType String entity) { 182 return mEntityConfidence.getConfidenceScore(entity); 183 } 184 185 /** 186 * Returns a list of actions that may be performed on the text. The list is ordered based on 187 * the likelihood that a user will use the action, with the most likely action appearing first. 188 */ 189 public List<RemoteAction> getActions() { 190 return mActions; 191 } 192 193 /** 194 * Returns an icon that may be rendered on a widget used to act on the classified text. 195 * 196 * @deprecated Use {@link #getActions()} instead. 197 */ 198 @Deprecated 199 @Nullable 200 public Drawable getIcon() { 201 return mLegacyIcon; 202 } 203 204 /** 205 * Returns a label that may be rendered on a widget used to act on the classified text. 206 * 207 * @deprecated Use {@link #getActions()} instead. 208 */ 209 @Deprecated 210 @Nullable 211 public CharSequence getLabel() { 212 return mLegacyLabel; 213 } 214 215 /** 216 * Returns an intent that may be fired to act on the classified text. 217 * 218 * @deprecated Use {@link #getActions()} instead. 219 */ 220 @Deprecated 221 @Nullable 222 public Intent getIntent() { 223 return mLegacyIntent; 224 } 225 226 /** 227 * Returns the OnClickListener that may be triggered to act on the classified text. This field 228 * is not parcelable and will be null for all objects read from a parcel. Instead, call 229 * Context#startActivity(Intent) with the result of #getSecondaryIntent(int). Note that this may 230 * fail if the activity doesn't have permission to send the intent. 231 * 232 * @deprecated Use {@link #getActions()} instead. 233 */ 234 @Nullable 235 public OnClickListener getOnClickListener() { 236 return mLegacyOnClickListener; 237 } 238 239 /** 240 * Returns the signature for this object. 241 * The TextClassifier that generates this object may use it as a way to internally identify 242 * this object. 243 */ 244 @NonNull 245 public String getSignature() { 246 return mSignature; 247 } 248 249 @Override 250 public String toString() { 251 return String.format(Locale.US, 252 "TextClassification {text=%s, entities=%s, actions=%s, signature=%s}", 253 mText, mEntityConfidence, mActions, mSignature); 254 } 255 256 /** 257 * Creates an OnClickListener that triggers the specified PendingIntent. 258 * 259 * @hide 260 */ 261 public static OnClickListener createIntentOnClickListener(@NonNull final PendingIntent intent) { 262 Preconditions.checkNotNull(intent); 263 return v -> { 264 try { 265 intent.send(); 266 } catch (PendingIntent.CanceledException e) { 267 Log.e(LOG_TAG, "Error creating OnClickListener from PendingIntent", e); 268 } 269 }; 270 } 271 272 /** 273 * Creates a PendingIntent for the specified intent. 274 * Returns null if the intent is not supported for the specified context. 275 * 276 * @throws IllegalArgumentException if context or intent is null 277 * @hide 278 */ 279 @Nullable 280 public static PendingIntent createPendingIntent( 281 @NonNull final Context context, @NonNull final Intent intent) { 282 switch (getIntentType(intent, context)) { 283 case IntentType.ACTIVITY: 284 return PendingIntent.getActivity(context, 0, intent, 0); 285 case IntentType.SERVICE: 286 return PendingIntent.getService(context, 0, intent, 0); 287 default: 288 return null; 289 } 290 } 291 292 /** 293 * Triggers the specified intent. 294 * 295 * @throws IllegalArgumentException if context or intent is null 296 * @hide 297 */ 298 public static void fireIntent(@NonNull final Context context, @NonNull final Intent intent) { 299 switch (getIntentType(intent, context)) { 300 case IntentType.ACTIVITY: 301 context.startActivity(intent); 302 return; 303 case IntentType.SERVICE: 304 context.startService(intent); 305 return; 306 default: 307 return; 308 } 309 } 310 311 @IntentType 312 private static int getIntentType(@NonNull Intent intent, @NonNull Context context) { 313 Preconditions.checkArgument(context != null); 314 Preconditions.checkArgument(intent != null); 315 316 final ResolveInfo activityRI = context.getPackageManager().resolveActivity(intent, 0); 317 if (activityRI != null) { 318 if (context.getPackageName().equals(activityRI.activityInfo.packageName)) { 319 return IntentType.ACTIVITY; 320 } 321 final boolean exported = activityRI.activityInfo.exported; 322 if (exported && hasPermission(context, activityRI.activityInfo.permission)) { 323 return IntentType.ACTIVITY; 324 } 325 } 326 327 final ResolveInfo serviceRI = context.getPackageManager().resolveService(intent, 0); 328 if (serviceRI != null) { 329 if (context.getPackageName().equals(serviceRI.serviceInfo.packageName)) { 330 return IntentType.SERVICE; 331 } 332 final boolean exported = serviceRI.serviceInfo.exported; 333 if (exported && hasPermission(context, serviceRI.serviceInfo.permission)) { 334 return IntentType.SERVICE; 335 } 336 } 337 338 return IntentType.UNSUPPORTED; 339 } 340 341 private static boolean hasPermission(@NonNull Context context, @NonNull String permission) { 342 return permission == null 343 || context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED; 344 } 345 346 /** 347 * Returns a Bitmap representation of the Drawable 348 * 349 * @param drawable The drawable to convert. 350 * @param maxDims The maximum edge length of the resulting bitmap (in pixels). 351 */ 352 @Nullable 353 private static Bitmap drawableToBitmap(@Nullable Drawable drawable, int maxDims) { 354 if (drawable == null) { 355 return null; 356 } 357 final int actualWidth = Math.max(1, drawable.getIntrinsicWidth()); 358 final int actualHeight = Math.max(1, drawable.getIntrinsicHeight()); 359 final double scaleWidth = ((double) maxDims) / actualWidth; 360 final double scaleHeight = ((double) maxDims) / actualHeight; 361 final double scale = Math.min(1.0, Math.min(scaleWidth, scaleHeight)); 362 final int width = (int) (actualWidth * scale); 363 final int height = (int) (actualHeight * scale); 364 if (drawable instanceof BitmapDrawable) { 365 final BitmapDrawable bitmapDrawable = (BitmapDrawable) drawable; 366 if (actualWidth != width || actualHeight != height) { 367 return Bitmap.createScaledBitmap( 368 bitmapDrawable.getBitmap(), width, height, /*filter=*/false); 369 } else { 370 return bitmapDrawable.getBitmap(); 371 } 372 } else { 373 final Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); 374 final Canvas canvas = new Canvas(bitmap); 375 drawable.setBounds(0, 0, canvas.getWidth(), canvas.getHeight()); 376 drawable.draw(canvas); 377 return bitmap; 378 } 379 } 380 381 /** 382 * Builder for building {@link TextClassification} objects. 383 * 384 * <p>e.g. 385 * 386 * <pre>{@code 387 * TextClassification classification = new TextClassification.Builder() 388 * .setText(classifiedText) 389 * .setEntityType(TextClassifier.TYPE_EMAIL, 0.9) 390 * .setEntityType(TextClassifier.TYPE_OTHER, 0.1) 391 * .addAction(remoteAction1) 392 * .addAction(remoteAction2) 393 * .build(); 394 * }</pre> 395 */ 396 public static final class Builder { 397 398 @NonNull private String mText; 399 @NonNull private List<RemoteAction> mActions = new ArrayList<>(); 400 @NonNull private final Map<String, Float> mEntityConfidence = new ArrayMap<>(); 401 @Nullable Drawable mLegacyIcon; 402 @Nullable String mLegacyLabel; 403 @Nullable Intent mLegacyIntent; 404 @Nullable OnClickListener mLegacyOnClickListener; 405 @NonNull private String mSignature = ""; 406 407 /** 408 * Sets the classified text. 409 */ 410 public Builder setText(@Nullable String text) { 411 mText = text; 412 return this; 413 } 414 415 /** 416 * Sets an entity type for the classification result and assigns a confidence score. 417 * If a confidence score had already been set for the specified entity type, this will 418 * override that score. 419 * 420 * @param confidenceScore a value from 0 (low confidence) to 1 (high confidence). 421 * 0 implies the entity does not exist for the classified text. 422 * Values greater than 1 are clamped to 1. 423 */ 424 public Builder setEntityType( 425 @NonNull @EntityType String type, 426 @FloatRange(from = 0.0, to = 1.0) float confidenceScore) { 427 mEntityConfidence.put(type, confidenceScore); 428 return this; 429 } 430 431 /** 432 * Adds an action that may be performed on the classified text. Actions should be added in 433 * order of likelihood that the user will use them, with the most likely action being added 434 * first. 435 */ 436 public Builder addAction(@NonNull RemoteAction action) { 437 Preconditions.checkArgument(action != null); 438 mActions.add(action); 439 return this; 440 } 441 442 /** 443 * Sets the icon for the <i>primary</i> action that may be rendered on a widget used to act 444 * on the classified text. 445 * 446 * @deprecated Use {@link #addAction(RemoteAction)} instead. 447 */ 448 @Deprecated 449 public Builder setIcon(@Nullable Drawable icon) { 450 mLegacyIcon = icon; 451 return this; 452 } 453 454 /** 455 * Sets the label for the <i>primary</i> action that may be rendered on a widget used to 456 * act on the classified text. 457 * 458 * @deprecated Use {@link #addAction(RemoteAction)} instead. 459 */ 460 @Deprecated 461 public Builder setLabel(@Nullable String label) { 462 mLegacyLabel = label; 463 return this; 464 } 465 466 /** 467 * Sets the intent for the <i>primary</i> action that may be fired to act on the classified 468 * text. 469 * 470 * @deprecated Use {@link #addAction(RemoteAction)} instead. 471 */ 472 @Deprecated 473 public Builder setIntent(@Nullable Intent intent) { 474 mLegacyIntent = intent; 475 return this; 476 } 477 478 /** 479 * Sets the OnClickListener for the <i>primary</i> action that may be triggered to act on 480 * the classified text. This field is not parcelable and will always be null when the 481 * object is read from a parcel. 482 * 483 * @deprecated Use {@link #addAction(RemoteAction)} instead. 484 */ 485 public Builder setOnClickListener(@Nullable OnClickListener onClickListener) { 486 mLegacyOnClickListener = onClickListener; 487 return this; 488 } 489 490 /** 491 * Sets a signature for the TextClassification object. 492 * The TextClassifier that generates the TextClassification object may use it as a way to 493 * internally identify the TextClassification object. 494 */ 495 public Builder setSignature(@NonNull String signature) { 496 mSignature = Preconditions.checkNotNull(signature); 497 return this; 498 } 499 500 /** 501 * Builds and returns a {@link TextClassification} object. 502 */ 503 public TextClassification build() { 504 return new TextClassification(mText, mLegacyIcon, mLegacyLabel, mLegacyIntent, 505 mLegacyOnClickListener, mActions, mEntityConfidence, mSignature); 506 } 507 } 508 509 /** 510 * Optional input parameters for generating TextClassification. 511 */ 512 public static final class Options implements Parcelable { 513 514 private @Nullable LocaleList mDefaultLocales; 515 private @Nullable Calendar mReferenceTime; 516 517 public Options() {} 518 519 /** 520 * @param defaultLocales ordered list of locale preferences that may be used to disambiguate 521 * the provided text. If no locale preferences exist, set this to null or an empty 522 * locale list. 523 */ 524 public Options setDefaultLocales(@Nullable LocaleList defaultLocales) { 525 mDefaultLocales = defaultLocales; 526 return this; 527 } 528 529 /** 530 * @param referenceTime reference time based on which relative dates (e.g. "tomorrow" should 531 * be interpreted. This should usually be the time when the text was originally 532 * composed. If no reference time is set, now is used. 533 */ 534 public Options setReferenceTime(Calendar referenceTime) { 535 mReferenceTime = referenceTime; 536 return this; 537 } 538 539 /** 540 * @return ordered list of locale preferences that can be used to disambiguate 541 * the provided text. 542 */ 543 @Nullable 544 public LocaleList getDefaultLocales() { 545 return mDefaultLocales; 546 } 547 548 /** 549 * @return reference time based on which relative dates (e.g. "tomorrow") should be 550 * interpreted. 551 */ 552 @Nullable 553 public Calendar getReferenceTime() { 554 return mReferenceTime; 555 } 556 557 @Override 558 public int describeContents() { 559 return 0; 560 } 561 562 @Override 563 public void writeToParcel(Parcel dest, int flags) { 564 dest.writeInt(mDefaultLocales != null ? 1 : 0); 565 if (mDefaultLocales != null) { 566 mDefaultLocales.writeToParcel(dest, flags); 567 } 568 dest.writeInt(mReferenceTime != null ? 1 : 0); 569 if (mReferenceTime != null) { 570 dest.writeSerializable(mReferenceTime); 571 } 572 } 573 574 public static final Parcelable.Creator<Options> CREATOR = 575 new Parcelable.Creator<Options>() { 576 @Override 577 public Options createFromParcel(Parcel in) { 578 return new Options(in); 579 } 580 581 @Override 582 public Options[] newArray(int size) { 583 return new Options[size]; 584 } 585 }; 586 587 private Options(Parcel in) { 588 if (in.readInt() > 0) { 589 mDefaultLocales = LocaleList.CREATOR.createFromParcel(in); 590 } 591 if (in.readInt() > 0) { 592 mReferenceTime = (Calendar) in.readSerializable(); 593 } 594 } 595 } 596 597 @Override 598 public int describeContents() { 599 return 0; 600 } 601 602 @Override 603 public void writeToParcel(Parcel dest, int flags) { 604 dest.writeString(mText); 605 final Bitmap legacyIconBitmap = drawableToBitmap(mLegacyIcon, MAX_LEGACY_ICON_SIZE); 606 dest.writeInt(legacyIconBitmap != null ? 1 : 0); 607 if (legacyIconBitmap != null) { 608 legacyIconBitmap.writeToParcel(dest, flags); 609 } 610 dest.writeString(mLegacyLabel); 611 dest.writeInt(mLegacyIntent != null ? 1 : 0); 612 if (mLegacyIntent != null) { 613 mLegacyIntent.writeToParcel(dest, flags); 614 } 615 // mOnClickListener is not parcelable. 616 dest.writeTypedList(mActions); 617 mEntityConfidence.writeToParcel(dest, flags); 618 dest.writeString(mSignature); 619 } 620 621 public static final Parcelable.Creator<TextClassification> CREATOR = 622 new Parcelable.Creator<TextClassification>() { 623 @Override 624 public TextClassification createFromParcel(Parcel in) { 625 return new TextClassification(in); 626 } 627 628 @Override 629 public TextClassification[] newArray(int size) { 630 return new TextClassification[size]; 631 } 632 }; 633 634 private TextClassification(Parcel in) { 635 mText = in.readString(); 636 mLegacyIcon = in.readInt() == 0 637 ? null 638 : new BitmapDrawable(Resources.getSystem(), Bitmap.CREATOR.createFromParcel(in)); 639 mLegacyLabel = in.readString(); 640 if (in.readInt() == 0) { 641 mLegacyIntent = null; 642 } else { 643 mLegacyIntent = Intent.CREATOR.createFromParcel(in); 644 mLegacyIntent.removeFlags( 645 Intent.FLAG_GRANT_READ_URI_PERMISSION | Intent.FLAG_GRANT_WRITE_URI_PERMISSION); 646 } 647 mLegacyOnClickListener = null; // not parcelable 648 mActions = in.createTypedArrayList(RemoteAction.CREATOR); 649 mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); 650 mSignature = in.readString(); 651 } 652} 653