TextClassification.java revision a0b48796478dde6d49ec6f1efebbcd280a592a41
1/* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package androidx.textclassifier; 18 19import android.content.Intent; 20import android.graphics.Bitmap; 21import android.graphics.Canvas; 22import android.graphics.drawable.BitmapDrawable; 23import android.graphics.drawable.Drawable; 24import android.os.Parcel; 25import android.os.Parcelable; 26import android.support.annotation.FloatRange; 27import android.support.annotation.IntRange; 28import android.support.annotation.NonNull; 29import android.support.annotation.Nullable; 30import android.support.annotation.RestrictTo; 31import android.support.v4.os.LocaleListCompat; 32import android.support.v4.util.ArrayMap; 33import android.support.v4.util.Preconditions; 34 35import java.util.ArrayList; 36import java.util.Calendar; 37import java.util.List; 38import java.util.Locale; 39import java.util.Map; 40 41import androidx.textclassifier.TextClassifier.EntityType; 42 43/** 44 * Information for generating a widget to handle classified text. 45 * 46 * <p>A TextClassification object contains icons, labels, and intents that may be used to build a 47 * widget that can be used to act on classified text. There is the concept of a <i>primary 48 * action</i> and other <i>secondary actions</i>. 49 * 50 * <p>e.g. building a view that, when clicked, shares the classified text with the preferred app: 51 * 52 * <pre>{@code 53 * // Called preferably outside the UiThread. 54 * TextClassification classification = textClassifier.classifyText(allText, 10, 25); 55 * 56 * // Called on the UiThread. 57 * Button button = new Button(context); 58 * button.setCompoundDrawablesWithIntrinsicBounds(classification.getIcon(), null, null, null); 59 * button.setText(classification.getLabel()); 60 * button.setOnClickListener(v -> context.startActivity(classification.getIntent())); 61 * }</pre> 62 * 63 * TODO: describe how to start action mode for classified text. 64 */ 65public final class TextClassification implements Parcelable { 66 67 /** 68 * @hide 69 */ 70 @RestrictTo(RestrictTo.Scope.LIBRARY) 71 static final TextClassification EMPTY = new TextClassification.Builder().build(); 72 73 // TODO: investigate a way to derive this based on device properties. 74 private static final int MAX_PRIMARY_ICON_SIZE = 192; 75 private static final int MAX_SECONDARY_ICON_SIZE = 144; 76 77 @Nullable private final String mText; 78 @Nullable private final Drawable mPrimaryIcon; 79 @Nullable private final String mPrimaryLabel; 80 @Nullable private final Intent mPrimaryIntent; 81 @NonNull private final List<Drawable> mSecondaryIcons; 82 @NonNull private final List<String> mSecondaryLabels; 83 @NonNull private final List<Intent> mSecondaryIntents; 84 @NonNull private final EntityConfidence mEntityConfidence; 85 @NonNull private final String mSignature; 86 87 private TextClassification( 88 @Nullable String text, 89 @Nullable Drawable primaryIcon, 90 @Nullable String primaryLabel, 91 @Nullable Intent primaryIntent, 92 @NonNull List<Drawable> secondaryIcons, 93 @NonNull List<String> secondaryLabels, 94 @NonNull List<Intent> secondaryIntents, 95 @NonNull Map<String, Float> entityConfidence, 96 @NonNull String signature) { 97 Preconditions.checkArgument(secondaryLabels.size() == secondaryIntents.size()); 98 Preconditions.checkArgument(secondaryIcons.size() == secondaryIntents.size()); 99 mText = text; 100 mPrimaryIcon = primaryIcon; 101 mPrimaryLabel = primaryLabel; 102 mPrimaryIntent = primaryIntent; 103 mSecondaryIcons = secondaryIcons; 104 mSecondaryLabels = secondaryLabels; 105 mSecondaryIntents = secondaryIntents; 106 mEntityConfidence = new EntityConfidence(entityConfidence); 107 mSignature = signature; 108 } 109 110 /** 111 * Gets the classified text. 112 */ 113 @Nullable 114 public String getText() { 115 return mText; 116 } 117 118 /** 119 * Returns the number of entities found in the classified text. 120 */ 121 @IntRange(from = 0) 122 public int getEntityCount() { 123 return mEntityConfidence.getEntities().size(); 124 } 125 126 /** 127 * Returns the entity at the specified index. Entities are ordered from high confidence 128 * to low confidence. 129 * 130 * @throws IndexOutOfBoundsException if the specified index is out of range. 131 * @see #getEntityCount() for the number of entities available. 132 */ 133 @NonNull 134 public @EntityType String getEntity(int index) { 135 return mEntityConfidence.getEntities().get(index); 136 } 137 138 /** 139 * Returns the confidence score for the specified entity. The value ranges from 140 * 0 (low confidence) to 1 (high confidence). 0 indicates that the entity was not found for the 141 * classified text. 142 */ 143 @FloatRange(from = 0.0, to = 1.0) 144 public float getConfidenceScore(@EntityType String entity) { 145 return mEntityConfidence.getConfidenceScore(entity); 146 } 147 148 /** 149 * Returns the number of <i>secondary</i> actions that are available to act on the classified 150 * text. 151 * 152 * <p><strong>Note: </strong> that there may or may not be a <i>primary</i> action. 153 * 154 * @see #getSecondaryIntent(int) 155 * @see #getSecondaryLabel(int) 156 * @see #getSecondaryIcon(int) 157 */ 158 @IntRange(from = 0) 159 public int getSecondaryActionsCount() { 160 return mSecondaryIntents.size(); 161 } 162 163 /** 164 * Returns one of the <i>secondary</i> icons that maybe rendered on a widget used to act on the 165 * classified text. 166 * 167 * @param index Index of the action to get the icon for. 168 * @throws IndexOutOfBoundsException if the specified index is out of range. 169 * @see #getSecondaryActionsCount() for the number of actions available. 170 * @see #getSecondaryIntent(int) 171 * @see #getSecondaryLabel(int) 172 * @see #getIcon() 173 */ 174 @Nullable 175 public Drawable getSecondaryIcon(int index) { 176 return mSecondaryIcons.get(index); 177 } 178 179 /** 180 * Returns an icon for the <i>primary</i> intent that may be rendered on a widget used to act 181 * on the classified text. 182 * 183 * @see #getSecondaryIcon(int) 184 */ 185 @Nullable 186 public Drawable getIcon() { 187 return mPrimaryIcon; 188 } 189 190 /** 191 * Returns one of the <i>secondary</i> labels that may be rendered on a widget used to act on 192 * the classified text. 193 * 194 * @param index Index of the action to get the label for. 195 * @throws IndexOutOfBoundsException if the specified index is out of range. 196 * @see #getSecondaryActionsCount() 197 * @see #getSecondaryIntent(int) 198 * @see #getSecondaryIcon(int) 199 * @see #getLabel() 200 */ 201 @Nullable 202 public CharSequence getSecondaryLabel(int index) { 203 return mSecondaryLabels.get(index); 204 } 205 206 /** 207 * Returns a label for the <i>primary</i> intent that may be rendered on a widget used to act 208 * on the classified text. 209 * 210 * @see #getSecondaryLabel(int) 211 */ 212 @Nullable 213 public CharSequence getLabel() { 214 return mPrimaryLabel; 215 } 216 217 /** 218 * Returns one of the <i>secondary</i> intents that may be fired to act on the classified text. 219 * 220 * @param index Index of the action to get the intent for. 221 * @throws IndexOutOfBoundsException if the specified index is out of range. 222 * @see #getSecondaryActionsCount() 223 * @see #getSecondaryLabel(int) 224 * @see #getSecondaryIcon(int) 225 * @see #getIntent() 226 */ 227 @Nullable 228 public Intent getSecondaryIntent(int index) { 229 return mSecondaryIntents.get(index); 230 } 231 232 /** 233 * Returns the <i>primary</i> intent that may be fired to act on the classified text. 234 * 235 * @see #getSecondaryIntent(int) 236 */ 237 @Nullable 238 public Intent getIntent() { 239 return mPrimaryIntent; 240 } 241 242 /** 243 * Returns the signature for this object. 244 * The TextClassifier that generates this object may use it as a way to internally identify 245 * this object. 246 */ 247 @NonNull 248 public String getSignature() { 249 return mSignature; 250 } 251 252 @Override 253 public String toString() { 254 return String.format(Locale.US, "TextClassification {" 255 + "text=%s, entities=%s, " 256 + "primaryLabel=%s, secondaryLabels=%s, " 257 + "primaryIntent=%s, secondaryIntents=%s, " 258 + "signature=%s}", 259 mText, mEntityConfidence, 260 mPrimaryLabel, mSecondaryLabels, 261 mPrimaryIntent, mSecondaryIntents, 262 mSignature); 263 } 264 265 @Override 266 public int describeContents() { 267 return 0; 268 } 269 270 @Override 271 public void writeToParcel(Parcel dest, int flags) { 272 dest.writeString(mText); 273 final Bitmap primaryIconBitmap = drawableToBitmap(mPrimaryIcon, MAX_PRIMARY_ICON_SIZE); 274 dest.writeInt(primaryIconBitmap != null ? 1 : 0); 275 if (primaryIconBitmap != null) { 276 primaryIconBitmap.writeToParcel(dest, flags); 277 } 278 dest.writeString(mPrimaryLabel); 279 dest.writeInt(mPrimaryIntent != null ? 1 : 0); 280 if (mPrimaryIntent != null) { 281 mPrimaryIntent.writeToParcel(dest, flags); 282 } 283 dest.writeTypedList(drawablesToBitmaps(mSecondaryIcons, MAX_SECONDARY_ICON_SIZE)); 284 dest.writeStringList(mSecondaryLabels); 285 dest.writeTypedList(mSecondaryIntents); 286 mEntityConfidence.writeToParcel(dest, flags); 287 dest.writeString(mSignature); 288 } 289 290 public static final Parcelable.Creator<TextClassification> CREATOR = 291 new Parcelable.Creator<TextClassification>() { 292 @Override 293 public TextClassification createFromParcel(Parcel in) { 294 return new TextClassification(in); 295 } 296 297 @Override 298 public TextClassification[] newArray(int size) { 299 return new TextClassification[size]; 300 } 301 }; 302 303 private TextClassification(Parcel in) { 304 mText = in.readString(); 305 mPrimaryIcon = in.readInt() == 0 306 ? null : new BitmapDrawable(null, Bitmap.CREATOR.createFromParcel(in)); 307 mPrimaryLabel = in.readString(); 308 mPrimaryIntent = in.readInt() == 0 ? null : Intent.CREATOR.createFromParcel(in); 309 mSecondaryIcons = bitmapsToDrawables(in.createTypedArrayList(Bitmap.CREATOR)); 310 mSecondaryLabels = in.createStringArrayList(); 311 mSecondaryIntents = in.createTypedArrayList(Intent.CREATOR); 312 mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); 313 mSignature = in.readString(); 314 } 315 316 /** 317 * Returns a Bitmap representation of the Drawable 318 * 319 * @param drawable The drawable to convert. 320 * @param maxDims The maximum edge length of the resulting bitmap (in pixels). 321 */ 322 @Nullable 323 private static Bitmap drawableToBitmap(@Nullable Drawable drawable, int maxDims) { 324 if (drawable == null) { 325 return null; 326 } 327 final int actualWidth = Math.max(1, drawable.getIntrinsicWidth()); 328 final int actualHeight = Math.max(1, drawable.getIntrinsicHeight()); 329 final double scaleWidth = ((double) maxDims) / actualWidth; 330 final double scaleHeight = ((double) maxDims) / actualHeight; 331 final double scale = Math.min(1.0, Math.min(scaleWidth, scaleHeight)); 332 final int width = (int) (actualWidth * scale); 333 final int height = (int) (actualHeight * scale); 334 if (drawable instanceof BitmapDrawable) { 335 final BitmapDrawable bitmapDrawable = (BitmapDrawable) drawable; 336 if (actualWidth != width || actualHeight != height) { 337 return Bitmap.createScaledBitmap( 338 bitmapDrawable.getBitmap(), width, height, /*filter=*/false); 339 } else { 340 return bitmapDrawable.getBitmap(); 341 } 342 } else { 343 final Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); 344 final Canvas canvas = new Canvas(bitmap); 345 drawable.setBounds(0, 0, canvas.getWidth(), canvas.getHeight()); 346 drawable.draw(canvas); 347 return bitmap; 348 } 349 } 350 351 /** 352 * Returns a list of drawables converted to Bitmaps 353 * 354 * @param drawables The drawables to convert. 355 * @param maxDims The maximum edge length of the resulting bitmaps (in pixels). 356 */ 357 private static List<Bitmap> drawablesToBitmaps(List<Drawable> drawables, int maxDims) { 358 final List<Bitmap> bitmaps = new ArrayList<>(drawables.size()); 359 for (Drawable drawable : drawables) { 360 bitmaps.add(drawableToBitmap(drawable, maxDims)); 361 } 362 return bitmaps; 363 } 364 365 /** Returns a list of drawable wrappers for a list of bitmaps. */ 366 private static List<Drawable> bitmapsToDrawables(List<Bitmap> bitmaps) { 367 final List<Drawable> drawables = new ArrayList<>(bitmaps.size()); 368 for (Bitmap bitmap : bitmaps) { 369 if (bitmap != null) { 370 drawables.add(new BitmapDrawable(null, bitmap)); 371 } else { 372 drawables.add(null); 373 } 374 } 375 return drawables; 376 } 377 378 /** 379 * Builder for building {@link TextClassification} objects. 380 * 381 * <p>e.g. 382 * 383 * <pre>{@code 384 * TextClassification classification = new TextClassification.Builder() 385 * .setText(classifiedText) 386 * .setEntityType(TextClassifier.TYPE_EMAIL, 0.9) 387 * .setEntityType(TextClassifier.TYPE_OTHER, 0.1) 388 * .setPrimaryAction(intent, label, icon) 389 * .addSecondaryAction(intent1, label1, icon1) 390 * .addSecondaryAction(intent2, label2, icon2) 391 * .build(); 392 * }</pre> 393 */ 394 public static final class Builder { 395 396 @NonNull private String mText; 397 @NonNull private final List<Drawable> mSecondaryIcons = new ArrayList<>(); 398 @NonNull private final List<String> mSecondaryLabels = new ArrayList<>(); 399 @NonNull private final List<Intent> mSecondaryIntents = new ArrayList<>(); 400 @NonNull private final Map<String, Float> mEntityConfidence = new ArrayMap<>(); 401 @Nullable Drawable mPrimaryIcon; 402 @Nullable String mPrimaryLabel; 403 @Nullable Intent mPrimaryIntent; 404 @NonNull private String mSignature = ""; 405 406 /** 407 * Sets the classified text. 408 */ 409 public Builder setText(@Nullable String text) { 410 mText = text; 411 return this; 412 } 413 414 /** 415 * Sets an entity type for the classification result and assigns a confidence score. 416 * If a confidence score had already been set for the specified entity type, this will 417 * override that score. 418 * 419 * @param confidenceScore a value from 0 (low confidence) to 1 (high confidence). 420 * 0 implies the entity does not exist for the classified text. 421 * Values greater than 1 are clamped to 1. 422 */ 423 public Builder setEntityType( 424 @NonNull @EntityType String type, 425 @FloatRange(from = 0.0, to = 1.0) float confidenceScore) { 426 mEntityConfidence.put(type, confidenceScore); 427 return this; 428 } 429 430 /** 431 * Adds an <i>secondary</i> action that may be performed on the classified text. 432 * Secondary actions are in addition to the <i>primary</i> action which may or may not 433 * exist. 434 * 435 * <p>The label and icon are used for rendering of widgets that offer the intent. 436 * Actions should be added in order of priority. 437 * 438 * <p><stong>Note: </stong> If all input parameters are set to null, this method will be a 439 * no-op. 440 * 441 * @see #setPrimaryAction(Intent, String, Drawable) 442 */ 443 public Builder addSecondaryAction( 444 @Nullable Intent intent, @Nullable String label, @Nullable Drawable icon) { 445 if (intent != null || label != null || icon != null) { 446 mSecondaryIntents.add(intent); 447 mSecondaryLabels.add(label); 448 mSecondaryIcons.add(icon); 449 } 450 return this; 451 } 452 453 /** 454 * Removes all the <i>secondary</i> actions. 455 */ 456 public Builder clearSecondaryActions() { 457 mSecondaryIntents.clear(); 458 mSecondaryLabels.clear(); 459 mSecondaryIcons.clear(); 460 return this; 461 } 462 463 /** 464 * Sets the <i>primary</i> action that may be performed on the classified text. This is 465 * equivalent to calling {@code setIntent(intent).setLabel(label).setIcon(icon)}. 466 * 467 * <p><strong>Note: </strong>If all input parameters are null, there will be no 468 * <i>primary</i> action but there may still be <i>secondary</i> actions. 469 * 470 * @see #addSecondaryAction(Intent, String, Drawable) 471 */ 472 public Builder setPrimaryAction( 473 @Nullable Intent intent, @Nullable String label, @Nullable Drawable icon) { 474 return setIntent(intent).setLabel(label).setIcon(icon); 475 } 476 477 /** 478 * Sets the icon for the <i>primary</i> action that may be rendered on a widget used to act 479 * on the classified text. 480 * 481 * @see #setPrimaryAction(Intent, String, Drawable) 482 */ 483 public Builder setIcon(@Nullable Drawable icon) { 484 mPrimaryIcon = icon; 485 return this; 486 } 487 488 /** 489 * Sets the label for the <i>primary</i> action that may be rendered on a widget used to 490 * act on the classified text. 491 * 492 * @see #setPrimaryAction(Intent, String, Drawable) 493 */ 494 public Builder setLabel(@Nullable String label) { 495 mPrimaryLabel = label; 496 return this; 497 } 498 499 /** 500 * Sets the intent for the <i>primary</i> action that may be fired to act on the classified 501 * text. 502 * 503 * @see #setPrimaryAction(Intent, String, Drawable) 504 */ 505 public Builder setIntent(@Nullable Intent intent) { 506 mPrimaryIntent = intent; 507 return this; 508 } 509 510 /** 511 * Sets a signature for the TextClassification object. 512 * The TextClassifier that generates the TextClassification object may use it as a way to 513 * internally identify the TextClassification object. 514 */ 515 public Builder setSignature(@NonNull String signature) { 516 mSignature = Preconditions.checkNotNull(signature); 517 return this; 518 } 519 520 /** 521 * Builds and returns a {@link TextClassification} object. 522 */ 523 public TextClassification build() { 524 return new TextClassification( 525 mText, 526 mPrimaryIcon, mPrimaryLabel, mPrimaryIntent, 527 mSecondaryIcons, mSecondaryLabels, mSecondaryIntents, 528 mEntityConfidence, mSignature); 529 } 530 } 531 532 /** 533 * Optional input parameters for generating TextClassification. 534 */ 535 public static final class Options implements Parcelable { 536 537 private @Nullable LocaleListCompat mDefaultLocales; 538 private @Nullable Calendar mReferenceTime; 539 private @Nullable String mCallingPackageName; 540 541 public Options() {} 542 543 /** 544 * @param defaultLocales ordered list of locale preferences that may be used to disambiguate 545 * the provided text. If no locale preferences exist, set this to null or an empty 546 * locale list. 547 */ 548 public Options setDefaultLocales(@Nullable LocaleListCompat defaultLocales) { 549 mDefaultLocales = defaultLocales; 550 return this; 551 } 552 553 /** 554 * @param referenceTime reference time based on which relative dates (e.g. "tomorrow" should 555 * be interpreted. This should usually be the time when the text was originally 556 * composed. If no reference time is set, now is used. 557 */ 558 public Options setReferenceTime(Calendar referenceTime) { 559 mReferenceTime = referenceTime; 560 return this; 561 } 562 563 /** 564 * @param packageName name of the package from which the call was made. 565 * 566 * @hide 567 */ 568 @RestrictTo(RestrictTo.Scope.LIBRARY) 569 public Options setCallingPackageName(@Nullable String packageName) { 570 mCallingPackageName = packageName; 571 return this; 572 } 573 574 /** 575 * @return ordered list of locale preferences that can be used to disambiguate 576 * the provided text. 577 */ 578 @Nullable 579 public LocaleListCompat getDefaultLocales() { 580 return mDefaultLocales; 581 } 582 583 /** 584 * @return reference time based on which relative dates (e.g. "tomorrow") should be 585 * interpreted. 586 */ 587 @Nullable 588 public Calendar getReferenceTime() { 589 return mReferenceTime; 590 } 591 592 /** 593 * @return name of the package from which the call was made. 594 */ 595 @Nullable 596 public String getCallingPackageName() { 597 return mCallingPackageName; 598 } 599 600 @Override 601 public int describeContents() { 602 return 0; 603 } 604 605 @Override 606 public void writeToParcel(Parcel dest, int flags) { 607 dest.writeInt(mDefaultLocales != null ? 1 : 0); 608 if (mDefaultLocales != null) { 609 dest.writeString(mDefaultLocales.toLanguageTags()); 610 } 611 dest.writeInt(mReferenceTime != null ? 1 : 0); 612 if (mReferenceTime != null) { 613 dest.writeSerializable(mReferenceTime); 614 } 615 dest.writeString(mCallingPackageName); 616 } 617 618 public static final Parcelable.Creator<Options> CREATOR = 619 new Parcelable.Creator<Options>() { 620 @Override 621 public Options createFromParcel(Parcel in) { 622 return new Options(in); 623 } 624 625 @Override 626 public Options[] newArray(int size) { 627 return new Options[size]; 628 } 629 }; 630 631 private Options(Parcel in) { 632 if (in.readInt() > 0) { 633 mDefaultLocales = LocaleListCompat.forLanguageTags(in.readString()); 634 } 635 if (in.readInt() > 0) { 636 mReferenceTime = (Calendar) in.readSerializable(); 637 } 638 mCallingPackageName = in.readString(); 639 } 640 } 641} 642