1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.graphics.Canvas; 20import android.os.Handler; 21import android.util.Log; 22import android.util.LongSparseArray; 23import android.util.Pair; 24 25import java.util.Iterator; 26import java.util.NoSuchElementException; 27import java.util.SortedMap; 28import java.util.TreeMap; 29import java.util.Vector; 30 31/** 32 * A subtitle track abstract base class that is responsible for parsing and displaying 33 * an instance of a particular type of subtitle. 34 * 35 * @hide 36 */ 37public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeListener { 38 private static final String TAG = "SubtitleTrack"; 39 private long mLastUpdateTimeMs; 40 private long mLastTimeMs; 41 42 private Runnable mRunnable; 43 44 /** @hide TODO private */ 45 final protected LongSparseArray<Run> mRunsByEndTime = new LongSparseArray<Run>(); 46 /** @hide TODO private */ 47 final protected LongSparseArray<Run> mRunsByID = new LongSparseArray<Run>(); 48 49 /** @hide TODO private */ 50 protected CueList mCues; 51 /** @hide TODO private */ 52 final protected Vector<Cue> mActiveCues = new Vector<Cue>(); 53 /** @hide */ 54 protected boolean mVisible; 55 56 /** @hide */ 57 public boolean DEBUG = false; 58 59 /** @hide */ 60 protected Handler mHandler = new Handler(); 61 62 private MediaFormat mFormat; 63 64 public SubtitleTrack(MediaFormat format) { 65 mFormat = format; 66 mCues = new CueList(); 67 clearActiveCues(); 68 mLastTimeMs = -1; 69 } 70 71 /** @hide */ 72 public final MediaFormat getFormat() { 73 return mFormat; 74 } 75 76 private long mNextScheduledTimeMs = -1; 77 78 protected void onData(SubtitleData data) { 79 long runID = data.getStartTimeUs() + 1; 80 onData(data.getData(), true /* eos */, runID); 81 setRunDiscardTimeMs( 82 runID, 83 (data.getStartTimeUs() + data.getDurationUs()) / 1000); 84 } 85 86 /** 87 * Called when there is input data for the subtitle track. The 88 * complete subtitle for a track can include multiple whole units 89 * (runs). Each of these units can have multiple sections. The 90 * contents of a run are submitted in sequential order, with eos 91 * indicating the last section of the run. Calls from different 92 * runs must not be intermixed. 93 * 94 * @param data subtitle data byte buffer 95 * @param eos true if this is the last section of the run. 96 * @param runID mostly-unique ID for this run of data. Subtitle cues 97 * with runID of 0 are discarded immediately after 98 * display. Cues with runID of ~0 are discarded 99 * only at the deletion of the track object. Cues 100 * with other runID-s are discarded at the end of the 101 * run, which defaults to the latest timestamp of 102 * any of its cues (with this runID). 103 */ 104 public abstract void onData(byte[] data, boolean eos, long runID); 105 106 /** 107 * Called when adding the subtitle rendering widget to the view hierarchy, 108 * as well as when showing or hiding the subtitle track, or when the video 109 * surface position has changed. 110 * 111 * @return the widget that renders this subtitle track. For most renderers 112 * there should be a single shared instance that is used for all 113 * tracks supported by that renderer, as at most one subtitle track 114 * is visible at one time. 115 */ 116 public abstract RenderingWidget getRenderingWidget(); 117 118 /** 119 * Called when the active cues have changed, and the contents of the subtitle 120 * view should be updated. 121 * 122 * @hide 123 */ 124 public abstract void updateView(Vector<Cue> activeCues); 125 126 /** @hide */ 127 protected synchronized void updateActiveCues(boolean rebuild, long timeMs) { 128 // out-of-order times mean seeking or new active cues being added 129 // (during their own timespan) 130 if (rebuild || mLastUpdateTimeMs > timeMs) { 131 clearActiveCues(); 132 } 133 134 for(Iterator<Pair<Long, Cue> > it = 135 mCues.entriesBetween(mLastUpdateTimeMs, timeMs).iterator(); it.hasNext(); ) { 136 Pair<Long, Cue> event = it.next(); 137 Cue cue = event.second; 138 139 if (cue.mEndTimeMs == event.first) { 140 // remove past cues 141 if (DEBUG) Log.v(TAG, "Removing " + cue); 142 mActiveCues.remove(cue); 143 if (cue.mRunID == 0) { 144 it.remove(); 145 } 146 } else if (cue.mStartTimeMs == event.first) { 147 // add new cues 148 // TRICKY: this will happen in start order 149 if (DEBUG) Log.v(TAG, "Adding " + cue); 150 if (cue.mInnerTimesMs != null) { 151 cue.onTime(timeMs); 152 } 153 mActiveCues.add(cue); 154 } else if (cue.mInnerTimesMs != null) { 155 // cue is modified 156 cue.onTime(timeMs); 157 } 158 } 159 160 /* complete any runs */ 161 while (mRunsByEndTime.size() > 0 && 162 mRunsByEndTime.keyAt(0) <= timeMs) { 163 removeRunsByEndTimeIndex(0); // removes element 164 } 165 mLastUpdateTimeMs = timeMs; 166 } 167 168 private void removeRunsByEndTimeIndex(int ix) { 169 Run run = mRunsByEndTime.valueAt(ix); 170 while (run != null) { 171 Cue cue = run.mFirstCue; 172 while (cue != null) { 173 mCues.remove(cue); 174 Cue nextCue = cue.mNextInRun; 175 cue.mNextInRun = null; 176 cue = nextCue; 177 } 178 mRunsByID.remove(run.mRunID); 179 Run nextRun = run.mNextRunAtEndTimeMs; 180 run.mPrevRunAtEndTimeMs = null; 181 run.mNextRunAtEndTimeMs = null; 182 run = nextRun; 183 } 184 mRunsByEndTime.removeAt(ix); 185 } 186 187 @Override 188 protected void finalize() throws Throwable { 189 /* remove all cues (untangle all cross-links) */ 190 int size = mRunsByEndTime.size(); 191 for(int ix = size - 1; ix >= 0; ix--) { 192 removeRunsByEndTimeIndex(ix); 193 } 194 195 super.finalize(); 196 } 197 198 private synchronized void takeTime(long timeMs) { 199 mLastTimeMs = timeMs; 200 } 201 202 /** @hide */ 203 protected synchronized void clearActiveCues() { 204 if (DEBUG) Log.v(TAG, "Clearing " + mActiveCues.size() + " active cues"); 205 mActiveCues.clear(); 206 mLastUpdateTimeMs = -1; 207 } 208 209 /** @hide */ 210 protected void scheduleTimedEvents() { 211 /* get times for the next event */ 212 if (mTimeProvider != null) { 213 mNextScheduledTimeMs = mCues.nextTimeAfter(mLastTimeMs); 214 if (DEBUG) Log.d(TAG, "sched @" + mNextScheduledTimeMs + " after " + mLastTimeMs); 215 mTimeProvider.notifyAt( 216 mNextScheduledTimeMs >= 0 ? 217 (mNextScheduledTimeMs * 1000) : MediaTimeProvider.NO_TIME, 218 this); 219 } 220 } 221 222 /** 223 * @hide 224 */ 225 @Override 226 public void onTimedEvent(long timeUs) { 227 if (DEBUG) Log.d(TAG, "onTimedEvent " + timeUs); 228 synchronized (this) { 229 long timeMs = timeUs / 1000; 230 updateActiveCues(false, timeMs); 231 takeTime(timeMs); 232 } 233 updateView(mActiveCues); 234 scheduleTimedEvents(); 235 } 236 237 /** 238 * @hide 239 */ 240 @Override 241 public void onSeek(long timeUs) { 242 if (DEBUG) Log.d(TAG, "onSeek " + timeUs); 243 synchronized (this) { 244 long timeMs = timeUs / 1000; 245 updateActiveCues(true, timeMs); 246 takeTime(timeMs); 247 } 248 updateView(mActiveCues); 249 scheduleTimedEvents(); 250 } 251 252 /** 253 * @hide 254 */ 255 @Override 256 public void onStop() { 257 synchronized (this) { 258 if (DEBUG) Log.d(TAG, "onStop"); 259 clearActiveCues(); 260 mLastTimeMs = -1; 261 } 262 updateView(mActiveCues); 263 mNextScheduledTimeMs = -1; 264 mTimeProvider.notifyAt(MediaTimeProvider.NO_TIME, this); 265 } 266 267 /** @hide */ 268 protected MediaTimeProvider mTimeProvider; 269 270 /** @hide */ 271 public void show() { 272 if (mVisible) { 273 return; 274 } 275 276 mVisible = true; 277 RenderingWidget renderingWidget = getRenderingWidget(); 278 if (renderingWidget != null) { 279 renderingWidget.setVisible(true); 280 } 281 if (mTimeProvider != null) { 282 mTimeProvider.scheduleUpdate(this); 283 } 284 } 285 286 /** @hide */ 287 public void hide() { 288 if (!mVisible) { 289 return; 290 } 291 292 if (mTimeProvider != null) { 293 mTimeProvider.cancelNotifications(this); 294 } 295 RenderingWidget renderingWidget = getRenderingWidget(); 296 if (renderingWidget != null) { 297 renderingWidget.setVisible(false); 298 } 299 mVisible = false; 300 } 301 302 /** @hide */ 303 protected synchronized boolean addCue(Cue cue) { 304 mCues.add(cue); 305 306 if (cue.mRunID != 0) { 307 Run run = mRunsByID.get(cue.mRunID); 308 if (run == null) { 309 run = new Run(); 310 mRunsByID.put(cue.mRunID, run); 311 run.mEndTimeMs = cue.mEndTimeMs; 312 } else if (run.mEndTimeMs < cue.mEndTimeMs) { 313 run.mEndTimeMs = cue.mEndTimeMs; 314 } 315 316 // link-up cues in the same run 317 cue.mNextInRun = run.mFirstCue; 318 run.mFirstCue = cue; 319 } 320 321 // if a cue is added that should be visible, need to refresh view 322 long nowMs = -1; 323 if (mTimeProvider != null) { 324 try { 325 nowMs = mTimeProvider.getCurrentTimeUs( 326 false /* precise */, true /* monotonic */) / 1000; 327 } catch (IllegalStateException e) { 328 // handle as it we are not playing 329 } 330 } 331 332 if (DEBUG) Log.v(TAG, "mVisible=" + mVisible + ", " + 333 cue.mStartTimeMs + " <= " + nowMs + ", " + 334 cue.mEndTimeMs + " >= " + mLastTimeMs); 335 336 if (mVisible && 337 cue.mStartTimeMs <= nowMs && 338 // we don't trust nowMs, so check any cue since last callback 339 cue.mEndTimeMs >= mLastTimeMs) { 340 if (mRunnable != null) { 341 mHandler.removeCallbacks(mRunnable); 342 } 343 final SubtitleTrack track = this; 344 final long thenMs = nowMs; 345 mRunnable = new Runnable() { 346 @Override 347 public void run() { 348 // even with synchronized, it is possible that we are going 349 // to do multiple updates as the runnable could be already 350 // running. 351 synchronized (track) { 352 mRunnable = null; 353 updateActiveCues(true, thenMs); 354 updateView(mActiveCues); 355 } 356 } 357 }; 358 // delay update so we don't update view on every cue. TODO why 10? 359 if (mHandler.postDelayed(mRunnable, 10 /* delay */)) { 360 if (DEBUG) Log.v(TAG, "scheduling update"); 361 } else { 362 if (DEBUG) Log.w(TAG, "failed to schedule subtitle view update"); 363 } 364 return true; 365 } 366 367 if (mVisible && 368 cue.mEndTimeMs >= mLastTimeMs && 369 (cue.mStartTimeMs < mNextScheduledTimeMs || 370 mNextScheduledTimeMs < 0)) { 371 scheduleTimedEvents(); 372 } 373 374 return false; 375 } 376 377 /** @hide */ 378 public synchronized void setTimeProvider(MediaTimeProvider timeProvider) { 379 if (mTimeProvider == timeProvider) { 380 return; 381 } 382 if (mTimeProvider != null) { 383 mTimeProvider.cancelNotifications(this); 384 } 385 mTimeProvider = timeProvider; 386 if (mTimeProvider != null) { 387 mTimeProvider.scheduleUpdate(this); 388 } 389 } 390 391 392 /** @hide */ 393 static class CueList { 394 private static final String TAG = "CueList"; 395 // simplistic, inefficient implementation 396 private SortedMap<Long, Vector<Cue> > mCues; 397 public boolean DEBUG = false; 398 399 private boolean addEvent(Cue cue, long timeMs) { 400 Vector<Cue> cues = mCues.get(timeMs); 401 if (cues == null) { 402 cues = new Vector<Cue>(2); 403 mCues.put(timeMs, cues); 404 } else if (cues.contains(cue)) { 405 // do not duplicate cues 406 return false; 407 } 408 409 cues.add(cue); 410 return true; 411 } 412 413 private void removeEvent(Cue cue, long timeMs) { 414 Vector<Cue> cues = mCues.get(timeMs); 415 if (cues != null) { 416 cues.remove(cue); 417 if (cues.size() == 0) { 418 mCues.remove(timeMs); 419 } 420 } 421 } 422 423 public void add(Cue cue) { 424 // ignore non-positive-duration cues 425 if (cue.mStartTimeMs >= cue.mEndTimeMs) 426 return; 427 428 if (!addEvent(cue, cue.mStartTimeMs)) { 429 return; 430 } 431 432 long lastTimeMs = cue.mStartTimeMs; 433 if (cue.mInnerTimesMs != null) { 434 for (long timeMs: cue.mInnerTimesMs) { 435 if (timeMs > lastTimeMs && timeMs < cue.mEndTimeMs) { 436 addEvent(cue, timeMs); 437 lastTimeMs = timeMs; 438 } 439 } 440 } 441 442 addEvent(cue, cue.mEndTimeMs); 443 } 444 445 public void remove(Cue cue) { 446 removeEvent(cue, cue.mStartTimeMs); 447 if (cue.mInnerTimesMs != null) { 448 for (long timeMs: cue.mInnerTimesMs) { 449 removeEvent(cue, timeMs); 450 } 451 } 452 removeEvent(cue, cue.mEndTimeMs); 453 } 454 455 public Iterable<Pair<Long, Cue>> entriesBetween( 456 final long lastTimeMs, final long timeMs) { 457 return new Iterable<Pair<Long, Cue> >() { 458 @Override 459 public Iterator<Pair<Long, Cue> > iterator() { 460 if (DEBUG) Log.d(TAG, "slice (" + lastTimeMs + ", " + timeMs + "]="); 461 try { 462 return new EntryIterator( 463 mCues.subMap(lastTimeMs + 1, timeMs + 1)); 464 } catch(IllegalArgumentException e) { 465 return new EntryIterator(null); 466 } 467 } 468 }; 469 } 470 471 public long nextTimeAfter(long timeMs) { 472 SortedMap<Long, Vector<Cue>> tail = null; 473 try { 474 tail = mCues.tailMap(timeMs + 1); 475 if (tail != null) { 476 return tail.firstKey(); 477 } else { 478 return -1; 479 } 480 } catch(IllegalArgumentException e) { 481 return -1; 482 } catch(NoSuchElementException e) { 483 return -1; 484 } 485 } 486 487 class EntryIterator implements Iterator<Pair<Long, Cue> > { 488 @Override 489 public boolean hasNext() { 490 return !mDone; 491 } 492 493 @Override 494 public Pair<Long, Cue> next() { 495 if (mDone) { 496 throw new NoSuchElementException(""); 497 } 498 mLastEntry = new Pair<Long, Cue>( 499 mCurrentTimeMs, mListIterator.next()); 500 mLastListIterator = mListIterator; 501 if (!mListIterator.hasNext()) { 502 nextKey(); 503 } 504 return mLastEntry; 505 } 506 507 @Override 508 public void remove() { 509 // only allow removing end tags 510 if (mLastListIterator == null || 511 mLastEntry.second.mEndTimeMs != mLastEntry.first) { 512 throw new IllegalStateException(""); 513 } 514 515 // remove end-cue 516 mLastListIterator.remove(); 517 mLastListIterator = null; 518 if (mCues.get(mLastEntry.first).size() == 0) { 519 mCues.remove(mLastEntry.first); 520 } 521 522 // remove rest of the cues 523 Cue cue = mLastEntry.second; 524 removeEvent(cue, cue.mStartTimeMs); 525 if (cue.mInnerTimesMs != null) { 526 for (long timeMs: cue.mInnerTimesMs) { 527 removeEvent(cue, timeMs); 528 } 529 } 530 } 531 532 public EntryIterator(SortedMap<Long, Vector<Cue> > cues) { 533 if (DEBUG) Log.v(TAG, cues + ""); 534 mRemainingCues = cues; 535 mLastListIterator = null; 536 nextKey(); 537 } 538 539 private void nextKey() { 540 do { 541 try { 542 if (mRemainingCues == null) { 543 throw new NoSuchElementException(""); 544 } 545 mCurrentTimeMs = mRemainingCues.firstKey(); 546 mListIterator = 547 mRemainingCues.get(mCurrentTimeMs).iterator(); 548 try { 549 mRemainingCues = 550 mRemainingCues.tailMap(mCurrentTimeMs + 1); 551 } catch (IllegalArgumentException e) { 552 mRemainingCues = null; 553 } 554 mDone = false; 555 } catch (NoSuchElementException e) { 556 mDone = true; 557 mRemainingCues = null; 558 mListIterator = null; 559 return; 560 } 561 } while (!mListIterator.hasNext()); 562 } 563 564 private long mCurrentTimeMs; 565 private Iterator<Cue> mListIterator; 566 private boolean mDone; 567 private SortedMap<Long, Vector<Cue> > mRemainingCues; 568 private Iterator<Cue> mLastListIterator; 569 private Pair<Long,Cue> mLastEntry; 570 } 571 572 CueList() { 573 mCues = new TreeMap<Long, Vector<Cue>>(); 574 } 575 } 576 577 /** @hide */ 578 public static class Cue { 579 public long mStartTimeMs; 580 public long mEndTimeMs; 581 public long[] mInnerTimesMs; 582 public long mRunID; 583 584 /** @hide */ 585 public Cue mNextInRun; 586 587 public void onTime(long timeMs) { } 588 } 589 590 /** @hide update mRunsByEndTime (with default end time) */ 591 protected void finishedRun(long runID) { 592 if (runID != 0 && runID != ~0) { 593 Run run = mRunsByID.get(runID); 594 if (run != null) { 595 run.storeByEndTimeMs(mRunsByEndTime); 596 } 597 } 598 } 599 600 /** @hide update mRunsByEndTime with given end time */ 601 public void setRunDiscardTimeMs(long runID, long timeMs) { 602 if (runID != 0 && runID != ~0) { 603 Run run = mRunsByID.get(runID); 604 if (run != null) { 605 run.mEndTimeMs = timeMs; 606 run.storeByEndTimeMs(mRunsByEndTime); 607 } 608 } 609 } 610 611 /** @hide whether this is a text track who fires events instead getting rendered */ 612 public boolean isTimedText() { 613 return getRenderingWidget() == null; 614 } 615 616 617 /** @hide */ 618 private static class Run { 619 public Cue mFirstCue; 620 public Run mNextRunAtEndTimeMs; 621 public Run mPrevRunAtEndTimeMs; 622 public long mEndTimeMs = -1; 623 public long mRunID = 0; 624 private long mStoredEndTimeMs = -1; 625 626 public void storeByEndTimeMs(LongSparseArray<Run> runsByEndTime) { 627 // remove old value if any 628 int ix = runsByEndTime.indexOfKey(mStoredEndTimeMs); 629 if (ix >= 0) { 630 if (mPrevRunAtEndTimeMs == null) { 631 assert(this == runsByEndTime.valueAt(ix)); 632 if (mNextRunAtEndTimeMs == null) { 633 runsByEndTime.removeAt(ix); 634 } else { 635 runsByEndTime.setValueAt(ix, mNextRunAtEndTimeMs); 636 } 637 } 638 removeAtEndTimeMs(); 639 } 640 641 // add new value 642 if (mEndTimeMs >= 0) { 643 mPrevRunAtEndTimeMs = null; 644 mNextRunAtEndTimeMs = runsByEndTime.get(mEndTimeMs); 645 if (mNextRunAtEndTimeMs != null) { 646 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = this; 647 } 648 runsByEndTime.put(mEndTimeMs, this); 649 mStoredEndTimeMs = mEndTimeMs; 650 } 651 } 652 653 public void removeAtEndTimeMs() { 654 Run prev = mPrevRunAtEndTimeMs; 655 656 if (mPrevRunAtEndTimeMs != null) { 657 mPrevRunAtEndTimeMs.mNextRunAtEndTimeMs = mNextRunAtEndTimeMs; 658 mPrevRunAtEndTimeMs = null; 659 } 660 if (mNextRunAtEndTimeMs != null) { 661 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = prev; 662 mNextRunAtEndTimeMs = null; 663 } 664 } 665 } 666 667 /** 668 * Interface for rendering subtitles onto a Canvas. 669 */ 670 public interface RenderingWidget { 671 /** 672 * Sets the widget's callback, which is used to send updates when the 673 * rendered data has changed. 674 * 675 * @param callback update callback 676 */ 677 public void setOnChangedListener(OnChangedListener callback); 678 679 /** 680 * Sets the widget's size. 681 * 682 * @param width width in pixels 683 * @param height height in pixels 684 */ 685 public void setSize(int width, int height); 686 687 /** 688 * Sets whether the widget should draw subtitles. 689 * 690 * @param visible true if subtitles should be drawn, false otherwise 691 */ 692 public void setVisible(boolean visible); 693 694 /** 695 * Renders subtitles onto a {@link Canvas}. 696 * 697 * @param c canvas on which to render subtitles 698 */ 699 public void draw(Canvas c); 700 701 /** 702 * Called when the widget is attached to a window. 703 */ 704 public void onAttachedToWindow(); 705 706 /** 707 * Called when the widget is detached from a window. 708 */ 709 public void onDetachedFromWindow(); 710 711 /** 712 * Callback used to send updates about changes to rendering data. 713 */ 714 public interface OnChangedListener { 715 /** 716 * Called when the rendering data has changed. 717 * 718 * @param renderingWidget the widget whose data has changed 719 */ 720 public void onChanged(RenderingWidget renderingWidget); 721 } 722 } 723} 724