NuPlayer2DecoderPassThrough.cpp revision 51b6956bc8e09eb2237a10a401baebb35096e6ab
1886b275e529e44a59c54b933453d9bc902973178Romain Guy/* 2886b275e529e44a59c54b933453d9bc902973178Romain Guy * Copyright 2017 The Android Open Source Project 3886b275e529e44a59c54b933453d9bc902973178Romain Guy * 4886b275e529e44a59c54b933453d9bc902973178Romain Guy * Licensed under the Apache License, Version 2.0 (the "License"); 5886b275e529e44a59c54b933453d9bc902973178Romain Guy * you may not use this file except in compliance with the License. 6886b275e529e44a59c54b933453d9bc902973178Romain Guy * You may obtain a copy of the License at 7886b275e529e44a59c54b933453d9bc902973178Romain Guy * 8886b275e529e44a59c54b933453d9bc902973178Romain Guy * http://www.apache.org/licenses/LICENSE-2.0 9886b275e529e44a59c54b933453d9bc902973178Romain Guy * 10886b275e529e44a59c54b933453d9bc902973178Romain Guy * Unless required by applicable law or agreed to in writing, software 11886b275e529e44a59c54b933453d9bc902973178Romain Guy * distributed under the License is distributed on an "AS IS" BASIS, 12886b275e529e44a59c54b933453d9bc902973178Romain Guy * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13886b275e529e44a59c54b933453d9bc902973178Romain Guy * See the License for the specific language governing permissions and 14886b275e529e44a59c54b933453d9bc902973178Romain Guy * limitations under the License. 15886b275e529e44a59c54b933453d9bc902973178Romain Guy */ 16886b275e529e44a59c54b933453d9bc902973178Romain Guy 17886b275e529e44a59c54b933453d9bc902973178Romain Guy//#define LOG_NDEBUG 0 18886b275e529e44a59c54b933453d9bc902973178Romain Guy#define LOG_TAG "NuPlayer2DecoderPassThrough" 19886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <utils/Log.h> 20886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <inttypes.h> 21886b275e529e44a59c54b933453d9bc902973178Romain Guy 22886b275e529e44a59c54b933453d9bc902973178Romain Guy#include "NuPlayer2DecoderPassThrough.h" 23886b275e529e44a59c54b933453d9bc902973178Romain Guy 24886b275e529e44a59c54b933453d9bc902973178Romain Guy#include "NuPlayer2Renderer.h" 25886b275e529e44a59c54b933453d9bc902973178Romain Guy#include "NuPlayer2Source.h" 26886b275e529e44a59c54b933453d9bc902973178Romain Guy 27886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <media/MediaCodecBuffer.h> 28886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <media/stagefright/foundation/ABuffer.h> 29886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <media/stagefright/foundation/ADebug.h> 30886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <media/stagefright/foundation/AMessage.h> 31886b275e529e44a59c54b933453d9bc902973178Romain Guy#include <media/stagefright/MediaErrors.h> 32886b275e529e44a59c54b933453d9bc902973178Romain Guy 33886b275e529e44a59c54b933453d9bc902973178Romain Guy#include "ATSParser.h" 34886b275e529e44a59c54b933453d9bc902973178Romain Guy 35886b275e529e44a59c54b933453d9bc902973178Romain Guynamespace android { 36886b275e529e44a59c54b933453d9bc902973178Romain Guy 37886b275e529e44a59c54b933453d9bc902973178Romain Guy// TODO optimize buffer size for power consumption 38886b275e529e44a59c54b933453d9bc902973178Romain Guy// The offload read buffer size is 32 KB but 24 KB uses less power. 39886b275e529e44a59c54b933453d9bc902973178Romain Guystatic const size_t kAggregateBufferSizeBytes = 24 * 1024; 40886b275e529e44a59c54b933453d9bc902973178Romain Guystatic const size_t kMaxCachedBytes = 200000; 41886b275e529e44a59c54b933453d9bc902973178Romain Guy 42886b275e529e44a59c54b933453d9bc902973178Romain GuyNuPlayer2::DecoderPassThrough::DecoderPassThrough( 43886b275e529e44a59c54b933453d9bc902973178Romain Guy const sp<AMessage> ¬ify, 44886b275e529e44a59c54b933453d9bc902973178Romain Guy const sp<Source> &source, 45886b275e529e44a59c54b933453d9bc902973178Romain Guy const sp<Renderer> &renderer) 46886b275e529e44a59c54b933453d9bc902973178Romain Guy : DecoderBase(notify), 47886b275e529e44a59c54b933453d9bc902973178Romain Guy mSource(source), 48886b275e529e44a59c54b933453d9bc902973178Romain Guy mRenderer(renderer), 49886b275e529e44a59c54b933453d9bc902973178Romain Guy mSkipRenderingUntilMediaTimeUs(-1ll), 50886b275e529e44a59c54b933453d9bc902973178Romain Guy mReachedEOS(true), 51886b275e529e44a59c54b933453d9bc902973178Romain Guy mPendingAudioErr(OK), 52886b275e529e44a59c54b933453d9bc902973178Romain Guy mPendingBuffersToDrain(0), 53886b275e529e44a59c54b933453d9bc902973178Romain Guy mCachedBytes(0), 54886b275e529e44a59c54b933453d9bc902973178Romain Guy mComponentName("pass through decoder") { 55886b275e529e44a59c54b933453d9bc902973178Romain Guy ALOGW_IF(renderer == NULL, "expect a non-NULL renderer"); 56886b275e529e44a59c54b933453d9bc902973178Romain Guy} 57886b275e529e44a59c54b933453d9bc902973178Romain Guy 58886b275e529e44a59c54b933453d9bc902973178Romain GuyNuPlayer2::DecoderPassThrough::~DecoderPassThrough() { 59886b275e529e44a59c54b933453d9bc902973178Romain Guy} 60886b275e529e44a59c54b933453d9bc902973178Romain Guy 61886b275e529e44a59c54b933453d9bc902973178Romain Guyvoid NuPlayer2::DecoderPassThrough::onConfigure(const sp<AMessage> &format) { 62886b275e529e44a59c54b933453d9bc902973178Romain Guy ALOGV("[%s] onConfigure", mComponentName.c_str()); 63886b275e529e44a59c54b933453d9bc902973178Romain Guy mCachedBytes = 0; 64886b275e529e44a59c54b933453d9bc902973178Romain Guy mPendingBuffersToDrain = 0; 65886b275e529e44a59c54b933453d9bc902973178Romain Guy mReachedEOS = false; 66886b275e529e44a59c54b933453d9bc902973178Romain Guy ++mBufferGeneration; 67886b275e529e44a59c54b933453d9bc902973178Romain Guy 68886b275e529e44a59c54b933453d9bc902973178Romain Guy onRequestInputBuffers(); 69886b275e529e44a59c54b933453d9bc902973178Romain Guy 70886b275e529e44a59c54b933453d9bc902973178Romain Guy int32_t hasVideo = 0; 71886b275e529e44a59c54b933453d9bc902973178Romain Guy format->findInt32("has-video", &hasVideo); 72886b275e529e44a59c54b933453d9bc902973178Romain Guy 73886b275e529e44a59c54b933453d9bc902973178Romain Guy // The audio sink is already opened before the PassThrough decoder is created. 74886b275e529e44a59c54b933453d9bc902973178Romain Guy // Opening again might be relevant if decoder is instantiated after shutdown and 75886b275e529e44a59c54b933453d9bc902973178Romain Guy // format is different. 76886b275e529e44a59c54b933453d9bc902973178Romain Guy status_t err = mRenderer->openAudioSink( 77886b275e529e44a59c54b933453d9bc902973178Romain Guy format, true /* offloadOnly */, hasVideo, 78886b275e529e44a59c54b933453d9bc902973178Romain Guy AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */, mSource->isStreaming()); 79886b275e529e44a59c54b933453d9bc902973178Romain Guy if (err != OK) { 80886b275e529e44a59c54b933453d9bc902973178Romain Guy handleError(err); 81886b275e529e44a59c54b933453d9bc902973178Romain Guy } 82886b275e529e44a59c54b933453d9bc902973178Romain Guy} 83886b275e529e44a59c54b933453d9bc902973178Romain Guy 84886b275e529e44a59c54b933453d9bc902973178Romain Guyvoid NuPlayer2::DecoderPassThrough::onSetParameters(const sp<AMessage> &/*params*/) { 85886b275e529e44a59c54b933453d9bc902973178Romain Guy ALOGW("onSetParameters() called unexpectedly"); 86886b275e529e44a59c54b933453d9bc902973178Romain Guy} 87886b275e529e44a59c54b933453d9bc902973178Romain Guy 88886b275e529e44a59c54b933453d9bc902973178Romain Guyvoid NuPlayer2::DecoderPassThrough::onSetRenderer( 89886b275e529e44a59c54b933453d9bc902973178Romain Guy const sp<Renderer> &renderer) { 90886b275e529e44a59c54b933453d9bc902973178Romain Guy // renderer can't be changed during offloading 91886b275e529e44a59c54b933453d9bc902973178Romain Guy ALOGW_IF(renderer != mRenderer, 92886b275e529e44a59c54b933453d9bc902973178Romain Guy "ignoring request to change renderer"); 93886b275e529e44a59c54b933453d9bc902973178Romain Guy} 94886b275e529e44a59c54b933453d9bc902973178Romain Guy 95886b275e529e44a59c54b933453d9bc902973178Romain Guybool NuPlayer2::DecoderPassThrough::isStaleReply(const sp<AMessage> &msg) { 96886b275e529e44a59c54b933453d9bc902973178Romain Guy int32_t generation; 97886b275e529e44a59c54b933453d9bc902973178Romain Guy CHECK(msg->findInt32("generation", &generation)); 98 return generation != mBufferGeneration; 99} 100 101bool NuPlayer2::DecoderPassThrough::isDoneFetching() const { 102 ALOGV("[%s] mCachedBytes = %zu, mReachedEOS = %d mPaused = %d", 103 mComponentName.c_str(), mCachedBytes, mReachedEOS, mPaused); 104 105 return mCachedBytes >= kMaxCachedBytes || mReachedEOS || mPaused; 106} 107 108/* 109 * returns true if we should request more data 110 */ 111bool NuPlayer2::DecoderPassThrough::doRequestBuffers() { 112 status_t err = OK; 113 while (!isDoneFetching()) { 114 sp<AMessage> msg = new AMessage(); 115 116 err = fetchInputData(msg); 117 if (err != OK) { 118 break; 119 } 120 121 onInputBufferFetched(msg); 122 } 123 124 return err == -EWOULDBLOCK 125 && mSource->feedMoreTSData() == OK; 126} 127 128status_t NuPlayer2::DecoderPassThrough::dequeueAccessUnit(sp<ABuffer> *accessUnit) { 129 status_t err; 130 131 // Did we save an accessUnit earlier because of a discontinuity? 132 if (mPendingAudioAccessUnit != NULL) { 133 *accessUnit = mPendingAudioAccessUnit; 134 mPendingAudioAccessUnit.clear(); 135 err = mPendingAudioErr; 136 ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); 137 } else { 138 err = mSource->dequeueAccessUnit(true /* audio */, accessUnit); 139 } 140 141 if (err == INFO_DISCONTINUITY || err == ERROR_END_OF_STREAM) { 142 if (mAggregateBuffer != NULL) { 143 // We already have some data so save this for later. 144 mPendingAudioErr = err; 145 mPendingAudioAccessUnit = *accessUnit; 146 (*accessUnit).clear(); 147 ALOGD("return aggregated buffer and save err(=%d) for later", err); 148 err = OK; 149 } 150 } 151 152 return err; 153} 154 155sp<ABuffer> NuPlayer2::DecoderPassThrough::aggregateBuffer( 156 const sp<ABuffer> &accessUnit) { 157 sp<ABuffer> aggregate; 158 159 if (accessUnit == NULL) { 160 // accessUnit is saved to mPendingAudioAccessUnit 161 // return current mAggregateBuffer 162 aggregate = mAggregateBuffer; 163 mAggregateBuffer.clear(); 164 return aggregate; 165 } 166 167 size_t smallSize = accessUnit->size(); 168 if ((mAggregateBuffer == NULL) 169 // Don't bother if only room for a few small buffers. 170 && (smallSize < (kAggregateBufferSizeBytes / 3))) { 171 // Create a larger buffer for combining smaller buffers from the extractor. 172 mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); 173 mAggregateBuffer->setRange(0, 0); // start empty 174 } 175 176 if (mAggregateBuffer != NULL) { 177 int64_t timeUs; 178 int64_t dummy; 179 bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); 180 bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); 181 // Will the smaller buffer fit? 182 size_t bigSize = mAggregateBuffer->size(); 183 size_t roomLeft = mAggregateBuffer->capacity() - bigSize; 184 // Should we save this small buffer for the next big buffer? 185 // If the first small buffer did not have a timestamp then save 186 // any buffer that does have a timestamp until the next big buffer. 187 if ((smallSize > roomLeft) 188 || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { 189 mPendingAudioErr = OK; 190 mPendingAudioAccessUnit = accessUnit; 191 aggregate = mAggregateBuffer; 192 mAggregateBuffer.clear(); 193 } else { 194 // Grab time from first small buffer if available. 195 if ((bigSize == 0) && smallTimestampValid) { 196 mAggregateBuffer->meta()->setInt64("timeUs", timeUs); 197 } 198 // Append small buffer to the bigger buffer. 199 memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); 200 bigSize += smallSize; 201 mAggregateBuffer->setRange(0, bigSize); 202 203 ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", 204 smallSize, bigSize, mAggregateBuffer->capacity()); 205 } 206 } else { 207 // decided not to aggregate 208 aggregate = accessUnit; 209 } 210 211 return aggregate; 212} 213 214status_t NuPlayer2::DecoderPassThrough::fetchInputData(sp<AMessage> &reply) { 215 sp<ABuffer> accessUnit; 216 217 do { 218 status_t err = dequeueAccessUnit(&accessUnit); 219 220 if (err == -EWOULDBLOCK) { 221 // Flush out the aggregate buffer to try to avoid underrun. 222 accessUnit = aggregateBuffer(NULL /* accessUnit */); 223 if (accessUnit != NULL) { 224 break; 225 } 226 return err; 227 } else if (err != OK) { 228 if (err == INFO_DISCONTINUITY) { 229 int32_t type; 230 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 231 232 bool formatChange = 233 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT) != 0; 234 235 bool timeChange = 236 (type & ATSParser::DISCONTINUITY_TIME) != 0; 237 238 ALOGI("audio discontinuity (formatChange=%d, time=%d)", 239 formatChange, timeChange); 240 241 if (formatChange || timeChange) { 242 sp<AMessage> msg = mNotify->dup(); 243 msg->setInt32("what", kWhatInputDiscontinuity); 244 // will perform seamless format change, 245 // only notify NuPlayer2 to scan sources 246 msg->setInt32("formatChange", false); 247 msg->post(); 248 } 249 250 if (timeChange) { 251 doFlush(false /* notifyComplete */); 252 err = OK; 253 } else if (formatChange) { 254 // do seamless format change 255 err = OK; 256 } else { 257 // This stream is unaffected by the discontinuity 258 return -EWOULDBLOCK; 259 } 260 } 261 262 reply->setInt32("err", err); 263 return OK; 264 } 265 266 accessUnit = aggregateBuffer(accessUnit); 267 } while (accessUnit == NULL); 268 269#if 0 270 int64_t mediaTimeUs; 271 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 272 ALOGV("feeding audio input buffer at media time %.2f secs", 273 mediaTimeUs / 1E6); 274#endif 275 276 reply->setBuffer("buffer", accessUnit); 277 278 return OK; 279} 280 281void NuPlayer2::DecoderPassThrough::onInputBufferFetched( 282 const sp<AMessage> &msg) { 283 if (mReachedEOS) { 284 return; 285 } 286 287 sp<ABuffer> buffer; 288 bool hasBuffer = msg->findBuffer("buffer", &buffer); 289 if (buffer == NULL) { 290 int32_t streamErr = ERROR_END_OF_STREAM; 291 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 292 if (streamErr == OK) { 293 return; 294 } 295 296 if (streamErr != ERROR_END_OF_STREAM) { 297 handleError(streamErr); 298 } 299 mReachedEOS = true; 300 if (mRenderer != NULL) { 301 mRenderer->queueEOS(true /* audio */, ERROR_END_OF_STREAM); 302 } 303 return; 304 } 305 306 sp<AMessage> extra; 307 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 308 int64_t resumeAtMediaTimeUs; 309 if (extra->findInt64( 310 "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) { 311 ALOGI("[%s] suppressing rendering until %lld us", 312 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 313 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 314 } 315 } 316 317 int32_t bufferSize = buffer->size(); 318 mCachedBytes += bufferSize; 319 320 int64_t timeUs = 0; 321 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 322 if (mSkipRenderingUntilMediaTimeUs >= 0) { 323 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 324 ALOGV("[%s] dropping buffer at time %lld as requested.", 325 mComponentName.c_str(), (long long)timeUs); 326 327 onBufferConsumed(bufferSize); 328 return; 329 } 330 331 mSkipRenderingUntilMediaTimeUs = -1; 332 } 333 334 if (mRenderer == NULL) { 335 onBufferConsumed(bufferSize); 336 return; 337 } 338 339 sp<AMessage> reply = new AMessage(kWhatBufferConsumed, this); 340 reply->setInt32("generation", mBufferGeneration); 341 reply->setInt32("size", bufferSize); 342 343 sp<MediaCodecBuffer> mcBuffer = new MediaCodecBuffer(nullptr, buffer); 344 mcBuffer->meta()->setInt64("timeUs", timeUs); 345 346 mRenderer->queueBuffer(true /* audio */, mcBuffer, reply); 347 348 ++mPendingBuffersToDrain; 349 ALOGV("onInputBufferFilled: #ToDrain = %zu, cachedBytes = %zu", 350 mPendingBuffersToDrain, mCachedBytes); 351} 352 353void NuPlayer2::DecoderPassThrough::onBufferConsumed(int32_t size) { 354 --mPendingBuffersToDrain; 355 mCachedBytes -= size; 356 ALOGV("onBufferConsumed: #ToDrain = %zu, cachedBytes = %zu", 357 mPendingBuffersToDrain, mCachedBytes); 358 onRequestInputBuffers(); 359} 360 361void NuPlayer2::DecoderPassThrough::onResume(bool notifyComplete) { 362 mPaused = false; 363 364 onRequestInputBuffers(); 365 366 if (notifyComplete) { 367 sp<AMessage> notify = mNotify->dup(); 368 notify->setInt32("what", kWhatResumeCompleted); 369 notify->post(); 370 } 371} 372 373void NuPlayer2::DecoderPassThrough::doFlush(bool notifyComplete) { 374 ++mBufferGeneration; 375 mSkipRenderingUntilMediaTimeUs = -1; 376 mPendingAudioAccessUnit.clear(); 377 mPendingAudioErr = OK; 378 mAggregateBuffer.clear(); 379 380 if (mRenderer != NULL) { 381 mRenderer->flush(true /* audio */, notifyComplete); 382 mRenderer->signalTimeDiscontinuity(); 383 } 384 385 mPendingBuffersToDrain = 0; 386 mCachedBytes = 0; 387 mReachedEOS = false; 388} 389 390void NuPlayer2::DecoderPassThrough::onFlush() { 391 doFlush(true /* notifyComplete */); 392 393 mPaused = true; 394 sp<AMessage> notify = mNotify->dup(); 395 notify->setInt32("what", kWhatFlushCompleted); 396 notify->post(); 397 398} 399 400void NuPlayer2::DecoderPassThrough::onShutdown(bool notifyComplete) { 401 ++mBufferGeneration; 402 mSkipRenderingUntilMediaTimeUs = -1; 403 404 if (notifyComplete) { 405 sp<AMessage> notify = mNotify->dup(); 406 notify->setInt32("what", kWhatShutdownCompleted); 407 notify->post(); 408 } 409 410 mReachedEOS = true; 411} 412 413void NuPlayer2::DecoderPassThrough::onMessageReceived(const sp<AMessage> &msg) { 414 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), 415 msg->debugString().c_str()); 416 417 switch (msg->what()) { 418 case kWhatBufferConsumed: 419 { 420 if (!isStaleReply(msg)) { 421 int32_t size; 422 CHECK(msg->findInt32("size", &size)); 423 onBufferConsumed(size); 424 } 425 break; 426 } 427 428 default: 429 DecoderBase::onMessageReceived(msg); 430 break; 431 } 432} 433 434} // namespace android 435