1// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/common/gpu/media/android_video_decode_accelerator.h" 6 7#include "base/bind.h" 8#include "base/logging.h" 9#include "base/message_loop/message_loop.h" 10#include "base/metrics/histogram.h" 11#include "content/common/gpu/gpu_channel.h" 12#include "gpu/command_buffer/service/gles2_cmd_decoder.h" 13#include "media/base/bitstream_buffer.h" 14#include "media/base/limits.h" 15#include "media/video/picture.h" 16#include "ui/gl/android/scoped_java_surface.h" 17#include "ui/gl/android/surface_texture.h" 18#include "ui/gl/gl_bindings.h" 19 20namespace content { 21 22// Helper macros for dealing with failure. If |result| evaluates false, emit 23// |log| to ERROR, register |error| with the decoder, and return. 24#define RETURN_ON_FAILURE(result, log, error) \ 25 do { \ 26 if (!(result)) { \ 27 DLOG(ERROR) << log; \ 28 base::MessageLoop::current()->PostTask( \ 29 FROM_HERE, \ 30 base::Bind(&AndroidVideoDecodeAccelerator::NotifyError, \ 31 weak_this_factory_.GetWeakPtr(), \ 32 error)); \ 33 state_ = ERROR; \ 34 return; \ 35 } \ 36 } while (0) 37 38// TODO(dwkang): We only need kMaxVideoFrames to pass media stack's prerolling 39// phase, but 1 is added due to crbug.com/176036. This should be tuned when we 40// have actual use case. 41enum { kNumPictureBuffers = media::limits::kMaxVideoFrames + 1 }; 42 43// Max number of bitstreams notified to the client with 44// NotifyEndOfBitstreamBuffer() before getting output from the bitstream. 45enum { kMaxBitstreamsNotifiedInAdvance = 32 }; 46 47// Because MediaCodec is thread-hostile (must be poked on a single thread) and 48// has no callback mechanism (b/11990118), we must drive it by polling for 49// complete frames (and available input buffers, when the codec is fully 50// saturated). This function defines the polling delay. The value used is an 51// arbitrary choice that trades off CPU utilization (spinning) against latency. 52// Mirrors android_video_encode_accelerator.cc:EncodePollDelay(). 53static inline const base::TimeDelta DecodePollDelay() { 54 // An alternative to this polling scheme could be to dedicate a new thread 55 // (instead of using the ChildThread) to run the MediaCodec, and make that 56 // thread use the timeout-based flavor of MediaCodec's dequeue methods when it 57 // believes the codec should complete "soon" (e.g. waiting for an input 58 // buffer, or waiting for a picture when it knows enough complete input 59 // pictures have been fed to saturate any internal buffering). This is 60 // speculative and it's unclear that this would be a win (nor that there's a 61 // reasonably device-agnostic way to fill in the "believes" above). 62 return base::TimeDelta::FromMilliseconds(10); 63} 64 65static inline const base::TimeDelta NoWaitTimeOut() { 66 return base::TimeDelta::FromMicroseconds(0); 67} 68 69AndroidVideoDecodeAccelerator::AndroidVideoDecodeAccelerator( 70 const base::WeakPtr<gpu::gles2::GLES2Decoder> decoder, 71 const base::Callback<bool(void)>& make_context_current) 72 : client_(NULL), 73 make_context_current_(make_context_current), 74 codec_(media::kCodecH264), 75 state_(NO_ERROR), 76 surface_texture_id_(0), 77 picturebuffers_requested_(false), 78 gl_decoder_(decoder), 79 weak_this_factory_(this) {} 80 81AndroidVideoDecodeAccelerator::~AndroidVideoDecodeAccelerator() { 82 DCHECK(thread_checker_.CalledOnValidThread()); 83} 84 85bool AndroidVideoDecodeAccelerator::Initialize(media::VideoCodecProfile profile, 86 Client* client) { 87 DCHECK(!media_codec_); 88 DCHECK(thread_checker_.CalledOnValidThread()); 89 90 client_ = client; 91 92 if (profile == media::VP8PROFILE_ANY) { 93 codec_ = media::kCodecVP8; 94 } else { 95 // TODO(dwkang): enable H264 once b/8125974 is fixed. 96 LOG(ERROR) << "Unsupported profile: " << profile; 97 return false; 98 } 99 100 // Only consider using MediaCodec if it's likely backed by hardware. 101 if (media::VideoCodecBridge::IsKnownUnaccelerated( 102 codec_, media::MEDIA_CODEC_DECODER)) { 103 return false; 104 } 105 106 if (!make_context_current_.Run()) { 107 LOG(ERROR) << "Failed to make this decoder's GL context current."; 108 return false; 109 } 110 111 if (!gl_decoder_) { 112 LOG(ERROR) << "Failed to get gles2 decoder instance."; 113 return false; 114 } 115 glGenTextures(1, &surface_texture_id_); 116 glActiveTexture(GL_TEXTURE0); 117 glBindTexture(GL_TEXTURE_EXTERNAL_OES, surface_texture_id_); 118 119 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 120 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 121 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, 122 GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 123 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, 124 GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 125 gl_decoder_->RestoreTextureUnitBindings(0); 126 gl_decoder_->RestoreActiveTexture(); 127 128 surface_texture_ = gfx::SurfaceTexture::Create(surface_texture_id_); 129 130 if (!ConfigureMediaCodec()) { 131 LOG(ERROR) << "Failed to create MediaCodec instance."; 132 return false; 133 } 134 135 return true; 136} 137 138void AndroidVideoDecodeAccelerator::DoIOTask() { 139 DCHECK(thread_checker_.CalledOnValidThread()); 140 if (state_ == ERROR) { 141 return; 142 } 143 144 QueueInput(); 145 DequeueOutput(); 146} 147 148void AndroidVideoDecodeAccelerator::QueueInput() { 149 DCHECK(thread_checker_.CalledOnValidThread()); 150 if (bitstreams_notified_in_advance_.size() > kMaxBitstreamsNotifiedInAdvance) 151 return; 152 if (pending_bitstream_buffers_.empty()) 153 return; 154 155 int input_buf_index = 0; 156 media::MediaCodecStatus status = media_codec_->DequeueInputBuffer( 157 NoWaitTimeOut(), &input_buf_index); 158 if (status != media::MEDIA_CODEC_OK) { 159 DCHECK(status == media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER || 160 status == media::MEDIA_CODEC_ERROR); 161 return; 162 } 163 164 base::Time queued_time = pending_bitstream_buffers_.front().second; 165 UMA_HISTOGRAM_TIMES("Media.AVDA.InputQueueTime", 166 base::Time::Now() - queued_time); 167 media::BitstreamBuffer bitstream_buffer = 168 pending_bitstream_buffers_.front().first; 169 pending_bitstream_buffers_.pop(); 170 171 if (bitstream_buffer.id() == -1) { 172 media_codec_->QueueEOS(input_buf_index); 173 return; 174 } 175 176 // Abuse the presentation time argument to propagate the bitstream 177 // buffer ID to the output, so we can report it back to the client in 178 // PictureReady(). 179 base::TimeDelta timestamp = 180 base::TimeDelta::FromMicroseconds(bitstream_buffer.id()); 181 182 scoped_ptr<base::SharedMemory> shm( 183 new base::SharedMemory(bitstream_buffer.handle(), true)); 184 185 RETURN_ON_FAILURE(shm->Map(bitstream_buffer.size()), 186 "Failed to SharedMemory::Map()", 187 UNREADABLE_INPUT); 188 189 status = 190 media_codec_->QueueInputBuffer(input_buf_index, 191 static_cast<const uint8*>(shm->memory()), 192 bitstream_buffer.size(), 193 timestamp); 194 RETURN_ON_FAILURE(status == media::MEDIA_CODEC_OK, 195 "Failed to QueueInputBuffer: " << status, 196 PLATFORM_FAILURE); 197 198 // We should call NotifyEndOfBitstreamBuffer(), when no more decoded output 199 // will be returned from the bitstream buffer. However, MediaCodec API is 200 // not enough to guarantee it. 201 // So, here, we calls NotifyEndOfBitstreamBuffer() in advance in order to 202 // keep getting more bitstreams from the client, and throttle them by using 203 // |bitstreams_notified_in_advance_|. 204 // TODO(dwkang): check if there is a way to remove this workaround. 205 base::MessageLoop::current()->PostTask( 206 FROM_HERE, 207 base::Bind(&AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer, 208 weak_this_factory_.GetWeakPtr(), 209 bitstream_buffer.id())); 210 bitstreams_notified_in_advance_.push_back(bitstream_buffer.id()); 211} 212 213void AndroidVideoDecodeAccelerator::DequeueOutput() { 214 DCHECK(thread_checker_.CalledOnValidThread()); 215 if (picturebuffers_requested_ && output_picture_buffers_.empty()) 216 return; 217 218 if (!output_picture_buffers_.empty() && free_picture_ids_.empty()) { 219 // Don't have any picture buffer to send. Need to wait more. 220 return; 221 } 222 223 bool eos = false; 224 base::TimeDelta timestamp; 225 int32 buf_index = 0; 226 do { 227 size_t offset = 0; 228 size_t size = 0; 229 230 media::MediaCodecStatus status = media_codec_->DequeueOutputBuffer( 231 NoWaitTimeOut(), &buf_index, &offset, &size, ×tamp, &eos, NULL); 232 switch (status) { 233 case media::MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER: 234 case media::MEDIA_CODEC_ERROR: 235 return; 236 237 case media::MEDIA_CODEC_OUTPUT_FORMAT_CHANGED: { 238 int32 width, height; 239 media_codec_->GetOutputFormat(&width, &height); 240 241 if (!picturebuffers_requested_) { 242 picturebuffers_requested_ = true; 243 size_ = gfx::Size(width, height); 244 base::MessageLoop::current()->PostTask( 245 FROM_HERE, 246 base::Bind(&AndroidVideoDecodeAccelerator::RequestPictureBuffers, 247 weak_this_factory_.GetWeakPtr())); 248 } else { 249 // Dynamic resolution change support is not specified by the Android 250 // platform at and before JB-MR1, so it's not possible to smoothly 251 // continue playback at this point. Instead, error out immediately, 252 // expecting clients to Reset() as appropriate to avoid this. 253 // b/7093648 254 RETURN_ON_FAILURE(size_ == gfx::Size(width, height), 255 "Dynamic resolution change is not supported.", 256 PLATFORM_FAILURE); 257 } 258 return; 259 } 260 261 case media::MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED: 262 RETURN_ON_FAILURE(media_codec_->GetOutputBuffers(), 263 "Cannot get output buffer from MediaCodec.", 264 PLATFORM_FAILURE); 265 break; 266 267 case media::MEDIA_CODEC_OK: 268 DCHECK_GE(buf_index, 0); 269 break; 270 271 default: 272 NOTREACHED(); 273 break; 274 } 275 } while (buf_index < 0); 276 277 // This ignores the emitted ByteBuffer and instead relies on rendering to the 278 // codec's SurfaceTexture and then copying from that texture to the client's 279 // PictureBuffer's texture. This means that each picture's data is written 280 // three times: once to the ByteBuffer, once to the SurfaceTexture, and once 281 // to the client's texture. It would be nicer to either: 282 // 1) Render directly to the client's texture from MediaCodec (one write); or 283 // 2) Upload the ByteBuffer to the client's texture (two writes). 284 // Unfortunately neither is possible: 285 // 1) MediaCodec's use of SurfaceTexture is a singleton, and the texture 286 // written to can't change during the codec's lifetime. b/11990461 287 // 2) The ByteBuffer is likely to contain the pixels in a vendor-specific, 288 // opaque/non-standard format. It's not possible to negotiate the decoder 289 // to emit a specific colorspace, even using HW CSC. b/10706245 290 // So, we live with these two extra copies per picture :( 291 media_codec_->ReleaseOutputBuffer(buf_index, true); 292 293 if (eos) { 294 base::MessageLoop::current()->PostTask( 295 FROM_HERE, 296 base::Bind(&AndroidVideoDecodeAccelerator::NotifyFlushDone, 297 weak_this_factory_.GetWeakPtr())); 298 } else { 299 int64 bitstream_buffer_id = timestamp.InMicroseconds(); 300 SendCurrentSurfaceToClient(static_cast<int32>(bitstream_buffer_id)); 301 302 // Removes ids former or equal than the id from decoder. Note that 303 // |bitstreams_notified_in_advance_| does not mean bitstream ids in decoder 304 // because of frame reordering issue. We just maintain this roughly and use 305 // for the throttling purpose. 306 std::list<int32>::iterator it; 307 for (it = bitstreams_notified_in_advance_.begin(); 308 it != bitstreams_notified_in_advance_.end(); 309 ++it) { 310 if (*it == bitstream_buffer_id) { 311 bitstreams_notified_in_advance_.erase( 312 bitstreams_notified_in_advance_.begin(), ++it); 313 break; 314 } 315 } 316 } 317} 318 319void AndroidVideoDecodeAccelerator::SendCurrentSurfaceToClient( 320 int32 bitstream_id) { 321 DCHECK(thread_checker_.CalledOnValidThread()); 322 DCHECK_NE(bitstream_id, -1); 323 DCHECK(!free_picture_ids_.empty()); 324 325 RETURN_ON_FAILURE(make_context_current_.Run(), 326 "Failed to make this decoder's GL context current.", 327 PLATFORM_FAILURE); 328 329 int32 picture_buffer_id = free_picture_ids_.front(); 330 free_picture_ids_.pop(); 331 332 float transfrom_matrix[16]; 333 surface_texture_->UpdateTexImage(); 334 surface_texture_->GetTransformMatrix(transfrom_matrix); 335 336 OutputBufferMap::const_iterator i = 337 output_picture_buffers_.find(picture_buffer_id); 338 RETURN_ON_FAILURE(i != output_picture_buffers_.end(), 339 "Can't find a PictureBuffer for " << picture_buffer_id, 340 PLATFORM_FAILURE); 341 uint32 picture_buffer_texture_id = i->second.texture_id(); 342 343 RETURN_ON_FAILURE(gl_decoder_.get(), 344 "Failed to get gles2 decoder instance.", 345 ILLEGAL_STATE); 346 // Defer initializing the CopyTextureCHROMIUMResourceManager until it is 347 // needed because it takes 10s of milliseconds to initialize. 348 if (!copier_) { 349 copier_.reset(new gpu::CopyTextureCHROMIUMResourceManager()); 350 copier_->Initialize(gl_decoder_.get()); 351 } 352 353 // Here, we copy |surface_texture_id_| to the picture buffer instead of 354 // setting new texture to |surface_texture_| by calling attachToGLContext() 355 // because: 356 // 1. Once we call detachFrameGLContext(), it deletes the texture previous 357 // attached. 358 // 2. SurfaceTexture requires us to apply a transform matrix when we show 359 // the texture. 360 // TODO(hkuang): get the StreamTexture transform matrix in GPU process 361 // instead of using default matrix crbug.com/226218. 362 const static GLfloat default_matrix[16] = {1.0f, 0.0f, 0.0f, 0.0f, 363 0.0f, 1.0f, 0.0f, 0.0f, 364 0.0f, 0.0f, 1.0f, 0.0f, 365 0.0f, 0.0f, 0.0f, 1.0f}; 366 copier_->DoCopyTextureWithTransform(gl_decoder_.get(), 367 GL_TEXTURE_EXTERNAL_OES, 368 surface_texture_id_, 369 picture_buffer_texture_id, 370 0, 371 size_.width(), 372 size_.height(), 373 false, 374 false, 375 false, 376 default_matrix); 377 378 base::MessageLoop::current()->PostTask( 379 FROM_HERE, 380 base::Bind( 381 &AndroidVideoDecodeAccelerator::NotifyPictureReady, 382 weak_this_factory_.GetWeakPtr(), 383 media::Picture(picture_buffer_id, bitstream_id, gfx::Rect(size_)))); 384} 385 386void AndroidVideoDecodeAccelerator::Decode( 387 const media::BitstreamBuffer& bitstream_buffer) { 388 DCHECK(thread_checker_.CalledOnValidThread()); 389 if (bitstream_buffer.id() != -1 && bitstream_buffer.size() == 0) { 390 base::MessageLoop::current()->PostTask( 391 FROM_HERE, 392 base::Bind(&AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer, 393 weak_this_factory_.GetWeakPtr(), 394 bitstream_buffer.id())); 395 return; 396 } 397 398 pending_bitstream_buffers_.push( 399 std::make_pair(bitstream_buffer, base::Time::Now())); 400 401 DoIOTask(); 402} 403 404void AndroidVideoDecodeAccelerator::AssignPictureBuffers( 405 const std::vector<media::PictureBuffer>& buffers) { 406 DCHECK(thread_checker_.CalledOnValidThread()); 407 DCHECK(output_picture_buffers_.empty()); 408 DCHECK(free_picture_ids_.empty()); 409 410 for (size_t i = 0; i < buffers.size(); ++i) { 411 RETURN_ON_FAILURE(buffers[i].size() == size_, 412 "Invalid picture buffer size was passed.", 413 INVALID_ARGUMENT); 414 int32 id = buffers[i].id(); 415 output_picture_buffers_.insert(std::make_pair(id, buffers[i])); 416 free_picture_ids_.push(id); 417 // Since the client might be re-using |picture_buffer_id| values, forget 418 // about previously-dismissed IDs now. See ReusePictureBuffer() comment 419 // about "zombies" for why we maintain this set in the first place. 420 dismissed_picture_ids_.erase(id); 421 } 422 423 RETURN_ON_FAILURE(output_picture_buffers_.size() == kNumPictureBuffers, 424 "Invalid picture buffers were passed.", 425 INVALID_ARGUMENT); 426 427 DoIOTask(); 428} 429 430void AndroidVideoDecodeAccelerator::ReusePictureBuffer( 431 int32 picture_buffer_id) { 432 DCHECK(thread_checker_.CalledOnValidThread()); 433 434 // This ReusePictureBuffer() might have been in a pipe somewhere (queued in 435 // IPC, or in a PostTask either at the sender or receiver) when we sent a 436 // DismissPictureBuffer() for this |picture_buffer_id|. Account for such 437 // potential "zombie" IDs here. 438 if (dismissed_picture_ids_.erase(picture_buffer_id)) 439 return; 440 441 free_picture_ids_.push(picture_buffer_id); 442 443 DoIOTask(); 444} 445 446void AndroidVideoDecodeAccelerator::Flush() { 447 DCHECK(thread_checker_.CalledOnValidThread()); 448 449 Decode(media::BitstreamBuffer(-1, base::SharedMemoryHandle(), 0)); 450} 451 452bool AndroidVideoDecodeAccelerator::ConfigureMediaCodec() { 453 DCHECK(thread_checker_.CalledOnValidThread()); 454 DCHECK(surface_texture_.get()); 455 456 gfx::ScopedJavaSurface surface(surface_texture_.get()); 457 458 // Pass a dummy 320x240 canvas size and let the codec signal the real size 459 // when it's known from the bitstream. 460 media_codec_.reset(media::VideoCodecBridge::CreateDecoder( 461 codec_, false, gfx::Size(320, 240), surface.j_surface().obj(), NULL)); 462 if (!media_codec_) 463 return false; 464 465 io_timer_.Start(FROM_HERE, 466 DecodePollDelay(), 467 this, 468 &AndroidVideoDecodeAccelerator::DoIOTask); 469 return true; 470} 471 472void AndroidVideoDecodeAccelerator::Reset() { 473 DCHECK(thread_checker_.CalledOnValidThread()); 474 475 while (!pending_bitstream_buffers_.empty()) { 476 int32 bitstream_buffer_id = pending_bitstream_buffers_.front().first.id(); 477 pending_bitstream_buffers_.pop(); 478 479 if (bitstream_buffer_id != -1) { 480 base::MessageLoop::current()->PostTask( 481 FROM_HERE, 482 base::Bind(&AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer, 483 weak_this_factory_.GetWeakPtr(), 484 bitstream_buffer_id)); 485 } 486 } 487 bitstreams_notified_in_advance_.clear(); 488 489 for (OutputBufferMap::iterator it = output_picture_buffers_.begin(); 490 it != output_picture_buffers_.end(); 491 ++it) { 492 client_->DismissPictureBuffer(it->first); 493 dismissed_picture_ids_.insert(it->first); 494 } 495 output_picture_buffers_.clear(); 496 std::queue<int32> empty; 497 std::swap(free_picture_ids_, empty); 498 CHECK(free_picture_ids_.empty()); 499 picturebuffers_requested_ = false; 500 501 // On some devices, and up to at least JB-MR1, 502 // - flush() can fail after EOS (b/8125974); and 503 // - mid-stream resolution change is unsupported (b/7093648). 504 // To cope with these facts, we always stop & restart the codec on Reset(). 505 io_timer_.Stop(); 506 media_codec_->Stop(); 507 ConfigureMediaCodec(); 508 state_ = NO_ERROR; 509 510 base::MessageLoop::current()->PostTask( 511 FROM_HERE, 512 base::Bind(&AndroidVideoDecodeAccelerator::NotifyResetDone, 513 weak_this_factory_.GetWeakPtr())); 514} 515 516void AndroidVideoDecodeAccelerator::Destroy() { 517 DCHECK(thread_checker_.CalledOnValidThread()); 518 519 weak_this_factory_.InvalidateWeakPtrs(); 520 if (media_codec_) { 521 io_timer_.Stop(); 522 media_codec_->Stop(); 523 } 524 if (surface_texture_id_) 525 glDeleteTextures(1, &surface_texture_id_); 526 if (copier_) 527 copier_->Destroy(); 528 delete this; 529} 530 531bool AndroidVideoDecodeAccelerator::CanDecodeOnIOThread() { 532 return false; 533} 534 535void AndroidVideoDecodeAccelerator::RequestPictureBuffers() { 536 client_->ProvidePictureBuffers(kNumPictureBuffers, size_, GL_TEXTURE_2D); 537} 538 539void AndroidVideoDecodeAccelerator::NotifyPictureReady( 540 const media::Picture& picture) { 541 client_->PictureReady(picture); 542} 543 544void AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer( 545 int input_buffer_id) { 546 client_->NotifyEndOfBitstreamBuffer(input_buffer_id); 547} 548 549void AndroidVideoDecodeAccelerator::NotifyFlushDone() { 550 client_->NotifyFlushDone(); 551} 552 553void AndroidVideoDecodeAccelerator::NotifyResetDone() { 554 client_->NotifyResetDone(); 555} 556 557void AndroidVideoDecodeAccelerator::NotifyError( 558 media::VideoDecodeAccelerator::Error error) { 559 client_->NotifyError(error); 560} 561 562} // namespace content 563