1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/renderer/pepper/video_decoder_shim.h"
6
7#include <GLES2/gl2.h>
8#include <GLES2/gl2ext.h>
9#include <GLES2/gl2extchromium.h>
10
11#include "base/bind.h"
12#include "base/numerics/safe_conversions.h"
13#include "base/single_thread_task_runner.h"
14#include "content/public/renderer/render_thread.h"
15#include "content/renderer/pepper/pepper_video_decoder_host.h"
16#include "content/renderer/render_thread_impl.h"
17#include "gpu/command_buffer/client/gles2_implementation.h"
18#include "media/base/decoder_buffer.h"
19#include "media/base/limits.h"
20#include "media/base/video_decoder.h"
21#include "media/filters/ffmpeg_video_decoder.h"
22#include "media/filters/vpx_video_decoder.h"
23#include "media/video/picture.h"
24#include "media/video/video_decode_accelerator.h"
25#include "ppapi/c/pp_errors.h"
26#include "third_party/libyuv/include/libyuv.h"
27#include "webkit/common/gpu/context_provider_web_context.h"
28
29namespace content {
30
31struct VideoDecoderShim::PendingDecode {
32  PendingDecode(uint32_t decode_id,
33                const scoped_refptr<media::DecoderBuffer>& buffer);
34  ~PendingDecode();
35
36  const uint32_t decode_id;
37  const scoped_refptr<media::DecoderBuffer> buffer;
38};
39
40VideoDecoderShim::PendingDecode::PendingDecode(
41    uint32_t decode_id,
42    const scoped_refptr<media::DecoderBuffer>& buffer)
43    : decode_id(decode_id), buffer(buffer) {
44}
45
46VideoDecoderShim::PendingDecode::~PendingDecode() {
47}
48
49struct VideoDecoderShim::PendingFrame {
50  explicit PendingFrame(uint32_t decode_id);
51  PendingFrame(uint32_t decode_id,
52               const gfx::Size& coded_size,
53               const gfx::Rect& visible_rect);
54  ~PendingFrame();
55
56  const uint32_t decode_id;
57  const gfx::Size coded_size;
58  const gfx::Rect visible_rect;
59  std::vector<uint8_t> argb_pixels;
60
61 private:
62  // This could be expensive to copy, so guard against that.
63  DISALLOW_COPY_AND_ASSIGN(PendingFrame);
64};
65
66VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
67    : decode_id(decode_id) {
68}
69
70VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id,
71                                             const gfx::Size& coded_size,
72                                             const gfx::Rect& visible_rect)
73    : decode_id(decode_id),
74      coded_size(coded_size),
75      visible_rect(visible_rect),
76      argb_pixels(coded_size.width() * coded_size.height() * 4) {
77}
78
79VideoDecoderShim::PendingFrame::~PendingFrame() {
80}
81
82// DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
83// calls from the VideoDecodeShim on the main thread and sending results back.
84// This class is constructed on the main thread, but used and destructed on the
85// media thread.
86class VideoDecoderShim::DecoderImpl {
87 public:
88  explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
89  ~DecoderImpl();
90
91  void Initialize(media::VideoDecoderConfig config);
92  void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
93  void Reset();
94  void Stop();
95
96 private:
97  void OnPipelineStatus(media::PipelineStatus status);
98  void DoDecode();
99  void OnDecodeComplete(uint32_t decode_id, media::VideoDecoder::Status status);
100  void OnOutputComplete(const scoped_refptr<media::VideoFrame>& frame);
101  void OnResetComplete();
102
103  // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
104  base::WeakPtr<VideoDecoderShim> shim_;
105  scoped_ptr<media::VideoDecoder> decoder_;
106  scoped_refptr<base::MessageLoopProxy> main_message_loop_;
107  // Queue of decodes waiting for the decoder.
108  typedef std::queue<PendingDecode> PendingDecodeQueue;
109  PendingDecodeQueue pending_decodes_;
110  int max_decodes_at_decoder_;
111  int num_decodes_at_decoder_;
112  // VideoDecoder returns pictures without information about the decode buffer
113  // that generated it. Save the decode_id from the last decode that completed,
114  // which is close for most decoders, which only decode one buffer at a time.
115  uint32_t decode_id_;
116};
117
118VideoDecoderShim::DecoderImpl::DecoderImpl(
119    const base::WeakPtr<VideoDecoderShim>& proxy)
120    : shim_(proxy),
121      main_message_loop_(base::MessageLoopProxy::current()),
122      max_decodes_at_decoder_(0),
123      num_decodes_at_decoder_(0),
124      decode_id_(0) {
125}
126
127VideoDecoderShim::DecoderImpl::~DecoderImpl() {
128  DCHECK(pending_decodes_.empty());
129}
130
131void VideoDecoderShim::DecoderImpl::Initialize(
132    media::VideoDecoderConfig config) {
133  DCHECK(!decoder_);
134#if !defined(MEDIA_DISABLE_LIBVPX)
135  if (config.codec() == media::kCodecVP9) {
136    decoder_.reset(
137        new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
138  } else
139#endif
140  {
141    scoped_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder(
142        new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
143    ffmpeg_video_decoder->set_decode_nalus(true);
144    decoder_ = ffmpeg_video_decoder.Pass();
145  }
146  max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests();
147  // We can use base::Unretained() safely in decoder callbacks because
148  // |decoder_| is owned by DecoderImpl. During Stop(), the |decoder_| will be
149  // destroyed and all outstanding callbacks will be fired.
150  decoder_->Initialize(
151      config,
152      true /* low_delay */,
153      base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus,
154                 base::Unretained(this)),
155      base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete,
156                 base::Unretained(this)));
157}
158
159void VideoDecoderShim::DecoderImpl::Decode(
160    uint32_t decode_id,
161    scoped_refptr<media::DecoderBuffer> buffer) {
162  DCHECK(decoder_);
163  pending_decodes_.push(PendingDecode(decode_id, buffer));
164  DoDecode();
165}
166
167void VideoDecoderShim::DecoderImpl::Reset() {
168  DCHECK(decoder_);
169  // Abort all pending decodes.
170  while (!pending_decodes_.empty()) {
171    const PendingDecode& decode = pending_decodes_.front();
172    scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
173    main_message_loop_->PostTask(FROM_HERE,
174                                 base::Bind(&VideoDecoderShim::OnDecodeComplete,
175                                            shim_,
176                                            media::VideoDecoder::kAborted,
177                                            decode.decode_id));
178    pending_decodes_.pop();
179  }
180  decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete,
181                             base::Unretained(this)));
182}
183
184void VideoDecoderShim::DecoderImpl::Stop() {
185  DCHECK(decoder_);
186  // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
187  // again.
188  while (!pending_decodes_.empty())
189    pending_decodes_.pop();
190  decoder_.reset();
191  // This instance is deleted once we exit this scope.
192}
193
194void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
195    media::PipelineStatus status) {
196  int32_t result;
197  switch (status) {
198    case media::PIPELINE_OK:
199      result = PP_OK;
200      break;
201    case media::DECODER_ERROR_NOT_SUPPORTED:
202      result = PP_ERROR_NOTSUPPORTED;
203      break;
204    default:
205      result = PP_ERROR_FAILED;
206      break;
207  }
208
209  // Calculate how many textures the shim should create.
210  uint32_t shim_texture_pool_size =
211      max_decodes_at_decoder_ + media::limits::kMaxVideoFrames;
212  main_message_loop_->PostTask(
213      FROM_HERE,
214      base::Bind(&VideoDecoderShim::OnInitializeComplete,
215                 shim_,
216                 result,
217                 shim_texture_pool_size));
218}
219
220void VideoDecoderShim::DecoderImpl::DoDecode() {
221  while (!pending_decodes_.empty() &&
222         num_decodes_at_decoder_ < max_decodes_at_decoder_) {
223    num_decodes_at_decoder_++;
224    const PendingDecode& decode = pending_decodes_.front();
225    decoder_->Decode(
226        decode.buffer,
227        base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
228                   base::Unretained(this),
229                   decode.decode_id));
230    pending_decodes_.pop();
231  }
232}
233
234void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
235    uint32_t decode_id,
236    media::VideoDecoder::Status status) {
237  num_decodes_at_decoder_--;
238  decode_id_ = decode_id;
239
240  int32_t result;
241  switch (status) {
242    case media::VideoDecoder::kOk:
243    case media::VideoDecoder::kAborted:
244      result = PP_OK;
245      break;
246    case media::VideoDecoder::kDecodeError:
247      result = PP_ERROR_RESOURCE_FAILED;
248      break;
249    default:
250      NOTREACHED();
251      result = PP_ERROR_FAILED;
252      break;
253  }
254
255  main_message_loop_->PostTask(
256      FROM_HERE,
257      base::Bind(
258          &VideoDecoderShim::OnDecodeComplete, shim_, result, decode_id));
259
260  DoDecode();
261}
262
263void VideoDecoderShim::DecoderImpl::OnOutputComplete(
264    const scoped_refptr<media::VideoFrame>& frame) {
265  scoped_ptr<PendingFrame> pending_frame;
266  if (!frame->end_of_stream()) {
267    pending_frame.reset(new PendingFrame(
268        decode_id_, frame->coded_size(), frame->visible_rect()));
269    // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
270    libyuv::I420ToABGR(frame->data(media::VideoFrame::kYPlane),
271                       frame->stride(media::VideoFrame::kYPlane),
272                       frame->data(media::VideoFrame::kUPlane),
273                       frame->stride(media::VideoFrame::kUPlane),
274                       frame->data(media::VideoFrame::kVPlane),
275                       frame->stride(media::VideoFrame::kVPlane),
276                       &pending_frame->argb_pixels.front(),
277                       frame->coded_size().width() * 4,
278                       frame->coded_size().width(),
279                       frame->coded_size().height());
280  } else {
281    pending_frame.reset(new PendingFrame(decode_id_));
282  }
283
284  main_message_loop_->PostTask(FROM_HERE,
285                               base::Bind(&VideoDecoderShim::OnOutputComplete,
286                                          shim_,
287                                          base::Passed(&pending_frame)));
288}
289
290void VideoDecoderShim::DecoderImpl::OnResetComplete() {
291  main_message_loop_->PostTask(
292      FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_));
293}
294
295VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host)
296    : state_(UNINITIALIZED),
297      host_(host),
298      media_task_runner_(
299          RenderThreadImpl::current()->GetMediaThreadTaskRunner()),
300      context_provider_(
301          RenderThreadImpl::current()->SharedMainThreadContextProvider()),
302      texture_pool_size_(0),
303      num_pending_decodes_(0),
304      weak_ptr_factory_(this) {
305  DCHECK(host_);
306  DCHECK(media_task_runner_.get());
307  DCHECK(context_provider_.get());
308  decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr()));
309}
310
311VideoDecoderShim::~VideoDecoderShim() {
312  DCHECK(RenderThreadImpl::current());
313  // Delete any remaining textures.
314  TextureIdMap::iterator it = texture_id_map_.begin();
315  for (; it != texture_id_map_.end(); ++it)
316    DeleteTexture(it->second);
317  texture_id_map_.clear();
318
319  FlushCommandBuffer();
320
321  weak_ptr_factory_.InvalidateWeakPtrs();
322  // No more callbacks from the delegate will be received now.
323
324  // The callback now holds the only reference to the DecoderImpl, which will be
325  // deleted when Stop completes.
326  media_task_runner_->PostTask(
327      FROM_HERE,
328      base::Bind(&VideoDecoderShim::DecoderImpl::Stop,
329                 base::Owned(decoder_impl_.release())));
330}
331
332bool VideoDecoderShim::Initialize(
333    media::VideoCodecProfile profile,
334    media::VideoDecodeAccelerator::Client* client) {
335  DCHECK_EQ(client, host_);
336  DCHECK(RenderThreadImpl::current());
337  DCHECK_EQ(state_, UNINITIALIZED);
338  media::VideoCodec codec = media::kUnknownVideoCodec;
339  if (profile <= media::H264PROFILE_MAX)
340    codec = media::kCodecH264;
341  else if (profile <= media::VP8PROFILE_MAX)
342    codec = media::kCodecVP8;
343  else if (profile <= media::VP9PROFILE_MAX)
344    codec = media::kCodecVP9;
345  DCHECK_NE(codec, media::kUnknownVideoCodec);
346
347  media::VideoDecoderConfig config(
348      codec,
349      profile,
350      media::VideoFrame::YV12,
351      gfx::Size(32, 24),  // Small sizes that won't fail.
352      gfx::Rect(32, 24),
353      gfx::Size(32, 24),
354      NULL /* extra_data */,  // TODO(bbudge) Verify this isn't needed.
355      0 /* extra_data_size */,
356      false /* decryption */);
357
358  media_task_runner_->PostTask(
359      FROM_HERE,
360      base::Bind(&VideoDecoderShim::DecoderImpl::Initialize,
361                 base::Unretained(decoder_impl_.get()),
362                 config));
363  // Return success, even though we are asynchronous, to mimic
364  // media::VideoDecodeAccelerator.
365  return true;
366}
367
368void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) {
369  DCHECK(RenderThreadImpl::current());
370  DCHECK_EQ(state_, DECODING);
371
372  // We need the address of the shared memory, so we can copy the buffer.
373  const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id());
374  DCHECK(buffer);
375
376  media_task_runner_->PostTask(
377      FROM_HERE,
378      base::Bind(
379          &VideoDecoderShim::DecoderImpl::Decode,
380          base::Unretained(decoder_impl_.get()),
381          bitstream_buffer.id(),
382          media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
383  num_pending_decodes_++;
384}
385
386void VideoDecoderShim::AssignPictureBuffers(
387    const std::vector<media::PictureBuffer>& buffers) {
388  DCHECK(RenderThreadImpl::current());
389  DCHECK_EQ(state_, DECODING);
390  if (buffers.empty()) {
391    NOTREACHED();
392    return;
393  }
394  DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size());
395  GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
396  std::vector<uint32_t> local_texture_ids(num_textures);
397  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
398  for (uint32_t i = 0; i < num_textures; i++) {
399    local_texture_ids[i] = gles2->CreateAndConsumeTextureCHROMIUM(
400        GL_TEXTURE_2D, pending_texture_mailboxes_[i].name);
401    // Map the plugin texture id to the local texture id.
402    uint32_t plugin_texture_id = buffers[i].texture_id();
403    texture_id_map_[plugin_texture_id] = local_texture_ids[i];
404    available_textures_.insert(plugin_texture_id);
405  }
406  pending_texture_mailboxes_.clear();
407  SendPictures();
408}
409
410void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) {
411  DCHECK(RenderThreadImpl::current());
412  uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
413  if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
414    DismissTexture(texture_id);
415  } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
416    available_textures_.insert(texture_id);
417    SendPictures();
418  } else {
419    NOTREACHED();
420  }
421}
422
423void VideoDecoderShim::Flush() {
424  DCHECK(RenderThreadImpl::current());
425  DCHECK_EQ(state_, DECODING);
426  state_ = FLUSHING;
427}
428
429void VideoDecoderShim::Reset() {
430  DCHECK(RenderThreadImpl::current());
431  DCHECK_EQ(state_, DECODING);
432  state_ = RESETTING;
433  media_task_runner_->PostTask(
434      FROM_HERE,
435      base::Bind(&VideoDecoderShim::DecoderImpl::Reset,
436                 base::Unretained(decoder_impl_.get())));
437}
438
439void VideoDecoderShim::Destroy() {
440  // This will be called, but our destructor does the actual work.
441}
442
443void VideoDecoderShim::OnInitializeComplete(int32_t result,
444                                            uint32_t texture_pool_size) {
445  DCHECK(RenderThreadImpl::current());
446  DCHECK(host_);
447
448  if (result == PP_OK) {
449    state_ = DECODING;
450    texture_pool_size_ = texture_pool_size;
451  }
452
453  host_->OnInitializeComplete(result);
454}
455
456void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) {
457  DCHECK(RenderThreadImpl::current());
458  DCHECK(host_);
459
460  if (result == PP_ERROR_RESOURCE_FAILED) {
461    host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
462    return;
463  }
464
465  num_pending_decodes_--;
466  completed_decodes_.push(decode_id);
467
468  // If frames are being queued because we're out of textures, don't notify
469  // the host that decode has completed. This exerts "back pressure" to keep
470  // the host from sending buffers that will cause pending_frames_ to grow.
471  if (pending_frames_.empty())
472    NotifyCompletedDecodes();
473}
474
475void VideoDecoderShim::OnOutputComplete(scoped_ptr<PendingFrame> frame) {
476  DCHECK(RenderThreadImpl::current());
477  DCHECK(host_);
478
479  if (!frame->argb_pixels.empty()) {
480    if (texture_size_ != frame->coded_size) {
481      // If the size has changed, all current textures must be dismissed. Add
482      // all textures to |textures_to_dismiss_| and dismiss any that aren't in
483      // use by the plugin. We will dismiss the rest as they are recycled.
484      for (TextureIdMap::const_iterator it = texture_id_map_.begin();
485           it != texture_id_map_.end();
486           ++it) {
487        textures_to_dismiss_.insert(it->second);
488      }
489      for (TextureIdSet::const_iterator it = available_textures_.begin();
490           it != available_textures_.end();
491           ++it) {
492        DismissTexture(*it);
493      }
494      available_textures_.clear();
495      FlushCommandBuffer();
496
497      DCHECK(pending_texture_mailboxes_.empty());
498      for (uint32_t i = 0; i < texture_pool_size_; i++)
499        pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
500
501      host_->RequestTextures(texture_pool_size_,
502                             frame->coded_size,
503                             GL_TEXTURE_2D,
504                             pending_texture_mailboxes_);
505      texture_size_ = frame->coded_size;
506    }
507
508    pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
509    SendPictures();
510  }
511}
512
513void VideoDecoderShim::SendPictures() {
514  DCHECK(RenderThreadImpl::current());
515  DCHECK(host_);
516  while (!pending_frames_.empty() && !available_textures_.empty()) {
517    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
518
519    TextureIdSet::iterator it = available_textures_.begin();
520    uint32_t texture_id = *it;
521    available_textures_.erase(it);
522
523    uint32_t local_texture_id = texture_id_map_[texture_id];
524    gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
525    gles2->ActiveTexture(GL_TEXTURE0);
526    gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
527    gles2->TexImage2D(GL_TEXTURE_2D,
528                      0,
529                      GL_RGBA,
530                      texture_size_.width(),
531                      texture_size_.height(),
532                      0,
533                      GL_RGBA,
534                      GL_UNSIGNED_BYTE,
535                      &frame->argb_pixels.front());
536
537    host_->PictureReady(
538        media::Picture(texture_id, frame->decode_id, frame->visible_rect));
539    pending_frames_.pop();
540  }
541
542  FlushCommandBuffer();
543
544  if (pending_frames_.empty()) {
545    // If frames aren't backing up, notify the host of any completed decodes so
546    // it can send more buffers.
547    NotifyCompletedDecodes();
548
549    if (state_ == FLUSHING && !num_pending_decodes_) {
550      state_ = DECODING;
551      host_->NotifyFlushDone();
552    }
553  }
554}
555
556void VideoDecoderShim::OnResetComplete() {
557  DCHECK(RenderThreadImpl::current());
558  DCHECK(host_);
559
560  while (!pending_frames_.empty())
561    pending_frames_.pop();
562  NotifyCompletedDecodes();
563
564  // Dismiss any old textures now.
565  while (!textures_to_dismiss_.empty())
566    DismissTexture(*textures_to_dismiss_.begin());
567
568  state_ = DECODING;
569  host_->NotifyResetDone();
570}
571
572void VideoDecoderShim::NotifyCompletedDecodes() {
573  while (!completed_decodes_.empty()) {
574    host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front());
575    completed_decodes_.pop();
576  }
577}
578
579void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
580  DCHECK(host_);
581  textures_to_dismiss_.erase(texture_id);
582  DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
583  DeleteTexture(texture_id_map_[texture_id]);
584  texture_id_map_.erase(texture_id);
585  host_->DismissPictureBuffer(texture_id);
586}
587
588void VideoDecoderShim::DeleteTexture(uint32_t texture_id) {
589  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
590  gles2->DeleteTextures(1, &texture_id);
591}
592
593void VideoDecoderShim::FlushCommandBuffer() {
594  context_provider_->ContextGL()->Flush();
595}
596
597}  // namespace content
598