1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/renderer/pepper/video_decoder_shim.h"
6
7#include <GLES2/gl2.h>
8#include <GLES2/gl2ext.h>
9#include <GLES2/gl2extchromium.h>
10
11#include "base/bind.h"
12#include "base/numerics/safe_conversions.h"
13#include "content/public/renderer/render_thread.h"
14#include "content/renderer/pepper/pepper_video_decoder_host.h"
15#include "content/renderer/render_thread_impl.h"
16#include "gpu/command_buffer/client/gles2_implementation.h"
17#include "media/base/decoder_buffer.h"
18#include "media/base/limits.h"
19#include "media/base/video_decoder.h"
20#include "media/filters/ffmpeg_video_decoder.h"
21#include "media/filters/vpx_video_decoder.h"
22#include "media/video/picture.h"
23#include "media/video/video_decode_accelerator.h"
24#include "ppapi/c/pp_errors.h"
25#include "third_party/libyuv/include/libyuv.h"
26#include "webkit/common/gpu/context_provider_web_context.h"
27
28namespace content {
29
30struct VideoDecoderShim::PendingDecode {
31  PendingDecode(uint32_t decode_id,
32                const scoped_refptr<media::DecoderBuffer>& buffer);
33  ~PendingDecode();
34
35  const uint32_t decode_id;
36  const scoped_refptr<media::DecoderBuffer> buffer;
37};
38
39VideoDecoderShim::PendingDecode::PendingDecode(
40    uint32_t decode_id,
41    const scoped_refptr<media::DecoderBuffer>& buffer)
42    : decode_id(decode_id), buffer(buffer) {
43}
44
45VideoDecoderShim::PendingDecode::~PendingDecode() {
46}
47
48struct VideoDecoderShim::PendingFrame {
49  explicit PendingFrame(uint32_t decode_id);
50  PendingFrame(uint32_t decode_id, const gfx::Size& size);
51  ~PendingFrame();
52
53  const uint32_t decode_id;
54  const gfx::Size size;
55  std::vector<uint8_t> argb_pixels;
56
57 private:
58  // This could be expensive to copy, so guard against that.
59  DISALLOW_COPY_AND_ASSIGN(PendingFrame);
60};
61
62VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
63    : decode_id(decode_id) {
64}
65
66VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id,
67                                             const gfx::Size& size)
68    : decode_id(decode_id),
69      size(size),
70      argb_pixels(size.width() * size.height() * 4) {
71}
72
73VideoDecoderShim::PendingFrame::~PendingFrame() {
74}
75
76// DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
77// calls from the VideoDecodeShim on the main thread and sending results back.
78// This class is constructed on the main thread, but used and destructed on the
79// media thread.
80class VideoDecoderShim::DecoderImpl {
81 public:
82  explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
83  ~DecoderImpl();
84
85  void Initialize(media::VideoDecoderConfig config);
86  void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
87  void Reset();
88  void Stop();
89
90 private:
91  void OnPipelineStatus(media::PipelineStatus status);
92  void DoDecode();
93  void OnDecodeComplete(uint32_t decode_id, media::VideoDecoder::Status status);
94  void OnOutputComplete(const scoped_refptr<media::VideoFrame>& frame);
95  void OnResetComplete();
96
97  // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
98  base::WeakPtr<VideoDecoderShim> shim_;
99  scoped_ptr<media::VideoDecoder> decoder_;
100  scoped_refptr<base::MessageLoopProxy> main_message_loop_;
101  // Queue of decodes waiting for the decoder.
102  typedef std::queue<PendingDecode> PendingDecodeQueue;
103  PendingDecodeQueue pending_decodes_;
104  int max_decodes_at_decoder_;
105  int num_decodes_at_decoder_;
106  // VideoDecoder returns pictures without information about the decode buffer
107  // that generated it. Save the decode_id from the last decode that completed,
108  // which is close for most decoders, which only decode one buffer at a time.
109  uint32_t decode_id_;
110};
111
112VideoDecoderShim::DecoderImpl::DecoderImpl(
113    const base::WeakPtr<VideoDecoderShim>& proxy)
114    : shim_(proxy),
115      main_message_loop_(base::MessageLoopProxy::current()),
116      max_decodes_at_decoder_(0),
117      num_decodes_at_decoder_(0),
118      decode_id_(0) {
119}
120
121VideoDecoderShim::DecoderImpl::~DecoderImpl() {
122  DCHECK(pending_decodes_.empty());
123}
124
125void VideoDecoderShim::DecoderImpl::Initialize(
126    media::VideoDecoderConfig config) {
127  DCHECK(!decoder_);
128  if (config.codec() == media::kCodecVP9) {
129    decoder_.reset(
130        new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
131  } else {
132    scoped_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder(
133        new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
134    ffmpeg_video_decoder->set_decode_nalus(true);
135    decoder_ = ffmpeg_video_decoder.Pass();
136  }
137  max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests();
138  // We can use base::Unretained() safely in decoder callbacks because we call
139  // VideoDecoder::Stop() before deletion. Stop() guarantees there will be no
140  // outstanding callbacks after it returns.
141  decoder_->Initialize(
142      config,
143      true /* low_delay */,
144      base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus,
145                 base::Unretained(this)),
146      base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete,
147                 base::Unretained(this)));
148}
149
150void VideoDecoderShim::DecoderImpl::Decode(
151    uint32_t decode_id,
152    scoped_refptr<media::DecoderBuffer> buffer) {
153  DCHECK(decoder_);
154  pending_decodes_.push(PendingDecode(decode_id, buffer));
155  DoDecode();
156}
157
158void VideoDecoderShim::DecoderImpl::Reset() {
159  DCHECK(decoder_);
160  // Abort all pending decodes.
161  while (!pending_decodes_.empty()) {
162    const PendingDecode& decode = pending_decodes_.front();
163    scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
164    main_message_loop_->PostTask(FROM_HERE,
165                                 base::Bind(&VideoDecoderShim::OnDecodeComplete,
166                                            shim_,
167                                            media::VideoDecoder::kAborted,
168                                            decode.decode_id));
169    pending_decodes_.pop();
170  }
171  decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete,
172                             base::Unretained(this)));
173}
174
175void VideoDecoderShim::DecoderImpl::Stop() {
176  DCHECK(decoder_);
177  // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
178  // again.
179  while (!pending_decodes_.empty())
180    pending_decodes_.pop();
181  decoder_->Stop();
182  // This instance is deleted once we exit this scope.
183}
184
185void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
186    media::PipelineStatus status) {
187  int32_t result;
188  switch (status) {
189    case media::PIPELINE_OK:
190      result = PP_OK;
191      break;
192    case media::DECODER_ERROR_NOT_SUPPORTED:
193      result = PP_ERROR_NOTSUPPORTED;
194      break;
195    default:
196      result = PP_ERROR_FAILED;
197      break;
198  }
199
200  // Calculate how many textures the shim should create.
201  uint32_t shim_texture_pool_size =
202      max_decodes_at_decoder_ + media::limits::kMaxVideoFrames;
203  main_message_loop_->PostTask(
204      FROM_HERE,
205      base::Bind(&VideoDecoderShim::OnInitializeComplete,
206                 shim_,
207                 result,
208                 shim_texture_pool_size));
209}
210
211void VideoDecoderShim::DecoderImpl::DoDecode() {
212  while (!pending_decodes_.empty() &&
213         num_decodes_at_decoder_ < max_decodes_at_decoder_) {
214    num_decodes_at_decoder_++;
215    const PendingDecode& decode = pending_decodes_.front();
216    decoder_->Decode(
217        decode.buffer,
218        base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
219                   base::Unretained(this),
220                   decode.decode_id));
221    pending_decodes_.pop();
222  }
223}
224
225void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
226    uint32_t decode_id,
227    media::VideoDecoder::Status status) {
228  num_decodes_at_decoder_--;
229  decode_id_ = decode_id;
230
231  int32_t result;
232  switch (status) {
233    case media::VideoDecoder::kOk:
234    case media::VideoDecoder::kAborted:
235      result = PP_OK;
236      break;
237    case media::VideoDecoder::kDecodeError:
238      result = PP_ERROR_RESOURCE_FAILED;
239      break;
240    default:
241      NOTREACHED();
242      result = PP_ERROR_FAILED;
243      break;
244  }
245
246  main_message_loop_->PostTask(
247      FROM_HERE,
248      base::Bind(
249          &VideoDecoderShim::OnDecodeComplete, shim_, result, decode_id));
250
251  DoDecode();
252}
253
254void VideoDecoderShim::DecoderImpl::OnOutputComplete(
255    const scoped_refptr<media::VideoFrame>& frame) {
256  scoped_ptr<PendingFrame> pending_frame;
257  if (!frame->end_of_stream()) {
258    pending_frame.reset(new PendingFrame(decode_id_, frame->coded_size()));
259    // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
260    libyuv::I420ToABGR(frame->data(media::VideoFrame::kYPlane),
261                       frame->stride(media::VideoFrame::kYPlane),
262                       frame->data(media::VideoFrame::kUPlane),
263                       frame->stride(media::VideoFrame::kUPlane),
264                       frame->data(media::VideoFrame::kVPlane),
265                       frame->stride(media::VideoFrame::kVPlane),
266                       &pending_frame->argb_pixels.front(),
267                       frame->coded_size().width() * 4,
268                       frame->coded_size().width(),
269                       frame->coded_size().height());
270  } else {
271    pending_frame.reset(new PendingFrame(decode_id_));
272  }
273
274  main_message_loop_->PostTask(FROM_HERE,
275                               base::Bind(&VideoDecoderShim::OnOutputComplete,
276                                          shim_,
277                                          base::Passed(&pending_frame)));
278}
279
280void VideoDecoderShim::DecoderImpl::OnResetComplete() {
281  main_message_loop_->PostTask(
282      FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_));
283}
284
285VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host)
286    : state_(UNINITIALIZED),
287      host_(host),
288      media_message_loop_(
289          RenderThreadImpl::current()->GetMediaThreadMessageLoopProxy()),
290      context_provider_(
291          RenderThreadImpl::current()->SharedMainThreadContextProvider()),
292      texture_pool_size_(0),
293      num_pending_decodes_(0),
294      weak_ptr_factory_(this) {
295  DCHECK(host_);
296  DCHECK(media_message_loop_);
297  DCHECK(context_provider_);
298  decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr()));
299}
300
301VideoDecoderShim::~VideoDecoderShim() {
302  DCHECK(RenderThreadImpl::current());
303  // Delete any remaining textures.
304  TextureIdMap::iterator it = texture_id_map_.begin();
305  for (; it != texture_id_map_.end(); ++it)
306    DeleteTexture(it->second);
307  texture_id_map_.clear();
308
309  FlushCommandBuffer();
310
311  weak_ptr_factory_.InvalidateWeakPtrs();
312  // No more callbacks from the delegate will be received now.
313
314  // The callback now holds the only reference to the DecoderImpl, which will be
315  // deleted when Stop completes.
316  media_message_loop_->PostTask(
317      FROM_HERE,
318      base::Bind(&VideoDecoderShim::DecoderImpl::Stop,
319                 base::Owned(decoder_impl_.release())));
320}
321
322bool VideoDecoderShim::Initialize(
323    media::VideoCodecProfile profile,
324    media::VideoDecodeAccelerator::Client* client) {
325  DCHECK_EQ(client, host_);
326  DCHECK(RenderThreadImpl::current());
327  DCHECK_EQ(state_, UNINITIALIZED);
328  media::VideoCodec codec = media::kUnknownVideoCodec;
329  if (profile <= media::H264PROFILE_MAX)
330    codec = media::kCodecH264;
331  else if (profile <= media::VP8PROFILE_MAX)
332    codec = media::kCodecVP8;
333  else if (profile <= media::VP9PROFILE_MAX)
334    codec = media::kCodecVP9;
335  DCHECK_NE(codec, media::kUnknownVideoCodec);
336
337  media::VideoDecoderConfig config(
338      codec,
339      profile,
340      media::VideoFrame::YV12,
341      gfx::Size(32, 24),  // Small sizes that won't fail.
342      gfx::Rect(32, 24),
343      gfx::Size(32, 24),
344      NULL /* extra_data */,  // TODO(bbudge) Verify this isn't needed.
345      0 /* extra_data_size */,
346      false /* decryption */);
347
348  media_message_loop_->PostTask(
349      FROM_HERE,
350      base::Bind(&VideoDecoderShim::DecoderImpl::Initialize,
351                 base::Unretained(decoder_impl_.get()),
352                 config));
353  // Return success, even though we are asynchronous, to mimic
354  // media::VideoDecodeAccelerator.
355  return true;
356}
357
358void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) {
359  DCHECK(RenderThreadImpl::current());
360  DCHECK_EQ(state_, DECODING);
361
362  // We need the address of the shared memory, so we can copy the buffer.
363  const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id());
364  DCHECK(buffer);
365
366  media_message_loop_->PostTask(
367      FROM_HERE,
368      base::Bind(
369          &VideoDecoderShim::DecoderImpl::Decode,
370          base::Unretained(decoder_impl_.get()),
371          bitstream_buffer.id(),
372          media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
373  num_pending_decodes_++;
374}
375
376void VideoDecoderShim::AssignPictureBuffers(
377    const std::vector<media::PictureBuffer>& buffers) {
378  DCHECK(RenderThreadImpl::current());
379  DCHECK_EQ(state_, DECODING);
380  if (buffers.empty()) {
381    NOTREACHED();
382    return;
383  }
384  DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size());
385  GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
386  std::vector<uint32_t> local_texture_ids(num_textures);
387  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
388  gles2->GenTextures(num_textures, &local_texture_ids.front());
389  for (uint32_t i = 0; i < num_textures; i++) {
390    gles2->ActiveTexture(GL_TEXTURE0);
391    gles2->BindTexture(GL_TEXTURE_2D, local_texture_ids[i]);
392    gles2->ConsumeTextureCHROMIUM(GL_TEXTURE_2D,
393                                  pending_texture_mailboxes_[i].name);
394    // Map the plugin texture id to the local texture id.
395    uint32_t plugin_texture_id = buffers[i].texture_id();
396    texture_id_map_[plugin_texture_id] = local_texture_ids[i];
397    available_textures_.insert(plugin_texture_id);
398  }
399  pending_texture_mailboxes_.clear();
400  SendPictures();
401}
402
403void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) {
404  DCHECK(RenderThreadImpl::current());
405  uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
406  if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
407    DismissTexture(texture_id);
408  } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
409    available_textures_.insert(texture_id);
410    SendPictures();
411  } else {
412    NOTREACHED();
413  }
414}
415
416void VideoDecoderShim::Flush() {
417  DCHECK(RenderThreadImpl::current());
418  DCHECK_EQ(state_, DECODING);
419  state_ = FLUSHING;
420}
421
422void VideoDecoderShim::Reset() {
423  DCHECK(RenderThreadImpl::current());
424  DCHECK_EQ(state_, DECODING);
425  state_ = RESETTING;
426  media_message_loop_->PostTask(
427      FROM_HERE,
428      base::Bind(&VideoDecoderShim::DecoderImpl::Reset,
429                 base::Unretained(decoder_impl_.get())));
430}
431
432void VideoDecoderShim::Destroy() {
433  // This will be called, but our destructor does the actual work.
434}
435
436void VideoDecoderShim::OnInitializeComplete(int32_t result,
437                                            uint32_t texture_pool_size) {
438  DCHECK(RenderThreadImpl::current());
439  DCHECK(host_);
440
441  if (result == PP_OK) {
442    state_ = DECODING;
443    texture_pool_size_ = texture_pool_size;
444  }
445
446  host_->OnInitializeComplete(result);
447}
448
449void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) {
450  DCHECK(RenderThreadImpl::current());
451  DCHECK(host_);
452
453  if (result == PP_ERROR_RESOURCE_FAILED) {
454    host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
455    return;
456  }
457
458  num_pending_decodes_--;
459  completed_decodes_.push(decode_id);
460
461  // If frames are being queued because we're out of textures, don't notify
462  // the host that decode has completed. This exerts "back pressure" to keep
463  // the host from sending buffers that will cause pending_frames_ to grow.
464  if (pending_frames_.empty())
465    NotifyCompletedDecodes();
466}
467
468void VideoDecoderShim::OnOutputComplete(scoped_ptr<PendingFrame> frame) {
469  DCHECK(RenderThreadImpl::current());
470  DCHECK(host_);
471
472  if (!frame->argb_pixels.empty()) {
473    if (texture_size_ != frame->size) {
474      // If the size has changed, all current textures must be dismissed. Add
475      // all textures to |textures_to_dismiss_| and dismiss any that aren't in
476      // use by the plugin. We will dismiss the rest as they are recycled.
477      for (TextureIdMap::const_iterator it = texture_id_map_.begin();
478           it != texture_id_map_.end();
479           ++it) {
480        textures_to_dismiss_.insert(it->second);
481      }
482      for (TextureIdSet::const_iterator it = available_textures_.begin();
483           it != available_textures_.end();
484           ++it) {
485        DismissTexture(*it);
486      }
487      available_textures_.clear();
488      FlushCommandBuffer();
489
490      DCHECK(pending_texture_mailboxes_.empty());
491      for (uint32_t i = 0; i < texture_pool_size_; i++)
492        pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
493
494      host_->RequestTextures(texture_pool_size_,
495                             frame->size,
496                             GL_TEXTURE_2D,
497                             pending_texture_mailboxes_);
498      texture_size_ = frame->size;
499    }
500
501    pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
502    SendPictures();
503  }
504}
505
506void VideoDecoderShim::SendPictures() {
507  DCHECK(RenderThreadImpl::current());
508  DCHECK(host_);
509  while (!pending_frames_.empty() && !available_textures_.empty()) {
510    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
511
512    TextureIdSet::iterator it = available_textures_.begin();
513    uint32_t texture_id = *it;
514    available_textures_.erase(it);
515
516    uint32_t local_texture_id = texture_id_map_[texture_id];
517    gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
518    gles2->ActiveTexture(GL_TEXTURE0);
519    gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
520    gles2->TexImage2D(GL_TEXTURE_2D,
521                      0,
522                      GL_RGBA,
523                      texture_size_.width(),
524                      texture_size_.height(),
525                      0,
526                      GL_RGBA,
527                      GL_UNSIGNED_BYTE,
528                      &frame->argb_pixels.front());
529
530    host_->PictureReady(media::Picture(texture_id, frame->decode_id));
531    pending_frames_.pop();
532  }
533
534  FlushCommandBuffer();
535
536  if (pending_frames_.empty()) {
537    // If frames aren't backing up, notify the host of any completed decodes so
538    // it can send more buffers.
539    NotifyCompletedDecodes();
540
541    if (state_ == FLUSHING && !num_pending_decodes_) {
542      state_ = DECODING;
543      host_->NotifyFlushDone();
544    }
545  }
546}
547
548void VideoDecoderShim::OnResetComplete() {
549  DCHECK(RenderThreadImpl::current());
550  DCHECK(host_);
551
552  while (!pending_frames_.empty())
553    pending_frames_.pop();
554  NotifyCompletedDecodes();
555
556  // Dismiss any old textures now.
557  while (!textures_to_dismiss_.empty())
558    DismissTexture(*textures_to_dismiss_.begin());
559
560  state_ = DECODING;
561  host_->NotifyResetDone();
562}
563
564void VideoDecoderShim::NotifyCompletedDecodes() {
565  while (!completed_decodes_.empty()) {
566    host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front());
567    completed_decodes_.pop();
568  }
569}
570
571void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
572  DCHECK(host_);
573  textures_to_dismiss_.erase(texture_id);
574  DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
575  DeleteTexture(texture_id_map_[texture_id]);
576  texture_id_map_.erase(texture_id);
577  host_->DismissPictureBuffer(texture_id);
578}
579
580void VideoDecoderShim::DeleteTexture(uint32_t texture_id) {
581  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
582  gles2->DeleteTextures(1, &texture_id);
583}
584
585void VideoDecoderShim::FlushCommandBuffer() {
586  context_provider_->ContextGL()->Flush();
587}
588
589}  // namespace content
590