1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/renderer/media/webmediaplayer_ms.h"
6
7#include <limits>
8
9#include "base/bind.h"
10#include "base/callback.h"
11#include "base/message_loop/message_loop.h"
12#include "base/metrics/histogram.h"
13#include "cc/blink/web_layer_impl.h"
14#include "cc/layers/video_layer.h"
15#include "content/public/renderer/render_view.h"
16#include "content/renderer/media/media_stream_audio_renderer.h"
17#include "content/renderer/media/media_stream_renderer_factory.h"
18#include "content/renderer/media/video_frame_provider.h"
19#include "content/renderer/render_frame_impl.h"
20#include "media/base/media_log.h"
21#include "media/base/video_frame.h"
22#include "media/base/video_rotation.h"
23#include "media/base/video_util.h"
24#include "media/blink/webmediaplayer_delegate.h"
25#include "media/blink/webmediaplayer_util.h"
26#include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
27#include "third_party/WebKit/public/platform/WebRect.h"
28#include "third_party/WebKit/public/platform/WebSize.h"
29#include "third_party/WebKit/public/platform/WebURL.h"
30#include "third_party/WebKit/public/web/WebFrame.h"
31#include "third_party/WebKit/public/web/WebView.h"
32#include "third_party/skia/include/core/SkBitmap.h"
33
34using blink::WebCanvas;
35using blink::WebMediaPlayer;
36using blink::WebRect;
37using blink::WebSize;
38
39namespace {
40
41// This function copies a YV12 or NATIVE_TEXTURE to a new YV12
42// media::VideoFrame.
43scoped_refptr<media::VideoFrame> CopyFrameToYV12(
44    const scoped_refptr<media::VideoFrame>& frame) {
45  DCHECK(frame->format() == media::VideoFrame::YV12 ||
46         frame->format() == media::VideoFrame::I420 ||
47         frame->format() == media::VideoFrame::NATIVE_TEXTURE);
48  scoped_refptr<media::VideoFrame> new_frame =
49      media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
50                                     frame->coded_size(),
51                                     frame->visible_rect(),
52                                     frame->natural_size(),
53                                     frame->timestamp());
54
55  if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
56    SkBitmap bitmap;
57    bitmap.allocN32Pixels(frame->visible_rect().width(),
58                          frame->visible_rect().height());
59    frame->ReadPixelsFromNativeTexture(bitmap);
60
61    media::CopyRGBToVideoFrame(
62        reinterpret_cast<uint8*>(bitmap.getPixels()),
63        bitmap.rowBytes(),
64        frame->visible_rect(),
65        new_frame.get());
66  } else {
67    size_t number_of_planes =
68        media::VideoFrame::NumPlanes(frame->format());
69    for (size_t i = 0; i < number_of_planes; ++i) {
70      media::CopyPlane(i, frame->data(i), frame->stride(i),
71                       frame->rows(i), new_frame.get());
72    }
73  }
74  return new_frame;
75}
76
77}  // anonymous namespace
78
79namespace content {
80
81WebMediaPlayerMS::WebMediaPlayerMS(
82    blink::WebFrame* frame,
83    blink::WebMediaPlayerClient* client,
84    base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
85    media::MediaLog* media_log,
86    scoped_ptr<MediaStreamRendererFactory> factory)
87    : frame_(frame),
88      network_state_(WebMediaPlayer::NetworkStateEmpty),
89      ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
90      buffered_(static_cast<size_t>(1)),
91      volume_(1.0f),
92      client_(client),
93      delegate_(delegate),
94      paused_(true),
95      current_frame_used_(false),
96      pending_repaint_(false),
97      video_frame_provider_client_(NULL),
98      received_first_frame_(false),
99      total_frame_count_(0),
100      dropped_frame_count_(0),
101      media_log_(media_log),
102      renderer_factory_(factory.Pass()) {
103  DVLOG(1) << "WebMediaPlayerMS::ctor";
104  media_log_->AddEvent(
105      media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
106}
107
108WebMediaPlayerMS::~WebMediaPlayerMS() {
109  DVLOG(1) << "WebMediaPlayerMS::dtor";
110  DCHECK(thread_checker_.CalledOnValidThread());
111
112  SetVideoFrameProviderClient(NULL);
113  GetClient()->setWebLayer(NULL);
114
115  if (video_frame_provider_.get())
116    video_frame_provider_->Stop();
117
118  if (audio_renderer_.get())
119    audio_renderer_->Stop();
120
121  media_log_->AddEvent(
122      media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
123
124  if (delegate_.get())
125    delegate_->PlayerGone(this);
126}
127
128void WebMediaPlayerMS::load(LoadType load_type,
129                            const blink::WebURL& url,
130                            CORSMode cors_mode) {
131  DVLOG(1) << "WebMediaPlayerMS::load";
132  DCHECK(thread_checker_.CalledOnValidThread());
133
134  // TODO(acolwell): Change this to DCHECK_EQ(load_type,
135  // LoadTypeMediaStream) once Blink-side changes land.
136  DCHECK_NE(load_type, LoadTypeMediaSource);
137
138  GURL gurl(url);
139
140  SetNetworkState(WebMediaPlayer::NetworkStateLoading);
141  SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
142  media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
143
144  video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
145      url,
146      base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
147      base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
148
149  RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
150  audio_renderer_ = renderer_factory_->GetAudioRenderer(
151    url,
152    frame->GetRenderView()->GetRoutingID(),
153    frame->GetRoutingID());
154
155  if (video_frame_provider_.get() || audio_renderer_.get()) {
156    if (audio_renderer_.get()) {
157      audio_renderer_->SetVolume(volume_);
158      audio_renderer_->Start();
159    }
160
161    if (video_frame_provider_.get()) {
162      video_frame_provider_->Start();
163    } else {
164      // This is audio-only mode.
165      DCHECK(audio_renderer_.get());
166      SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
167      SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
168    }
169  } else {
170    SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
171  }
172}
173
174void WebMediaPlayerMS::play() {
175  DVLOG(1) << "WebMediaPlayerMS::play";
176  DCHECK(thread_checker_.CalledOnValidThread());
177
178  if (paused_) {
179    if (video_frame_provider_.get())
180      video_frame_provider_->Play();
181
182    if (audio_renderer_.get())
183      audio_renderer_->Play();
184
185    if (delegate_.get())
186      delegate_->DidPlay(this);
187  }
188
189  paused_ = false;
190
191  media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
192}
193
194void WebMediaPlayerMS::pause() {
195  DVLOG(1) << "WebMediaPlayerMS::pause";
196  DCHECK(thread_checker_.CalledOnValidThread());
197
198  if (video_frame_provider_.get())
199    video_frame_provider_->Pause();
200
201  if (!paused_) {
202    if (audio_renderer_.get())
203      audio_renderer_->Pause();
204
205    if (delegate_.get())
206      delegate_->DidPause(this);
207  }
208
209  paused_ = true;
210
211  media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
212
213  if (!current_frame_.get())
214    return;
215
216  // Copy the frame so that rendering can show the last received frame.
217  // The original frame must not be referenced when the player is paused since
218  // there might be a finite number of available buffers. E.g, video that
219  // originates from a video camera.
220  scoped_refptr<media::VideoFrame> new_frame = CopyFrameToYV12(current_frame_);
221  base::AutoLock auto_lock(current_frame_lock_);
222  current_frame_ = new_frame;
223}
224
225bool WebMediaPlayerMS::supportsSave() const {
226  DCHECK(thread_checker_.CalledOnValidThread());
227  return false;
228}
229
230void WebMediaPlayerMS::seek(double seconds) {
231  DCHECK(thread_checker_.CalledOnValidThread());
232}
233
234void WebMediaPlayerMS::setRate(double rate) {
235  DCHECK(thread_checker_.CalledOnValidThread());
236}
237
238void WebMediaPlayerMS::setVolume(double volume) {
239  DCHECK(thread_checker_.CalledOnValidThread());
240  DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
241  volume_ = volume;
242  if (audio_renderer_.get())
243    audio_renderer_->SetVolume(volume_);
244}
245
246void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
247  DCHECK(thread_checker_.CalledOnValidThread());
248}
249
250bool WebMediaPlayerMS::hasVideo() const {
251  DCHECK(thread_checker_.CalledOnValidThread());
252  return (video_frame_provider_.get() != NULL);
253}
254
255bool WebMediaPlayerMS::hasAudio() const {
256  DCHECK(thread_checker_.CalledOnValidThread());
257  return (audio_renderer_.get() != NULL);
258}
259
260blink::WebSize WebMediaPlayerMS::naturalSize() const {
261  DCHECK(thread_checker_.CalledOnValidThread());
262
263  gfx::Size size;
264  if (current_frame_.get())
265    size = current_frame_->natural_size();
266  DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
267  return blink::WebSize(size);
268}
269
270bool WebMediaPlayerMS::paused() const {
271  DCHECK(thread_checker_.CalledOnValidThread());
272  return paused_;
273}
274
275bool WebMediaPlayerMS::seeking() const {
276  DCHECK(thread_checker_.CalledOnValidThread());
277  return false;
278}
279
280double WebMediaPlayerMS::duration() const {
281  DCHECK(thread_checker_.CalledOnValidThread());
282  return std::numeric_limits<double>::infinity();
283}
284
285double WebMediaPlayerMS::currentTime() const {
286  DCHECK(thread_checker_.CalledOnValidThread());
287  if (current_time_.ToInternalValue() != 0) {
288    return current_time_.InSecondsF();
289  } else if (audio_renderer_.get()) {
290    return audio_renderer_->GetCurrentRenderTime().InSecondsF();
291  }
292  return 0.0;
293}
294
295WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
296  DCHECK(thread_checker_.CalledOnValidThread());
297  DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
298  return network_state_;
299}
300
301WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
302  DCHECK(thread_checker_.CalledOnValidThread());
303  DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
304  return ready_state_;
305}
306
307blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
308  DCHECK(thread_checker_.CalledOnValidThread());
309  return buffered_;
310}
311
312double WebMediaPlayerMS::maxTimeSeekable() const {
313  DCHECK(thread_checker_.CalledOnValidThread());
314  return 0.0;
315}
316
317bool WebMediaPlayerMS::didLoadingProgress() {
318  DCHECK(thread_checker_.CalledOnValidThread());
319  return true;
320}
321
322void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
323                             const blink::WebRect& rect,
324                             unsigned char alpha) {
325  paint(canvas, rect, alpha, SkXfermode::kSrcOver_Mode);
326}
327
328void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
329                             const blink::WebRect& rect,
330                             unsigned char alpha,
331                             SkXfermode::Mode mode) {
332  DVLOG(3) << "WebMediaPlayerMS::paint";
333  DCHECK(thread_checker_.CalledOnValidThread());
334
335  gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
336  video_renderer_.Paint(
337      current_frame_, canvas, dest_rect, alpha, mode, media::VIDEO_ROTATION_0);
338
339  {
340    base::AutoLock auto_lock(current_frame_lock_);
341    if (current_frame_.get())
342      current_frame_used_ = true;
343  }
344}
345
346bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
347  DCHECK(thread_checker_.CalledOnValidThread());
348  return true;
349}
350
351bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
352  DCHECK(thread_checker_.CalledOnValidThread());
353  return true;
354}
355
356double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
357  return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
358}
359
360unsigned WebMediaPlayerMS::decodedFrameCount() const {
361  DCHECK(thread_checker_.CalledOnValidThread());
362  DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
363  return total_frame_count_;
364}
365
366unsigned WebMediaPlayerMS::droppedFrameCount() const {
367  DCHECK(thread_checker_.CalledOnValidThread());
368  DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
369  return dropped_frame_count_;
370}
371
372unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
373  DCHECK(thread_checker_.CalledOnValidThread());
374  NOTIMPLEMENTED();
375  return 0;
376}
377
378unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
379  DCHECK(thread_checker_.CalledOnValidThread());
380  NOTIMPLEMENTED();
381  return 0;
382}
383
384void WebMediaPlayerMS::SetVideoFrameProviderClient(
385    cc::VideoFrameProvider::Client* client) {
386  // This is called from both the main renderer thread and the compositor
387  // thread (when the main thread is blocked).
388  if (video_frame_provider_client_)
389    video_frame_provider_client_->StopUsingProvider();
390  video_frame_provider_client_ = client;
391}
392
393scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
394  DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
395  base::AutoLock auto_lock(current_frame_lock_);
396  DCHECK(!pending_repaint_);
397  if (!current_frame_.get())
398    return NULL;
399  pending_repaint_ = true;
400  current_frame_used_ = true;
401  return current_frame_;
402}
403
404void WebMediaPlayerMS::PutCurrentFrame(
405    const scoped_refptr<media::VideoFrame>& frame) {
406  DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
407  DCHECK(pending_repaint_);
408  pending_repaint_ = false;
409}
410
411void WebMediaPlayerMS::OnFrameAvailable(
412    const scoped_refptr<media::VideoFrame>& frame) {
413  DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
414  DCHECK(thread_checker_.CalledOnValidThread());
415  ++total_frame_count_;
416  if (!received_first_frame_) {
417    received_first_frame_ = true;
418    {
419      base::AutoLock auto_lock(current_frame_lock_);
420      DCHECK(!current_frame_used_);
421      current_frame_ = frame;
422    }
423    SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
424    SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
425    GetClient()->sizeChanged();
426
427    if (video_frame_provider_.get()) {
428      video_weblayer_.reset(new cc_blink::WebLayerImpl(
429          cc::VideoLayer::Create(this, media::VIDEO_ROTATION_0)));
430      video_weblayer_->setOpaque(true);
431      GetClient()->setWebLayer(video_weblayer_.get());
432    }
433  }
434
435  // Do not update |current_frame_| when paused.
436  if (paused_)
437    return;
438
439  bool size_changed = !current_frame_.get() ||
440                      current_frame_->natural_size() != frame->natural_size();
441
442  {
443    base::AutoLock auto_lock(current_frame_lock_);
444    if (!current_frame_used_ && current_frame_.get())
445      ++dropped_frame_count_;
446    current_frame_ = frame;
447    current_time_ = frame->timestamp();
448    current_frame_used_ = false;
449  }
450
451  if (size_changed)
452    GetClient()->sizeChanged();
453
454  GetClient()->repaint();
455}
456
457void WebMediaPlayerMS::RepaintInternal() {
458  DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
459  DCHECK(thread_checker_.CalledOnValidThread());
460  GetClient()->repaint();
461}
462
463void WebMediaPlayerMS::OnSourceError() {
464  DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
465  DCHECK(thread_checker_.CalledOnValidThread());
466  SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
467  RepaintInternal();
468}
469
470void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
471  DCHECK(thread_checker_.CalledOnValidThread());
472  network_state_ = state;
473  // Always notify to ensure client has the latest value.
474  GetClient()->networkStateChanged();
475}
476
477void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
478  DCHECK(thread_checker_.CalledOnValidThread());
479  ready_state_ = state;
480  // Always notify to ensure client has the latest value.
481  GetClient()->readyStateChanged();
482}
483
484blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
485  DCHECK(thread_checker_.CalledOnValidThread());
486  DCHECK(client_);
487  return client_;
488}
489
490}  // namespace content
491