1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/media/capture/desktop_capture_device_aura.h"
6
7#include "base/logging.h"
8#include "base/metrics/histogram.h"
9#include "base/timer/timer.h"
10#include "cc/output/copy_output_request.h"
11#include "cc/output/copy_output_result.h"
12#include "content/browser/compositor/image_transport_factory.h"
13#include "content/browser/media/capture/content_video_capture_device_core.h"
14#include "content/browser/media/capture/desktop_capture_device_uma_types.h"
15#include "content/common/gpu/client/gl_helper.h"
16#include "content/public/browser/browser_thread.h"
17#include "content/public/browser/power_save_blocker.h"
18#include "media/base/video_util.h"
19#include "media/video/capture/video_capture_types.h"
20#include "skia/ext/image_operations.h"
21#include "third_party/skia/include/core/SkBitmap.h"
22#include "ui/aura/client/screen_position_client.h"
23#include "ui/aura/env.h"
24#include "ui/aura/window.h"
25#include "ui/aura/window_observer.h"
26#include "ui/aura/window_tree_host.h"
27#include "ui/base/cursor/cursors_aura.h"
28#include "ui/compositor/compositor.h"
29#include "ui/compositor/dip_util.h"
30#include "ui/compositor/layer.h"
31#include "ui/gfx/screen.h"
32
33namespace content {
34
35namespace {
36
37int clip_byte(int x) {
38  return std::max(0, std::min(x, 255));
39}
40
41int alpha_blend(int alpha, int src, int dst) {
42  return (src * alpha + dst * (255 - alpha)) / 255;
43}
44
45// Helper function to composite a cursor bitmap on a YUV420 video frame.
46void RenderCursorOnVideoFrame(
47    const scoped_refptr<media::VideoFrame>& target,
48    const SkBitmap& cursor_bitmap,
49    const gfx::Point& cursor_position) {
50  DCHECK(target.get());
51  DCHECK(!cursor_bitmap.isNull());
52
53  gfx::Rect rect = gfx::IntersectRects(
54      gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
55          gfx::Vector2d(cursor_position.x(), cursor_position.y()),
56      target->visible_rect());
57
58  cursor_bitmap.lockPixels();
59  for (int y = rect.y(); y < rect.bottom(); ++y) {
60    int cursor_y = y - cursor_position.y();
61    uint8* yplane = target->data(media::VideoFrame::kYPlane) +
62        y * target->row_bytes(media::VideoFrame::kYPlane);
63    uint8* uplane = target->data(media::VideoFrame::kUPlane) +
64        (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
65    uint8* vplane = target->data(media::VideoFrame::kVPlane) +
66        (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
67    for (int x = rect.x(); x < rect.right(); ++x) {
68      int cursor_x = x - cursor_position.x();
69      SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
70      int alpha = SkColorGetA(color);
71      int color_r = SkColorGetR(color);
72      int color_g = SkColorGetG(color);
73      int color_b = SkColorGetB(color);
74      int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
75                                128) >> 8) + 16);
76      yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
77
78      // Only sample U and V at even coordinates.
79      if ((x % 2 == 0) && (y % 2 == 0)) {
80        int color_u = clip_byte(((color_r * -38 + color_g * -74 +
81                                  color_b * 112 + 128) >> 8) + 128);
82        int color_v = clip_byte(((color_r * 112 + color_g * -94 +
83                                  color_b * -18 + 128) >> 8) + 128);
84        uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
85        vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
86      }
87    }
88  }
89  cursor_bitmap.unlockPixels();
90}
91
92class DesktopVideoCaptureMachine
93    : public VideoCaptureMachine,
94      public aura::WindowObserver,
95      public ui::CompositorObserver,
96      public base::SupportsWeakPtr<DesktopVideoCaptureMachine> {
97 public:
98  DesktopVideoCaptureMachine(const DesktopMediaID& source);
99  virtual ~DesktopVideoCaptureMachine();
100
101  // VideoCaptureFrameSource overrides.
102  virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
103                     const media::VideoCaptureParams& params) OVERRIDE;
104  virtual void Stop(const base::Closure& callback) OVERRIDE;
105
106  // Implements aura::WindowObserver.
107  virtual void OnWindowBoundsChanged(aura::Window* window,
108                                     const gfx::Rect& old_bounds,
109                                     const gfx::Rect& new_bounds) OVERRIDE;
110  virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE;
111  virtual void OnWindowAddedToRootWindow(aura::Window* window) OVERRIDE;
112  virtual void OnWindowRemovingFromRootWindow(aura::Window* window,
113                                              aura::Window* new_root) OVERRIDE;
114
115  // Implements ui::CompositorObserver.
116  virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
117  virtual void OnCompositingStarted(ui::Compositor* compositor,
118                                    base::TimeTicks start_time) OVERRIDE {}
119  virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
120  virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
121  virtual void OnCompositingLockStateChanged(
122      ui::Compositor* compositor) OVERRIDE {}
123
124 private:
125  // Captures a frame.
126  // |dirty| is false for timer polls and true for compositor updates.
127  void Capture(bool dirty);
128
129  // Update capture size. Must be called on the UI thread.
130  void UpdateCaptureSize();
131
132  // Response callback for cc::Layer::RequestCopyOfOutput().
133  void DidCopyOutput(
134      scoped_refptr<media::VideoFrame> video_frame,
135      base::TimeTicks start_time,
136      const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
137      scoped_ptr<cc::CopyOutputResult> result);
138
139  // A helper which does the real work for DidCopyOutput. Returns true if
140  // succeeded.
141  bool ProcessCopyOutputResponse(
142      scoped_refptr<media::VideoFrame> video_frame,
143      base::TimeTicks start_time,
144      const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
145      scoped_ptr<cc::CopyOutputResult> result);
146
147  // Helper function to update cursor state.
148  // |region_in_frame| defines the desktop bound in the captured frame.
149  // Returns the current cursor position in captured frame.
150  gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame);
151
152  // Clears cursor state.
153  void ClearCursorState();
154
155  // The window associated with the desktop.
156  aura::Window* desktop_window_;
157
158  // The timer that kicks off period captures.
159  base::Timer timer_;
160
161  // The id of the window being captured.
162  DesktopMediaID window_id_;
163
164  // Makes all the decisions about which frames to copy, and how.
165  scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
166
167  // The capture parameters for this capture.
168  media::VideoCaptureParams capture_params_;
169
170  // YUV readback pipeline.
171  scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_;
172
173  // Cursor state.
174  ui::Cursor last_cursor_;
175  gfx::Point cursor_hot_point_;
176  SkBitmap scaled_cursor_bitmap_;
177
178  // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
179  // screen from sleeping for the drive-by web.
180  scoped_ptr<PowerSaveBlocker> power_save_blocker_;
181
182  DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine);
183};
184
185DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
186    const DesktopMediaID& source)
187    : desktop_window_(NULL),
188      timer_(true, true),
189      window_id_(source) {}
190
191DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
192
193bool DesktopVideoCaptureMachine::Start(
194    const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
195    const media::VideoCaptureParams& params) {
196  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
197
198  desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_);
199  if (!desktop_window_)
200    return false;
201
202  // If the associated layer is already destroyed then return failure.
203  ui::Layer* layer = desktop_window_->layer();
204  if (!layer)
205    return false;
206
207  DCHECK(oracle_proxy.get());
208  oracle_proxy_ = oracle_proxy;
209  capture_params_ = params;
210
211  // Update capture size.
212  UpdateCaptureSize();
213
214  // Start observing window events.
215  desktop_window_->AddObserver(this);
216
217  // Start observing compositor updates.
218  if (desktop_window_->GetHost())
219    desktop_window_->GetHost()->compositor()->AddObserver(this);
220
221  power_save_blocker_.reset(PowerSaveBlocker::Create(
222      PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
223      "DesktopCaptureDevice is running").release());
224
225  // Starts timer.
226  timer_.Start(FROM_HERE, oracle_proxy_->min_capture_period(),
227               base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(),
228                          false));
229
230  return true;
231}
232
233void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) {
234  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
235  power_save_blocker_.reset();
236
237  // Stop observing compositor and window events.
238  if (desktop_window_) {
239    if (desktop_window_->GetHost())
240      desktop_window_->GetHost()->compositor()->RemoveObserver(this);
241    desktop_window_->RemoveObserver(this);
242    desktop_window_ = NULL;
243  }
244
245  // Stop timer.
246  timer_.Stop();
247
248  callback.Run();
249}
250
251void DesktopVideoCaptureMachine::UpdateCaptureSize() {
252  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
253  if (oracle_proxy_.get() && desktop_window_) {
254     ui::Layer* layer = desktop_window_->layer();
255     oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
256         layer, layer->bounds().size()));
257  }
258  ClearCursorState();
259}
260
261void DesktopVideoCaptureMachine::Capture(bool dirty) {
262  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
263
264  // Do not capture if the desktop window is already destroyed.
265  if (!desktop_window_)
266    return;
267
268  scoped_refptr<media::VideoFrame> frame;
269  ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
270
271  const base::TimeTicks start_time = base::TimeTicks::Now();
272  const VideoCaptureOracle::Event event =
273      dirty ? VideoCaptureOracle::kCompositorUpdate
274            : VideoCaptureOracle::kTimerPoll;
275  if (oracle_proxy_->ObserveEventAndDecideCapture(
276          event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
277    scoped_ptr<cc::CopyOutputRequest> request =
278        cc::CopyOutputRequest::CreateRequest(
279            base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput,
280                       AsWeakPtr(), frame, start_time, capture_frame_cb));
281    gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(),
282                                      desktop_window_->bounds().height());
283    request->set_area(window_rect);
284    desktop_window_->layer()->RequestCopyOfOutput(request.Pass());
285  }
286}
287
288void CopyOutputFinishedForVideo(
289    base::TimeTicks start_time,
290    const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
291    const scoped_refptr<media::VideoFrame>& target,
292    const SkBitmap& cursor_bitmap,
293    const gfx::Point& cursor_position,
294    scoped_ptr<cc::SingleReleaseCallback> release_callback,
295    bool result) {
296  if (!cursor_bitmap.isNull())
297    RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
298  release_callback->Run(0, false);
299  capture_frame_cb.Run(target, start_time, result);
300}
301
302void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
303                              uint32 sync_point) {
304  cb->Run(sync_point, false);
305}
306
307void DesktopVideoCaptureMachine::DidCopyOutput(
308    scoped_refptr<media::VideoFrame> video_frame,
309    base::TimeTicks start_time,
310    const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
311    scoped_ptr<cc::CopyOutputResult> result) {
312  static bool first_call = true;
313
314  bool succeeded = ProcessCopyOutputResponse(
315      video_frame, start_time, capture_frame_cb, result.Pass());
316
317  base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
318
319  // The two UMA_ blocks must be put in its own scope since it creates a static
320  // variable which expected constant histogram name.
321  if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
322    UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
323  } else {
324    UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
325  }
326
327  if (first_call) {
328    first_call = false;
329    if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
330      IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
331                                               : FIRST_SCREEN_CAPTURE_FAILED);
332    } else {
333      IncrementDesktopCaptureCounter(succeeded
334                                         ? FIRST_WINDOW_CAPTURE_SUCCEEDED
335                                         : FIRST_WINDOW_CAPTURE_FAILED);
336    }
337  }
338}
339
340bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse(
341    scoped_refptr<media::VideoFrame> video_frame,
342    base::TimeTicks start_time,
343    const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
344    scoped_ptr<cc::CopyOutputResult> result) {
345  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
346  if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_)
347    return false;
348
349  if (capture_params_.requested_format.pixel_format ==
350      media::PIXEL_FORMAT_TEXTURE) {
351    DCHECK(!video_frame.get());
352    cc::TextureMailbox texture_mailbox;
353    scoped_ptr<cc::SingleReleaseCallback> release_callback;
354    result->TakeTexture(&texture_mailbox, &release_callback);
355    DCHECK(texture_mailbox.IsTexture());
356    if (!texture_mailbox.IsTexture())
357      return false;
358    video_frame = media::VideoFrame::WrapNativeTexture(
359        make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(),
360                                               texture_mailbox.target(),
361                                               texture_mailbox.sync_point())),
362        base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)),
363        result->size(),
364        gfx::Rect(result->size()),
365        result->size(),
366        base::TimeDelta(),
367        media::VideoFrame::ReadPixelsCB());
368    capture_frame_cb.Run(video_frame, start_time, true);
369    return true;
370  }
371
372  // Compute the dest size we want after the letterboxing resize. Make the
373  // coordinates and sizes even because we letterbox in YUV space
374  // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
375  // line up correctly.
376  // The video frame's coded_size() and the result's size() are both physical
377  // pixels.
378  gfx::Rect region_in_frame =
379      media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()),
380                                    result->size());
381  region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
382                              region_in_frame.y() & ~1,
383                              region_in_frame.width() & ~1,
384                              region_in_frame.height() & ~1);
385  if (region_in_frame.IsEmpty())
386    return false;
387
388  ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
389  GLHelper* gl_helper = factory->GetGLHelper();
390  if (!gl_helper)
391    return false;
392
393  cc::TextureMailbox texture_mailbox;
394  scoped_ptr<cc::SingleReleaseCallback> release_callback;
395  result->TakeTexture(&texture_mailbox, &release_callback);
396  DCHECK(texture_mailbox.IsTexture());
397  if (!texture_mailbox.IsTexture())
398    return false;
399
400  gfx::Rect result_rect(result->size());
401  if (!yuv_readback_pipeline_ ||
402      yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
403      yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
404      yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
405    yuv_readback_pipeline_.reset(
406        gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
407                                             result_rect.size(),
408                                             result_rect,
409                                             video_frame->coded_size(),
410                                             region_in_frame,
411                                             true,
412                                             true));
413  }
414
415  gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
416  yuv_readback_pipeline_->ReadbackYUV(
417      texture_mailbox.mailbox(),
418      texture_mailbox.sync_point(),
419      video_frame.get(),
420      base::Bind(&CopyOutputFinishedForVideo,
421                 start_time,
422                 capture_frame_cb,
423                 video_frame,
424                 scaled_cursor_bitmap_,
425                 cursor_position_in_frame,
426                 base::Passed(&release_callback)));
427  return true;
428}
429
430gfx::Point DesktopVideoCaptureMachine::UpdateCursorState(
431    const gfx::Rect& region_in_frame) {
432  const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds();
433  gfx::NativeCursor cursor =
434      desktop_window_->GetHost()->last_cursor();
435  if (last_cursor_ != cursor) {
436    SkBitmap cursor_bitmap;
437    if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
438      scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
439          cursor_bitmap,
440          skia::ImageOperations::RESIZE_BEST,
441          cursor_bitmap.width() * region_in_frame.width() /
442              desktop_bounds.width(),
443          cursor_bitmap.height() * region_in_frame.height() /
444              desktop_bounds.height());
445      last_cursor_ = cursor;
446    } else {
447      // Clear cursor state if ui::GetCursorBitmap failed so that we do not
448      // render cursor on the captured frame.
449      ClearCursorState();
450    }
451  }
452
453  gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
454  aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())->
455      ConvertPointFromScreen(desktop_window_, &cursor_position);
456  const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
457      desktop_window_->layer(), cursor_hot_point_);
458  cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
459                         -desktop_bounds.y() - hot_point_in_dip.y());
460  return gfx::Point(
461      region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
462          desktop_bounds.width(),
463      region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
464          desktop_bounds.height());
465}
466
467void DesktopVideoCaptureMachine::ClearCursorState() {
468  last_cursor_ = ui::Cursor();
469  cursor_hot_point_ = gfx::Point();
470  scaled_cursor_bitmap_.reset();
471}
472
473void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
474    aura::Window* window,
475    const gfx::Rect& old_bounds,
476    const gfx::Rect& new_bounds) {
477  DCHECK(desktop_window_ && window == desktop_window_);
478
479  // Post task to update capture size on UI thread.
480  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
481      &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
482}
483
484void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) {
485  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
486
487  Stop(base::Bind(&base::DoNothing));
488
489  oracle_proxy_->ReportError("OnWindowDestroyed()");
490}
491
492void DesktopVideoCaptureMachine::OnWindowAddedToRootWindow(
493    aura::Window* window) {
494  DCHECK(window == desktop_window_);
495  window->GetHost()->compositor()->AddObserver(this);
496}
497
498void DesktopVideoCaptureMachine::OnWindowRemovingFromRootWindow(
499    aura::Window* window,
500    aura::Window* new_root) {
501  DCHECK(window == desktop_window_);
502  window->GetHost()->compositor()->RemoveObserver(this);
503}
504
505void DesktopVideoCaptureMachine::OnCompositingEnded(
506    ui::Compositor* compositor) {
507  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
508      &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true));
509}
510
511}  // namespace
512
513DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
514    const DesktopMediaID& source)
515    : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
516        new DesktopVideoCaptureMachine(source)))) {}
517
518DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
519  DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
520}
521
522// static
523media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
524    const DesktopMediaID& source) {
525  IncrementDesktopCaptureCounter(source.type == DesktopMediaID::TYPE_SCREEN
526                                     ? SCREEN_CAPTURER_CREATED
527                                     : WINDOW_CAPTURER_CREATED);
528  return new DesktopCaptureDeviceAura(source);
529}
530
531void DesktopCaptureDeviceAura::AllocateAndStart(
532    const media::VideoCaptureParams& params,
533    scoped_ptr<Client> client) {
534  DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
535  core_->AllocateAndStart(params, client.Pass());
536}
537
538void DesktopCaptureDeviceAura::StopAndDeAllocate() {
539  core_->StopAndDeAllocate();
540}
541
542}  // namespace content
543