1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/browser/media/capture/desktop_capture_device_aura.h" 6 7#include "base/logging.h" 8#include "base/metrics/histogram.h" 9#include "base/timer/timer.h" 10#include "cc/output/copy_output_request.h" 11#include "cc/output/copy_output_result.h" 12#include "content/browser/compositor/image_transport_factory.h" 13#include "content/browser/media/capture/content_video_capture_device_core.h" 14#include "content/browser/media/capture/desktop_capture_device_uma_types.h" 15#include "content/common/gpu/client/gl_helper.h" 16#include "content/public/browser/browser_thread.h" 17#include "content/public/browser/power_save_blocker.h" 18#include "media/base/video_util.h" 19#include "media/video/capture/video_capture_types.h" 20#include "skia/ext/image_operations.h" 21#include "third_party/skia/include/core/SkBitmap.h" 22#include "ui/aura/client/screen_position_client.h" 23#include "ui/aura/env.h" 24#include "ui/aura/window.h" 25#include "ui/aura/window_observer.h" 26#include "ui/aura/window_tree_host.h" 27#include "ui/base/cursor/cursors_aura.h" 28#include "ui/compositor/compositor.h" 29#include "ui/compositor/dip_util.h" 30#include "ui/compositor/layer.h" 31#include "ui/gfx/screen.h" 32 33namespace content { 34 35namespace { 36 37int clip_byte(int x) { 38 return std::max(0, std::min(x, 255)); 39} 40 41int alpha_blend(int alpha, int src, int dst) { 42 return (src * alpha + dst * (255 - alpha)) / 255; 43} 44 45// Helper function to composite a cursor bitmap on a YUV420 video frame. 46void RenderCursorOnVideoFrame( 47 const scoped_refptr<media::VideoFrame>& target, 48 const SkBitmap& cursor_bitmap, 49 const gfx::Point& cursor_position) { 50 DCHECK(target.get()); 51 DCHECK(!cursor_bitmap.isNull()); 52 53 gfx::Rect rect = gfx::IntersectRects( 54 gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) + 55 gfx::Vector2d(cursor_position.x(), cursor_position.y()), 56 target->visible_rect()); 57 58 cursor_bitmap.lockPixels(); 59 for (int y = rect.y(); y < rect.bottom(); ++y) { 60 int cursor_y = y - cursor_position.y(); 61 uint8* yplane = target->data(media::VideoFrame::kYPlane) + 62 y * target->row_bytes(media::VideoFrame::kYPlane); 63 uint8* uplane = target->data(media::VideoFrame::kUPlane) + 64 (y / 2) * target->row_bytes(media::VideoFrame::kUPlane); 65 uint8* vplane = target->data(media::VideoFrame::kVPlane) + 66 (y / 2) * target->row_bytes(media::VideoFrame::kVPlane); 67 for (int x = rect.x(); x < rect.right(); ++x) { 68 int cursor_x = x - cursor_position.x(); 69 SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y); 70 int alpha = SkColorGetA(color); 71 int color_r = SkColorGetR(color); 72 int color_g = SkColorGetG(color); 73 int color_b = SkColorGetB(color); 74 int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 + 75 128) >> 8) + 16); 76 yplane[x] = alpha_blend(alpha, color_y, yplane[x]); 77 78 // Only sample U and V at even coordinates. 79 if ((x % 2 == 0) && (y % 2 == 0)) { 80 int color_u = clip_byte(((color_r * -38 + color_g * -74 + 81 color_b * 112 + 128) >> 8) + 128); 82 int color_v = clip_byte(((color_r * 112 + color_g * -94 + 83 color_b * -18 + 128) >> 8) + 128); 84 uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]); 85 vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]); 86 } 87 } 88 } 89 cursor_bitmap.unlockPixels(); 90} 91 92class DesktopVideoCaptureMachine 93 : public VideoCaptureMachine, 94 public aura::WindowObserver, 95 public ui::CompositorObserver, 96 public base::SupportsWeakPtr<DesktopVideoCaptureMachine> { 97 public: 98 DesktopVideoCaptureMachine(const DesktopMediaID& source); 99 virtual ~DesktopVideoCaptureMachine(); 100 101 // VideoCaptureFrameSource overrides. 102 virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 103 const media::VideoCaptureParams& params) OVERRIDE; 104 virtual void Stop(const base::Closure& callback) OVERRIDE; 105 106 // Implements aura::WindowObserver. 107 virtual void OnWindowBoundsChanged(aura::Window* window, 108 const gfx::Rect& old_bounds, 109 const gfx::Rect& new_bounds) OVERRIDE; 110 virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE; 111 virtual void OnWindowAddedToRootWindow(aura::Window* window) OVERRIDE; 112 virtual void OnWindowRemovingFromRootWindow(aura::Window* window, 113 aura::Window* new_root) OVERRIDE; 114 115 // Implements ui::CompositorObserver. 116 virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {} 117 virtual void OnCompositingStarted(ui::Compositor* compositor, 118 base::TimeTicks start_time) OVERRIDE {} 119 virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE; 120 virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {} 121 virtual void OnCompositingLockStateChanged( 122 ui::Compositor* compositor) OVERRIDE {} 123 124 private: 125 // Captures a frame. 126 // |dirty| is false for timer polls and true for compositor updates. 127 void Capture(bool dirty); 128 129 // Update capture size. Must be called on the UI thread. 130 void UpdateCaptureSize(); 131 132 // Response callback for cc::Layer::RequestCopyOfOutput(). 133 void DidCopyOutput( 134 scoped_refptr<media::VideoFrame> video_frame, 135 base::TimeTicks start_time, 136 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, 137 scoped_ptr<cc::CopyOutputResult> result); 138 139 // A helper which does the real work for DidCopyOutput. Returns true if 140 // succeeded. 141 bool ProcessCopyOutputResponse( 142 scoped_refptr<media::VideoFrame> video_frame, 143 base::TimeTicks start_time, 144 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, 145 scoped_ptr<cc::CopyOutputResult> result); 146 147 // Helper function to update cursor state. 148 // |region_in_frame| defines the desktop bound in the captured frame. 149 // Returns the current cursor position in captured frame. 150 gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame); 151 152 // Clears cursor state. 153 void ClearCursorState(); 154 155 // The window associated with the desktop. 156 aura::Window* desktop_window_; 157 158 // The timer that kicks off period captures. 159 base::Timer timer_; 160 161 // The id of the window being captured. 162 DesktopMediaID window_id_; 163 164 // Makes all the decisions about which frames to copy, and how. 165 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; 166 167 // The capture parameters for this capture. 168 media::VideoCaptureParams capture_params_; 169 170 // YUV readback pipeline. 171 scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_; 172 173 // Cursor state. 174 ui::Cursor last_cursor_; 175 gfx::Point cursor_hot_point_; 176 SkBitmap scaled_cursor_bitmap_; 177 178 // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the 179 // screen from sleeping for the drive-by web. 180 scoped_ptr<PowerSaveBlocker> power_save_blocker_; 181 182 DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine); 183}; 184 185DesktopVideoCaptureMachine::DesktopVideoCaptureMachine( 186 const DesktopMediaID& source) 187 : desktop_window_(NULL), 188 timer_(true, true), 189 window_id_(source) {} 190 191DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {} 192 193bool DesktopVideoCaptureMachine::Start( 194 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 195 const media::VideoCaptureParams& params) { 196 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 197 198 desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_); 199 if (!desktop_window_) 200 return false; 201 202 // If the associated layer is already destroyed then return failure. 203 ui::Layer* layer = desktop_window_->layer(); 204 if (!layer) 205 return false; 206 207 DCHECK(oracle_proxy.get()); 208 oracle_proxy_ = oracle_proxy; 209 capture_params_ = params; 210 211 // Update capture size. 212 UpdateCaptureSize(); 213 214 // Start observing window events. 215 desktop_window_->AddObserver(this); 216 217 // Start observing compositor updates. 218 if (desktop_window_->GetHost()) 219 desktop_window_->GetHost()->compositor()->AddObserver(this); 220 221 power_save_blocker_.reset(PowerSaveBlocker::Create( 222 PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep, 223 "DesktopCaptureDevice is running").release()); 224 225 // Starts timer. 226 timer_.Start(FROM_HERE, oracle_proxy_->min_capture_period(), 227 base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(), 228 false)); 229 230 return true; 231} 232 233void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) { 234 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 235 power_save_blocker_.reset(); 236 237 // Stop observing compositor and window events. 238 if (desktop_window_) { 239 if (desktop_window_->GetHost()) 240 desktop_window_->GetHost()->compositor()->RemoveObserver(this); 241 desktop_window_->RemoveObserver(this); 242 desktop_window_ = NULL; 243 } 244 245 // Stop timer. 246 timer_.Stop(); 247 248 callback.Run(); 249} 250 251void DesktopVideoCaptureMachine::UpdateCaptureSize() { 252 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 253 if (oracle_proxy_.get() && desktop_window_) { 254 ui::Layer* layer = desktop_window_->layer(); 255 oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel( 256 layer, layer->bounds().size())); 257 } 258 ClearCursorState(); 259} 260 261void DesktopVideoCaptureMachine::Capture(bool dirty) { 262 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 263 264 // Do not capture if the desktop window is already destroyed. 265 if (!desktop_window_) 266 return; 267 268 scoped_refptr<media::VideoFrame> frame; 269 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; 270 271 const base::TimeTicks start_time = base::TimeTicks::Now(); 272 const VideoCaptureOracle::Event event = 273 dirty ? VideoCaptureOracle::kCompositorUpdate 274 : VideoCaptureOracle::kTimerPoll; 275 if (oracle_proxy_->ObserveEventAndDecideCapture( 276 event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) { 277 scoped_ptr<cc::CopyOutputRequest> request = 278 cc::CopyOutputRequest::CreateRequest( 279 base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput, 280 AsWeakPtr(), frame, start_time, capture_frame_cb)); 281 gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(), 282 desktop_window_->bounds().height()); 283 request->set_area(window_rect); 284 desktop_window_->layer()->RequestCopyOfOutput(request.Pass()); 285 } 286} 287 288void CopyOutputFinishedForVideo( 289 base::TimeTicks start_time, 290 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, 291 const scoped_refptr<media::VideoFrame>& target, 292 const SkBitmap& cursor_bitmap, 293 const gfx::Point& cursor_position, 294 scoped_ptr<cc::SingleReleaseCallback> release_callback, 295 bool result) { 296 if (!cursor_bitmap.isNull()) 297 RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position); 298 release_callback->Run(0, false); 299 capture_frame_cb.Run(target, start_time, result); 300} 301 302void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb, 303 uint32 sync_point) { 304 cb->Run(sync_point, false); 305} 306 307void DesktopVideoCaptureMachine::DidCopyOutput( 308 scoped_refptr<media::VideoFrame> video_frame, 309 base::TimeTicks start_time, 310 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, 311 scoped_ptr<cc::CopyOutputResult> result) { 312 static bool first_call = true; 313 314 bool succeeded = ProcessCopyOutputResponse( 315 video_frame, start_time, capture_frame_cb, result.Pass()); 316 317 base::TimeDelta capture_time = base::TimeTicks::Now() - start_time; 318 319 // The two UMA_ blocks must be put in its own scope since it creates a static 320 // variable which expected constant histogram name. 321 if (window_id_.type == DesktopMediaID::TYPE_SCREEN) { 322 UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time); 323 } else { 324 UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time); 325 } 326 327 if (first_call) { 328 first_call = false; 329 if (window_id_.type == DesktopMediaID::TYPE_SCREEN) { 330 IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED 331 : FIRST_SCREEN_CAPTURE_FAILED); 332 } else { 333 IncrementDesktopCaptureCounter(succeeded 334 ? FIRST_WINDOW_CAPTURE_SUCCEEDED 335 : FIRST_WINDOW_CAPTURE_FAILED); 336 } 337 } 338} 339 340bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse( 341 scoped_refptr<media::VideoFrame> video_frame, 342 base::TimeTicks start_time, 343 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, 344 scoped_ptr<cc::CopyOutputResult> result) { 345 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 346 if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_) 347 return false; 348 349 if (capture_params_.requested_format.pixel_format == 350 media::PIXEL_FORMAT_TEXTURE) { 351 DCHECK(!video_frame.get()); 352 cc::TextureMailbox texture_mailbox; 353 scoped_ptr<cc::SingleReleaseCallback> release_callback; 354 result->TakeTexture(&texture_mailbox, &release_callback); 355 DCHECK(texture_mailbox.IsTexture()); 356 if (!texture_mailbox.IsTexture()) 357 return false; 358 video_frame = media::VideoFrame::WrapNativeTexture( 359 make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(), 360 texture_mailbox.target(), 361 texture_mailbox.sync_point())), 362 base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)), 363 result->size(), 364 gfx::Rect(result->size()), 365 result->size(), 366 base::TimeDelta(), 367 media::VideoFrame::ReadPixelsCB()); 368 capture_frame_cb.Run(video_frame, start_time, true); 369 return true; 370 } 371 372 // Compute the dest size we want after the letterboxing resize. Make the 373 // coordinates and sizes even because we letterbox in YUV space 374 // (see CopyRGBToVideoFrame). They need to be even for the UV samples to 375 // line up correctly. 376 // The video frame's coded_size() and the result's size() are both physical 377 // pixels. 378 gfx::Rect region_in_frame = 379 media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()), 380 result->size()); 381 region_in_frame = gfx::Rect(region_in_frame.x() & ~1, 382 region_in_frame.y() & ~1, 383 region_in_frame.width() & ~1, 384 region_in_frame.height() & ~1); 385 if (region_in_frame.IsEmpty()) 386 return false; 387 388 ImageTransportFactory* factory = ImageTransportFactory::GetInstance(); 389 GLHelper* gl_helper = factory->GetGLHelper(); 390 if (!gl_helper) 391 return false; 392 393 cc::TextureMailbox texture_mailbox; 394 scoped_ptr<cc::SingleReleaseCallback> release_callback; 395 result->TakeTexture(&texture_mailbox, &release_callback); 396 DCHECK(texture_mailbox.IsTexture()); 397 if (!texture_mailbox.IsTexture()) 398 return false; 399 400 gfx::Rect result_rect(result->size()); 401 if (!yuv_readback_pipeline_ || 402 yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() || 403 yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect || 404 yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) { 405 yuv_readback_pipeline_.reset( 406 gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST, 407 result_rect.size(), 408 result_rect, 409 video_frame->coded_size(), 410 region_in_frame, 411 true, 412 true)); 413 } 414 415 gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame); 416 yuv_readback_pipeline_->ReadbackYUV( 417 texture_mailbox.mailbox(), 418 texture_mailbox.sync_point(), 419 video_frame.get(), 420 base::Bind(&CopyOutputFinishedForVideo, 421 start_time, 422 capture_frame_cb, 423 video_frame, 424 scaled_cursor_bitmap_, 425 cursor_position_in_frame, 426 base::Passed(&release_callback))); 427 return true; 428} 429 430gfx::Point DesktopVideoCaptureMachine::UpdateCursorState( 431 const gfx::Rect& region_in_frame) { 432 const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds(); 433 gfx::NativeCursor cursor = 434 desktop_window_->GetHost()->last_cursor(); 435 if (last_cursor_ != cursor) { 436 SkBitmap cursor_bitmap; 437 if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) { 438 scaled_cursor_bitmap_ = skia::ImageOperations::Resize( 439 cursor_bitmap, 440 skia::ImageOperations::RESIZE_BEST, 441 cursor_bitmap.width() * region_in_frame.width() / 442 desktop_bounds.width(), 443 cursor_bitmap.height() * region_in_frame.height() / 444 desktop_bounds.height()); 445 last_cursor_ = cursor; 446 } else { 447 // Clear cursor state if ui::GetCursorBitmap failed so that we do not 448 // render cursor on the captured frame. 449 ClearCursorState(); 450 } 451 } 452 453 gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location(); 454 aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())-> 455 ConvertPointFromScreen(desktop_window_, &cursor_position); 456 const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP( 457 desktop_window_->layer(), cursor_hot_point_); 458 cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(), 459 -desktop_bounds.y() - hot_point_in_dip.y()); 460 return gfx::Point( 461 region_in_frame.x() + cursor_position.x() * region_in_frame.width() / 462 desktop_bounds.width(), 463 region_in_frame.y() + cursor_position.y() * region_in_frame.height() / 464 desktop_bounds.height()); 465} 466 467void DesktopVideoCaptureMachine::ClearCursorState() { 468 last_cursor_ = ui::Cursor(); 469 cursor_hot_point_ = gfx::Point(); 470 scaled_cursor_bitmap_.reset(); 471} 472 473void DesktopVideoCaptureMachine::OnWindowBoundsChanged( 474 aura::Window* window, 475 const gfx::Rect& old_bounds, 476 const gfx::Rect& new_bounds) { 477 DCHECK(desktop_window_ && window == desktop_window_); 478 479 // Post task to update capture size on UI thread. 480 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind( 481 &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr())); 482} 483 484void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) { 485 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 486 487 Stop(base::Bind(&base::DoNothing)); 488 489 oracle_proxy_->ReportError("OnWindowDestroyed()"); 490} 491 492void DesktopVideoCaptureMachine::OnWindowAddedToRootWindow( 493 aura::Window* window) { 494 DCHECK(window == desktop_window_); 495 window->GetHost()->compositor()->AddObserver(this); 496} 497 498void DesktopVideoCaptureMachine::OnWindowRemovingFromRootWindow( 499 aura::Window* window, 500 aura::Window* new_root) { 501 DCHECK(window == desktop_window_); 502 window->GetHost()->compositor()->RemoveObserver(this); 503} 504 505void DesktopVideoCaptureMachine::OnCompositingEnded( 506 ui::Compositor* compositor) { 507 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind( 508 &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true)); 509} 510 511} // namespace 512 513DesktopCaptureDeviceAura::DesktopCaptureDeviceAura( 514 const DesktopMediaID& source) 515 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>( 516 new DesktopVideoCaptureMachine(source)))) {} 517 518DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() { 519 DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying."; 520} 521 522// static 523media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create( 524 const DesktopMediaID& source) { 525 IncrementDesktopCaptureCounter(source.type == DesktopMediaID::TYPE_SCREEN 526 ? SCREEN_CAPTURER_CREATED 527 : WINDOW_CAPTURER_CREATED); 528 return new DesktopCaptureDeviceAura(source); 529} 530 531void DesktopCaptureDeviceAura::AllocateAndStart( 532 const media::VideoCaptureParams& params, 533 scoped_ptr<Client> client) { 534 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString(); 535 core_->AllocateAndStart(params, client.Pass()); 536} 537 538void DesktopCaptureDeviceAura::StopAndDeAllocate() { 539 core_->StopAndDeAllocate(); 540} 541 542} // namespace content 543