pepper_view.cc revision 7dbb3d5cf0c15f500944d211057644d6a2f37371
1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Use of this source code is governed by a BSD-style license that can be 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// found in the LICENSE file. 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/plugin/pepper_view.h" 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <functional> 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "base/message_loop.h" 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "base/strings/string_util.h" 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "base/synchronization/waitable_event.h" 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "base/time/time.h" 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/completion_callback.h" 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/dev/graphics_2d_dev.h" 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/dev/view_dev.h" 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/image_data.h" 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/point.h" 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/rect.h" 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "ppapi/cpp/size.h" 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/base/util.h" 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/chromoting_stats.h" 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/client_context.h" 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/frame_producer.h" 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/plugin/chromoting_instance.h" 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "remoting/client/plugin/pepper_util.h" 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverusing base::Passed; 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace { 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// DesktopFrame that wraps a supplied pp::ImageData 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass PepperDesktopFrame : public webrtc::DesktopFrame { 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public: 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Wraps the supplied ImageData. 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver explicit PepperDesktopFrame(const pp::ImageData& buffer); 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Access to underlying pepper representation. 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver const pp::ImageData& buffer() const { 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buffer_; 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private: 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pp::ImageData buffer_; 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver}; 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 47324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverPepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer) 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : DesktopFrame(webrtc::DesktopSize(buffer.size().width(), 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buffer.size().height()), 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buffer.stride(), 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver reinterpret_cast<uint8_t*>(buffer.data()), 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver NULL), 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buffer_(buffer) {} 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} // namespace 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace remoting { 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace { 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// The maximum number of image buffers to be allocated at any point of time. 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverconst size_t kMaxPendingBuffersCount = 2; 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} // namespace 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 66324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverPepperView::PepperView(ChromotingInstance* instance, 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ClientContext* context, 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FrameProducer* producer) 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : instance_(instance), 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver context_(context), 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver producer_(producer), 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver merge_buffer_(NULL), 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver merge_clip_area_(SkIRect::MakeEmpty()), 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver dips_size_(SkISize::Make(0, 0)), 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver dips_to_device_scale_(1.0f), 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver view_size_(SkISize::Make(0, 0)), 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver dips_to_view_scale_(1.0f), 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver clip_area_(SkIRect::MakeEmpty()), 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver source_size_(SkISize::Make(0, 0)), 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver source_dpi_(SkIPoint::Make(0, 0)), 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver flush_pending_(false), 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is_initialized_(false), 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver frame_received_(false) { 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver InitiateDrawing(); 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 87324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverPepperView::~PepperView() { 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // The producer should now return any pending buffers. At this point, however, 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // ReturnBuffer() tasks scheduled by the producer will not be delivered, 90 // so we free all the buffers once the producer's queue is empty. 91 base::WaitableEvent done_event(true, false); 92 producer_->RequestReturnBuffers( 93 base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event))); 94 done_event.Wait(); 95 96 merge_buffer_ = NULL; 97 while (!buffers_.empty()) { 98 FreeBuffer(buffers_.front()); 99 } 100} 101 102void PepperView::SetView(const pp::View& view) { 103 bool view_changed = false; 104 105 pp::Rect pp_size = view.GetRect(); 106 SkISize new_dips_size = SkISize::Make(pp_size.width(), pp_size.height()); 107 pp::ViewDev view_dev(view); 108 float new_dips_to_device_scale = view_dev.GetDeviceScale(); 109 110 if (dips_size_ != new_dips_size || 111 dips_to_device_scale_ != new_dips_to_device_scale) { 112 view_changed = true; 113 dips_to_device_scale_ = new_dips_to_device_scale; 114 dips_size_ = new_dips_size; 115 116 // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and 117 // there are actually |view_device_scale_| physical pixels for every one 118 // Density Independent Pixel (DIP). If we specify a scale of 1.0 to 119 // Graphics2D then we can render at DIP resolution and let PPAPI up-scale 120 // for high-DPI devices. 121 dips_to_view_scale_ = 1.0f; 122 view_size_ = dips_size_; 123 124 // If the view's DIP dimensions don't match the source then let the frame 125 // producer do the scaling, and render at device resolution. 126 if (dips_size_ != source_size_) { 127 dips_to_view_scale_ = dips_to_device_scale_; 128 view_size_ = SkISize::Make( 129 ceilf(dips_size_.width() * dips_to_view_scale_), 130 ceilf(dips_size_.height() * dips_to_view_scale_)); 131 } 132 133 // Create a 2D rendering context at the chosen frame dimensions. 134 pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height()); 135 graphics2d_ = pp::Graphics2D(instance_, pp_size, false); 136 137 // Specify the scale from our coordinates to DIPs. 138 pp::Graphics2D_Dev graphics2d_dev(graphics2d_); 139 graphics2d_dev.SetScale(1.0f / dips_to_view_scale_); 140 141 bool result = instance_->BindGraphics(graphics2d_); 142 143 // There is no good way to handle this error currently. 144 DCHECK(result) << "Couldn't bind the device context."; 145 } 146 147 pp::Rect pp_clip = view.GetClipRect(); 148 SkIRect new_clip = SkIRect::MakeLTRB( 149 floorf(pp_clip.x() * dips_to_view_scale_), 150 floorf(pp_clip.y() * dips_to_view_scale_), 151 ceilf(pp_clip.right() * dips_to_view_scale_), 152 ceilf(pp_clip.bottom() * dips_to_view_scale_)); 153 if (clip_area_ != new_clip) { 154 view_changed = true; 155 156 // YUV to RGB conversion may require even X and Y coordinates for 157 // the top left corner of the clipping area. 158 clip_area_ = AlignRect(new_clip); 159 clip_area_.intersect(SkIRect::MakeSize(view_size_)); 160 } 161 162 if (view_changed) { 163 producer_->SetOutputSizeAndClip(view_size_, clip_area_); 164 InitiateDrawing(); 165 } 166} 167 168void PepperView::ApplyBuffer(const SkISize& view_size, 169 const SkIRect& clip_area, 170 webrtc::DesktopFrame* buffer, 171 const SkRegion& region) { 172 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 173 174 if (!frame_received_) { 175 instance_->OnFirstFrameReceived(); 176 frame_received_ = true; 177 } 178 // We cannot use the data in the buffer if its dimensions don't match the 179 // current view size. 180 // TODO(alexeypa): We could rescale and draw it (or even draw it without 181 // rescaling) to reduce the perceived lag while we are waiting for 182 // the properly scaled data. 183 if (view_size_ != view_size) { 184 FreeBuffer(buffer); 185 InitiateDrawing(); 186 } else { 187 FlushBuffer(clip_area, buffer, region); 188 } 189} 190 191void PepperView::ReturnBuffer(webrtc::DesktopFrame* buffer) { 192 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 193 194 // Reuse the buffer if it is large enough, otherwise drop it on the floor 195 // and allocate a new one. 196 if (buffer->size().width() >= clip_area_.width() && 197 buffer->size().height() >= clip_area_.height()) { 198 producer_->DrawBuffer(buffer); 199 } else { 200 FreeBuffer(buffer); 201 InitiateDrawing(); 202 } 203} 204 205void PepperView::SetSourceSize(const SkISize& source_size, 206 const SkIPoint& source_dpi) { 207 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 208 209 if (source_size_ == source_size && source_dpi_ == source_dpi) 210 return; 211 212 source_size_ = source_size; 213 source_dpi_ = source_dpi; 214 215 // Notify JavaScript of the change in source size. 216 instance_->SetDesktopSize(source_size, source_dpi); 217} 218 219webrtc::DesktopFrame* PepperView::AllocateBuffer() { 220 if (buffers_.size() >= kMaxPendingBuffersCount) 221 return NULL; 222 223 if (clip_area_.width()==0 || clip_area_.height()==0) 224 return NULL; 225 226 // Create an image buffer of the required size, but don't zero it. 227 pp::ImageData buffer_data(instance_, 228 PP_IMAGEDATAFORMAT_BGRA_PREMUL, 229 pp::Size(clip_area_.width(), 230 clip_area_.height()), 231 false); 232 if (buffer_data.is_null()) { 233 LOG(WARNING) << "Not enough memory for frame buffers."; 234 return NULL; 235 } 236 237 webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data); 238 buffers_.push_back(buffer); 239 return buffer; 240} 241 242void PepperView::FreeBuffer(webrtc::DesktopFrame* buffer) { 243 DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end()); 244 245 buffers_.remove(buffer); 246 delete buffer; 247} 248 249void PepperView::InitiateDrawing() { 250 webrtc::DesktopFrame* buffer = AllocateBuffer(); 251 while (buffer) { 252 producer_->DrawBuffer(buffer); 253 buffer = AllocateBuffer(); 254 } 255} 256 257void PepperView::FlushBuffer(const SkIRect& clip_area, 258 webrtc::DesktopFrame* buffer, 259 const SkRegion& region) { 260 // Defer drawing if the flush is already in progress. 261 if (flush_pending_) { 262 // |merge_buffer_| is guaranteed to be free here because we allocate only 263 // two buffers simultaneously. If more buffers are allowed this code should 264 // apply all pending changes to the screen. 265 DCHECK(merge_buffer_ == NULL); 266 267 merge_clip_area_ = clip_area; 268 merge_buffer_ = buffer; 269 merge_region_ = region; 270 return; 271 } 272 273 // Notify Pepper API about the updated areas and flush pixels to the screen. 274 base::Time start_time = base::Time::Now(); 275 276 for (SkRegion::Iterator i(region); !i.done(); i.next()) { 277 SkIRect rect = i.rect(); 278 279 // Re-clip |region| with the current clipping area |clip_area_| because 280 // the latter could change from the time the buffer was drawn. 281 if (!rect.intersect(clip_area_)) 282 continue; 283 284 // Specify the rectangle coordinates relative to the clipping area. 285 rect.offset(-clip_area.left(), -clip_area.top()); 286 287 // Pepper Graphics 2D has a strange and badly documented API that the 288 // point here is the offset from the source rect. Why? 289 graphics2d_.PaintImageData( 290 static_cast<PepperDesktopFrame*>(buffer)->buffer(), 291 pp::Point(clip_area.left(), clip_area.top()), 292 pp::Rect(rect.left(), rect.top(), rect.width(), rect.height())); 293 } 294 295 // Notify the producer that some parts of the region weren't painted because 296 // the clipping area has changed already. 297 if (clip_area != clip_area_) { 298 SkRegion not_painted = region; 299 not_painted.op(clip_area_, SkRegion::kDifference_Op); 300 if (!not_painted.isEmpty()) { 301 producer_->InvalidateRegion(not_painted); 302 } 303 } 304 305 // Flush the updated areas to the screen. 306 int error = graphics2d_.Flush( 307 PpCompletionCallback(base::Bind( 308 &PepperView::OnFlushDone, AsWeakPtr(), start_time, buffer))); 309 CHECK(error == PP_OK_COMPLETIONPENDING); 310 flush_pending_ = true; 311 312 // If the buffer we just rendered has a shape then pass that to JavaScript. 313 const SkRegion* buffer_shape = producer_->GetBufferShape(); 314 if (buffer_shape) 315 instance_->SetDesktopShape(*buffer_shape); 316} 317 318void PepperView::OnFlushDone(base::Time paint_start, 319 webrtc::DesktopFrame* buffer, 320 int result) { 321 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 322 DCHECK(flush_pending_); 323 324 instance_->GetStats()->video_paint_ms()->Record( 325 (base::Time::Now() - paint_start).InMilliseconds()); 326 327 flush_pending_ = false; 328 ReturnBuffer(buffer); 329 330 // If there is a buffer queued for rendering then render it now. 331 if (merge_buffer_ != NULL) { 332 buffer = merge_buffer_; 333 merge_buffer_ = NULL; 334 FlushBuffer(merge_clip_area_, buffer, merge_region_); 335 } 336} 337 338} // namespace remoting 339