1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "remoting/client/plugin/pepper_view.h"
6
7#include <functional>
8
9#include "base/message_loop/message_loop.h"
10#include "base/strings/string_util.h"
11#include "base/synchronization/waitable_event.h"
12#include "base/time/time.h"
13#include "ppapi/cpp/completion_callback.h"
14#include "ppapi/cpp/image_data.h"
15#include "ppapi/cpp/point.h"
16#include "ppapi/cpp/rect.h"
17#include "ppapi/cpp/size.h"
18#include "remoting/base/util.h"
19#include "remoting/client/chromoting_stats.h"
20#include "remoting/client/client_context.h"
21#include "remoting/client/frame_producer.h"
22#include "remoting/client/plugin/chromoting_instance.h"
23#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
24
25using base::Passed;
26
27namespace {
28
29// DesktopFrame that wraps a supplied pp::ImageData
30class PepperDesktopFrame : public webrtc::DesktopFrame {
31 public:
32  // Wraps the supplied ImageData.
33  explicit PepperDesktopFrame(const pp::ImageData& buffer);
34
35  // Access to underlying pepper representation.
36  const pp::ImageData& buffer() const {
37    return buffer_;
38  }
39
40 private:
41  pp::ImageData buffer_;
42};
43
44PepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer)
45  : DesktopFrame(webrtc::DesktopSize(buffer.size().width(),
46                                     buffer.size().height()),
47                 buffer.stride(),
48                 reinterpret_cast<uint8_t*>(buffer.data()),
49                 NULL),
50    buffer_(buffer) {}
51
52}  // namespace
53
54namespace remoting {
55
56namespace {
57
58// The maximum number of image buffers to be allocated at any point of time.
59const size_t kMaxPendingBuffersCount = 2;
60
61}  // namespace
62
63PepperView::PepperView(ChromotingInstance* instance,
64                       ClientContext* context)
65  : instance_(instance),
66    context_(context),
67    producer_(NULL),
68    merge_buffer_(NULL),
69    dips_to_device_scale_(1.0f),
70    dips_to_view_scale_(1.0f),
71    flush_pending_(false),
72    is_initialized_(false),
73    frame_received_(false),
74    callback_factory_(this) {
75}
76
77PepperView::~PepperView() {
78  // The producer should now return any pending buffers. At this point, however,
79  // ReturnBuffer() tasks scheduled by the producer will not be delivered,
80  // so we free all the buffers once the producer's queue is empty.
81  base::WaitableEvent done_event(true, false);
82  producer_->RequestReturnBuffers(
83      base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event)));
84  done_event.Wait();
85
86  merge_buffer_ = NULL;
87  while (!buffers_.empty()) {
88    FreeBuffer(buffers_.front());
89  }
90}
91
92void PepperView::Initialize(FrameProducer* producer) {
93  producer_ = producer;
94  webrtc::DesktopFrame* buffer = AllocateBuffer();
95  while (buffer) {
96    producer_->DrawBuffer(buffer);
97    buffer = AllocateBuffer();
98  }
99}
100
101void PepperView::SetView(const pp::View& view) {
102  bool view_changed = false;
103
104  pp::Rect pp_size = view.GetRect();
105  webrtc::DesktopSize new_dips_size(pp_size.width(), pp_size.height());
106  float new_dips_to_device_scale = view.GetDeviceScale();
107
108  if (!dips_size_.equals(new_dips_size) ||
109      dips_to_device_scale_ != new_dips_to_device_scale) {
110    view_changed = true;
111    dips_to_device_scale_ = new_dips_to_device_scale;
112    dips_size_ = new_dips_size;
113
114    // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and
115    // there are actually |view_device_scale_| physical pixels for every one
116    // Density Independent Pixel (DIP).  If we specify a scale of 1.0 to
117    // Graphics2D then we can render at DIP resolution and let PPAPI up-scale
118    // for high-DPI devices.
119    dips_to_view_scale_ = 1.0f;
120    view_size_ = dips_size_;
121
122    // If the view's DIP dimensions don't match the source then let the frame
123    // producer do the scaling, and render at device resolution.
124    if (!dips_size_.equals(source_size_)) {
125      dips_to_view_scale_ = dips_to_device_scale_;
126      view_size_.set(ceilf(dips_size_.width() * dips_to_view_scale_),
127                     ceilf(dips_size_.height() * dips_to_view_scale_));
128    }
129
130    // Create a 2D rendering context at the chosen frame dimensions.
131    pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height());
132    graphics2d_ = pp::Graphics2D(instance_, pp_size, false);
133
134    // Specify the scale from our coordinates to DIPs.
135    graphics2d_.SetScale(1.0f / dips_to_view_scale_);
136
137    bool result = instance_->BindGraphics(graphics2d_);
138
139    // There is no good way to handle this error currently.
140    DCHECK(result) << "Couldn't bind the device context.";
141  }
142
143  // Ignore clip rectangle provided by the browser because it may not be
144  // correct. See crbug.com/360240 . In case when the plugin is not visible
145  // (e.g. another tab is selected) |clip_area_| is set to empty rectangle,
146  // otherwise it's set to a rectangle that covers the whole plugin.
147  //
148  // TODO(sergeyu): Use view.GetClipRect() here after bug 360240 is fixed.
149  webrtc::DesktopRect new_clip =
150      view.IsVisible() ? webrtc::DesktopRect::MakeWH(
151                             ceilf(pp_size.width() * dips_to_view_scale_),
152                             ceilf(pp_size.height() * dips_to_view_scale_))
153                       : webrtc::DesktopRect();
154  if (!clip_area_.equals(new_clip)) {
155    view_changed = true;
156
157    // YUV to RGB conversion may require even X and Y coordinates for
158    // the top left corner of the clipping area.
159    clip_area_ = AlignRect(new_clip);
160    clip_area_.IntersectWith(webrtc::DesktopRect::MakeSize(view_size_));
161  }
162
163  if (view_changed) {
164    producer_->SetOutputSizeAndClip(view_size_, clip_area_);
165    Initialize(producer_);
166  }
167}
168
169void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size,
170                             const webrtc::DesktopRect& clip_area,
171                             webrtc::DesktopFrame* buffer,
172                             const webrtc::DesktopRegion& region,
173                             const webrtc::DesktopRegion& shape) {
174  DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
175
176  if (!frame_received_) {
177    instance_->OnFirstFrameReceived();
178    frame_received_ = true;
179  }
180  // We cannot use the data in the buffer if its dimensions don't match the
181  // current view size.
182  // TODO(alexeypa): We could rescale and draw it (or even draw it without
183  // rescaling) to reduce the perceived lag while we are waiting for
184  // the properly scaled data.
185  if (!view_size_.equals(view_size)) {
186    FreeBuffer(buffer);
187    Initialize(producer_);
188  } else {
189    FlushBuffer(clip_area, buffer, region);
190    instance_->SetDesktopShape(shape);
191  }
192}
193
194void PepperView::ReturnBuffer(webrtc::DesktopFrame* buffer) {
195  DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
196
197  // Reuse the buffer if it is large enough, otherwise drop it on the floor
198  // and allocate a new one.
199  if (buffer->size().width() >= clip_area_.width() &&
200      buffer->size().height() >= clip_area_.height()) {
201    producer_->DrawBuffer(buffer);
202  } else {
203    FreeBuffer(buffer);
204    Initialize(producer_);
205  }
206}
207
208void PepperView::SetSourceSize(const webrtc::DesktopSize& source_size,
209                               const webrtc::DesktopVector& source_dpi) {
210  DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
211
212  if (source_size_.equals(source_size) && source_dpi_.equals(source_dpi))
213    return;
214
215  source_size_ = source_size;
216  source_dpi_ = source_dpi;
217
218  // Notify JavaScript of the change in source size.
219  instance_->SetDesktopSize(source_size, source_dpi);
220}
221
222FrameConsumer::PixelFormat PepperView::GetPixelFormat() {
223  return FORMAT_BGRA;
224}
225
226webrtc::DesktopFrame* PepperView::AllocateBuffer() {
227  if (buffers_.size() >= kMaxPendingBuffersCount)
228    return NULL;
229
230  if (clip_area_.width()==0 || clip_area_.height()==0)
231    return NULL;
232
233  // Create an image buffer of the required size, but don't zero it.
234  pp::ImageData buffer_data(instance_,
235                    PP_IMAGEDATAFORMAT_BGRA_PREMUL,
236                    pp::Size(clip_area_.width(),
237                             clip_area_.height()),
238                    false);
239  if (buffer_data.is_null()) {
240    LOG(WARNING) << "Not enough memory for frame buffers.";
241    return NULL;
242  }
243
244  webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data);
245  buffers_.push_back(buffer);
246  return buffer;
247}
248
249void PepperView::FreeBuffer(webrtc::DesktopFrame* buffer) {
250  DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end());
251
252  buffers_.remove(buffer);
253  delete buffer;
254}
255
256void PepperView::FlushBuffer(const webrtc::DesktopRect& clip_area,
257                             webrtc::DesktopFrame* buffer,
258                             const webrtc::DesktopRegion& region) {
259  // Defer drawing if the flush is already in progress.
260  if (flush_pending_) {
261    // |merge_buffer_| is guaranteed to be free here because we allocate only
262    // two buffers simultaneously. If more buffers are allowed this code should
263    // apply all pending changes to the screen.
264    DCHECK(merge_buffer_ == NULL);
265
266    merge_clip_area_ = clip_area;
267    merge_buffer_ = buffer;
268    merge_region_ = region;
269    return;
270  }
271
272  // Notify Pepper API about the updated areas and flush pixels to the screen.
273  base::Time start_time = base::Time::Now();
274
275  for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) {
276    webrtc::DesktopRect rect = i.rect();
277
278    // Re-clip |region| with the current clipping area |clip_area_| because
279    // the latter could change from the time the buffer was drawn.
280    rect.IntersectWith(clip_area_);
281    if (rect.is_empty())
282      continue;
283
284    // Specify the rectangle coordinates relative to the clipping area.
285    rect.Translate(-clip_area.left(), -clip_area.top());
286
287    // Pepper Graphics 2D has a strange and badly documented API that the
288    // point here is the offset from the source rect. Why?
289    graphics2d_.PaintImageData(
290        static_cast<PepperDesktopFrame*>(buffer)->buffer(),
291        pp::Point(clip_area.left(), clip_area.top()),
292        pp::Rect(rect.left(), rect.top(), rect.width(), rect.height()));
293  }
294
295  // Notify the producer that some parts of the region weren't painted because
296  // the clipping area has changed already.
297  if (!clip_area.equals(clip_area_)) {
298    webrtc::DesktopRegion not_painted = region;
299    not_painted.Subtract(clip_area_);
300    if (!not_painted.is_empty()) {
301      producer_->InvalidateRegion(not_painted);
302    }
303  }
304
305  // Flush the updated areas to the screen.
306  pp::CompletionCallback callback =
307      callback_factory_.NewCallback(&PepperView::OnFlushDone,
308                                    start_time,
309                                    buffer);
310  int error = graphics2d_.Flush(callback);
311  CHECK(error == PP_OK_COMPLETIONPENDING);
312  flush_pending_ = true;
313}
314
315void PepperView::OnFlushDone(int result,
316                             const base::Time& paint_start,
317                             webrtc::DesktopFrame* buffer) {
318  DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
319  DCHECK(flush_pending_);
320
321  instance_->GetStats()->video_paint_ms()->Record(
322      (base::Time::Now() - paint_start).InMilliseconds());
323
324  flush_pending_ = false;
325  ReturnBuffer(buffer);
326
327  // If there is a buffer queued for rendering then render it now.
328  if (merge_buffer_ != NULL) {
329    buffer = merge_buffer_;
330    merge_buffer_ = NULL;
331    FlushBuffer(merge_clip_area_, buffer, merge_region_);
332  }
333}
334
335}  // namespace remoting
336