1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "remoting/client/software_video_renderer.h"
6
7#include <list>
8
9#include "base/bind.h"
10#include "base/callback.h"
11#include "base/callback_helpers.h"
12#include "base/location.h"
13#include "base/logging.h"
14#include "base/single_thread_task_runner.h"
15#include "remoting/base/util.h"
16#include "remoting/client/frame_consumer.h"
17#include "remoting/codec/video_decoder.h"
18#include "remoting/codec/video_decoder_verbatim.h"
19#if !defined(MEDIA_DISABLE_LIBVPX)
20#include "remoting/codec/video_decoder_vpx.h"
21#endif  // !defined(MEDIA_DISABLE_LIBVPX)
22#include "remoting/protocol/session_config.h"
23#include "third_party/libyuv/include/libyuv/convert_argb.h"
24#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
25
26using base::Passed;
27using remoting::protocol::ChannelConfig;
28using remoting::protocol::SessionConfig;
29
30namespace remoting {
31
32// This class wraps a VideoDecoder and byte-swaps the pixels for compatibility
33// with the android.graphics.Bitmap class.
34// TODO(lambroslambrou): Refactor so that the VideoDecoder produces data
35// in the right byte-order, instead of swapping it here.
36class RgbToBgrVideoDecoderFilter : public VideoDecoder {
37 public:
38  RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent)
39      : parent_(parent.Pass()) {
40  }
41
42  virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE {
43    parent_->Initialize(screen_size);
44  }
45
46  virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE {
47    return parent_->DecodePacket(packet);
48  }
49
50  virtual void Invalidate(const webrtc::DesktopSize& view_size,
51                          const webrtc::DesktopRegion& region) OVERRIDE {
52    return parent_->Invalidate(view_size, region);
53  }
54
55  virtual void RenderFrame(const webrtc::DesktopSize& view_size,
56                           const webrtc::DesktopRect& clip_area,
57                           uint8* image_buffer,
58                           int image_stride,
59                           webrtc::DesktopRegion* output_region) OVERRIDE {
60    parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride,
61                         output_region);
62
63    for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd();
64         i.Advance()) {
65      webrtc::DesktopRect rect = i.rect();
66      uint8* pixels = image_buffer + (rect.top() * image_stride) +
67        (rect.left() * kBytesPerPixel);
68      libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride,
69                         rect.width(), rect.height());
70    }
71  }
72
73  virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE {
74    return parent_->GetImageShape();
75  }
76
77 private:
78  scoped_ptr<VideoDecoder> parent_;
79};
80
81class SoftwareVideoRenderer::Core {
82 public:
83  Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
84       scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
85       scoped_refptr<FrameConsumerProxy> consumer);
86  ~Core();
87
88  void Initialize(const protocol::SessionConfig& config);
89  void DrawBuffer(webrtc::DesktopFrame* buffer);
90  void InvalidateRegion(const webrtc::DesktopRegion& region);
91  void RequestReturnBuffers(const base::Closure& done);
92  void SetOutputSizeAndClip(
93      const webrtc::DesktopSize& view_size,
94      const webrtc::DesktopRect& clip_area);
95
96  // Decodes the contents of |packet|. DecodePacket may keep a reference to
97  // |packet| so the |packet| must remain alive and valid until |done| is
98  // executed.
99  void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done);
100
101 private:
102  // Paints the invalidated region to the next available buffer and returns it
103  // to the consumer.
104  void SchedulePaint();
105  void DoPaint();
106
107  scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
108  scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_;
109  scoped_refptr<FrameConsumerProxy> consumer_;
110  scoped_ptr<VideoDecoder> decoder_;
111
112  // Remote screen size in pixels.
113  webrtc::DesktopSize source_size_;
114
115  // Vertical and horizontal DPI of the remote screen.
116  webrtc::DesktopVector source_dpi_;
117
118  // The current dimensions of the frame consumer view.
119  webrtc::DesktopSize view_size_;
120  webrtc::DesktopRect clip_area_;
121
122  // The drawing buffers supplied by the frame consumer.
123  std::list<webrtc::DesktopFrame*> buffers_;
124
125  // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint().
126  bool paint_scheduled_;
127
128  base::WeakPtrFactory<Core> weak_factory_;
129};
130
131SoftwareVideoRenderer::Core::Core(
132    scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
133    scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
134    scoped_refptr<FrameConsumerProxy> consumer)
135    : main_task_runner_(main_task_runner),
136      decode_task_runner_(decode_task_runner),
137      consumer_(consumer),
138      paint_scheduled_(false),
139      weak_factory_(this) {
140}
141
142SoftwareVideoRenderer::Core::~Core() {
143}
144
145void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) {
146  DCHECK(decode_task_runner_->BelongsToCurrentThread());
147
148  // Initialize decoder based on the selected codec.
149  ChannelConfig::Codec codec = config.video_config().codec;
150  if (codec == ChannelConfig::CODEC_VERBATIM) {
151    decoder_.reset(new VideoDecoderVerbatim());
152#if !defined(MEDIA_DISABLE_LIBVPX)
153  } else if (codec == ChannelConfig::CODEC_VP8) {
154    decoder_ = VideoDecoderVpx::CreateForVP8();
155  } else if (codec == ChannelConfig::CODEC_VP9) {
156    decoder_ = VideoDecoderVpx::CreateForVP9();
157  } else {
158#endif  // !defined(MEDIA_DISABLE_LIBVPX)
159    NOTREACHED() << "Invalid Encoding found: " << codec;
160  }
161
162  if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) {
163    scoped_ptr<VideoDecoder> wrapper(
164        new RgbToBgrVideoDecoderFilter(decoder_.Pass()));
165    decoder_ = wrapper.Pass();
166  }
167}
168
169void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet,
170                                                const base::Closure& done) {
171  DCHECK(decode_task_runner_->BelongsToCurrentThread());
172
173  bool decoder_needs_reset = false;
174  bool notify_size_or_dpi_change = false;
175
176  // If the packet includes screen size or DPI information, store them.
177  if (packet->format().has_screen_width() &&
178      packet->format().has_screen_height()) {
179    webrtc::DesktopSize source_size(packet->format().screen_width(),
180                                    packet->format().screen_height());
181    if (!source_size_.equals(source_size)) {
182      source_size_ = source_size;
183      decoder_needs_reset = true;
184      notify_size_or_dpi_change = true;
185    }
186  }
187  if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
188    webrtc::DesktopVector source_dpi(packet->format().x_dpi(),
189                                     packet->format().y_dpi());
190    if (!source_dpi.equals(source_dpi_)) {
191      source_dpi_ = source_dpi;
192      notify_size_or_dpi_change = true;
193    }
194  }
195
196  // If we've never seen a screen size, ignore the packet.
197  if (source_size_.is_empty()) {
198    main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
199    return;
200  }
201
202  if (decoder_needs_reset)
203    decoder_->Initialize(source_size_);
204  if (notify_size_or_dpi_change)
205    consumer_->SetSourceSize(source_size_, source_dpi_);
206
207  if (decoder_->DecodePacket(*packet.get())) {
208    SchedulePaint();
209  } else {
210    LOG(ERROR) << "DecodePacket() failed.";
211  }
212
213  main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
214}
215
216void SoftwareVideoRenderer::Core::SchedulePaint() {
217  DCHECK(decode_task_runner_->BelongsToCurrentThread());
218  if (paint_scheduled_)
219    return;
220  paint_scheduled_ = true;
221  decode_task_runner_->PostTask(
222      FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint,
223                            weak_factory_.GetWeakPtr()));
224}
225
226void SoftwareVideoRenderer::Core::DoPaint() {
227  DCHECK(decode_task_runner_->BelongsToCurrentThread());
228  DCHECK(paint_scheduled_);
229  paint_scheduled_ = false;
230
231  // If the view size is empty or we have no output buffers ready, return.
232  if (buffers_.empty() || view_size_.is_empty())
233    return;
234
235  // If no Decoder is initialized, or the host dimensions are empty, return.
236  if (!decoder_.get() || source_size_.is_empty())
237    return;
238
239  // Draw the invalidated region to the buffer.
240  webrtc::DesktopFrame* buffer = buffers_.front();
241  webrtc::DesktopRegion output_region;
242  decoder_->RenderFrame(view_size_, clip_area_,
243                        buffer->data(), buffer->stride(), &output_region);
244
245  // Notify the consumer that painting is done.
246  if (!output_region.is_empty()) {
247    buffers_.pop_front();
248    consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region,
249                           *decoder_->GetImageShape());
250  }
251}
252
253void SoftwareVideoRenderer::Core::RequestReturnBuffers(
254    const base::Closure& done) {
255  DCHECK(decode_task_runner_->BelongsToCurrentThread());
256
257  while (!buffers_.empty()) {
258    consumer_->ReturnBuffer(buffers_.front());
259    buffers_.pop_front();
260  }
261
262  if (!done.is_null())
263    done.Run();
264}
265
266void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) {
267  DCHECK(decode_task_runner_->BelongsToCurrentThread());
268  DCHECK(clip_area_.width() <= buffer->size().width() &&
269         clip_area_.height() <= buffer->size().height());
270
271  buffers_.push_back(buffer);
272  SchedulePaint();
273}
274
275void SoftwareVideoRenderer::Core::InvalidateRegion(
276    const webrtc::DesktopRegion& region) {
277  DCHECK(decode_task_runner_->BelongsToCurrentThread());
278
279  if (decoder_.get()) {
280    decoder_->Invalidate(view_size_, region);
281    SchedulePaint();
282  }
283}
284
285void SoftwareVideoRenderer::Core::SetOutputSizeAndClip(
286    const webrtc::DesktopSize& view_size,
287    const webrtc::DesktopRect& clip_area) {
288  DCHECK(decode_task_runner_->BelongsToCurrentThread());
289
290  // The whole frame needs to be repainted if the scaling factor has changed.
291  if (!view_size_.equals(view_size) && decoder_.get()) {
292    webrtc::DesktopRegion region;
293    region.AddRect(webrtc::DesktopRect::MakeSize(view_size));
294    decoder_->Invalidate(view_size, region);
295  }
296
297  if (!view_size_.equals(view_size) ||
298      !clip_area_.equals(clip_area)) {
299    view_size_ = view_size;
300    clip_area_ = clip_area;
301
302    // Return buffers that are smaller than needed to the consumer for
303    // reuse/reallocation.
304    std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin();
305    while (i != buffers_.end()) {
306      if ((*i)->size().width() < clip_area_.width() ||
307          (*i)->size().height() < clip_area_.height()) {
308        consumer_->ReturnBuffer(*i);
309        i = buffers_.erase(i);
310      } else {
311        ++i;
312      }
313    }
314
315    SchedulePaint();
316  }
317}
318
319SoftwareVideoRenderer::SoftwareVideoRenderer(
320    scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
321    scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
322    scoped_refptr<FrameConsumerProxy> consumer)
323    : decode_task_runner_(decode_task_runner),
324      core_(new Core(main_task_runner, decode_task_runner, consumer)),
325      latest_sequence_number_(0),
326      weak_factory_(this) {
327  DCHECK(CalledOnValidThread());
328}
329
330SoftwareVideoRenderer::~SoftwareVideoRenderer() {
331  DCHECK(CalledOnValidThread());
332  decode_task_runner_->DeleteSoon(FROM_HERE, core_.release());
333}
334
335void SoftwareVideoRenderer::Initialize(
336    const protocol::SessionConfig& config) {
337  DCHECK(CalledOnValidThread());
338  decode_task_runner_->PostTask(
339      FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize,
340                            base::Unretained(core_.get()), config));
341}
342
343ChromotingStats* SoftwareVideoRenderer::GetStats() {
344  DCHECK(CalledOnValidThread());
345  return &stats_;
346}
347
348void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
349                                                const base::Closure& done) {
350  DCHECK(CalledOnValidThread());
351
352  // If the video packet is empty then drop it. Empty packets are used to
353  // maintain activity on the network.
354  if (!packet->has_data() || packet->data().size() == 0) {
355    done.Run();
356    return;
357  }
358
359  // Add one frame to the counter.
360  stats_.video_frame_rate()->Record(1);
361
362  // Record other statistics received from host.
363  stats_.video_bandwidth()->Record(packet->data().size());
364  if (packet->has_capture_time_ms())
365    stats_.video_capture_ms()->Record(packet->capture_time_ms());
366  if (packet->has_encode_time_ms())
367    stats_.video_encode_ms()->Record(packet->encode_time_ms());
368  if (packet->has_client_sequence_number() &&
369      packet->client_sequence_number() > latest_sequence_number_) {
370    latest_sequence_number_ = packet->client_sequence_number();
371    base::TimeDelta round_trip_latency =
372        base::Time::Now() -
373        base::Time::FromInternalValue(packet->client_sequence_number());
374    stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds());
375  }
376
377  // Measure the latency between the last packet being received and presented.
378  base::Time decode_start = base::Time::Now();
379
380  base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone,
381                                         weak_factory_.GetWeakPtr(),
382                                         decode_start, done);
383
384  decode_task_runner_->PostTask(FROM_HERE, base::Bind(
385      &SoftwareVideoRenderer::Core::DecodePacket,
386      base::Unretained(core_.get()), base::Passed(&packet), decode_done));
387}
388
389void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) {
390  decode_task_runner_->PostTask(
391      FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer,
392                            base::Unretained(core_.get()), buffer));
393}
394
395void SoftwareVideoRenderer::InvalidateRegion(
396    const webrtc::DesktopRegion& region) {
397  decode_task_runner_->PostTask(
398      FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion,
399                            base::Unretained(core_.get()), region));
400}
401
402void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) {
403  decode_task_runner_->PostTask(
404      FROM_HERE,
405      base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers,
406                 base::Unretained(core_.get()), done));
407}
408
409void SoftwareVideoRenderer::SetOutputSizeAndClip(
410    const webrtc::DesktopSize& view_size,
411    const webrtc::DesktopRect& clip_area) {
412  decode_task_runner_->PostTask(
413      FROM_HERE,
414      base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip,
415                 base::Unretained(core_.get()), view_size, clip_area));
416}
417
418void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start,
419                                          const base::Closure& done) {
420  DCHECK(CalledOnValidThread());
421
422  // Record the latency between the packet being received and presented.
423  stats_.video_decode_ms()->Record(
424      (base::Time::Now() - decode_start).InMilliseconds());
425
426  done.Run();
427}
428
429}  // namespace remoting
430