1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/cast/video_receiver/video_receiver.h"
6
7#include <algorithm>
8
9#include "base/bind.h"
10#include "base/logging.h"
11#include "base/message_loop/message_loop.h"
12#include "crypto/encryptor.h"
13#include "crypto/symmetric_key.h"
14#include "media/cast/cast_defines.h"
15#include "media/cast/framer/framer.h"
16#include "media/cast/video_receiver/video_decoder.h"
17
18namespace media {
19namespace cast {
20
21const int64 kMinSchedulingDelayMs = 1;
22
23static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000;
24static const int kTimeOffsetFilter = 8;
25static const int64_t kMinProcessIntervalMs = 5;
26
27// Local implementation of RtpData (defined in rtp_rtcp_defines.h).
28// Used to pass payload data into the video receiver.
29class LocalRtpVideoData : public RtpData {
30 public:
31  explicit LocalRtpVideoData(VideoReceiver* video_receiver)
32      : video_receiver_(video_receiver) {}
33
34  virtual ~LocalRtpVideoData() {}
35
36  virtual void OnReceivedPayloadData(const uint8* payload_data,
37                                     size_t payload_size,
38                                     const RtpCastHeader* rtp_header) OVERRIDE {
39    video_receiver_->IncomingParsedRtpPacket(payload_data, payload_size,
40                                             *rtp_header);
41  }
42
43 private:
44  VideoReceiver* video_receiver_;
45};
46
47// Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
48// Used to convey cast-specific feedback from receiver to sender.
49// Callback triggered by the Framer (cast message builder).
50class LocalRtpVideoFeedback : public RtpPayloadFeedback {
51 public:
52  explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver)
53      : video_receiver_(video_receiver) {
54  }
55
56  virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
57    video_receiver_->CastFeedback(cast_message);
58  }
59
60 private:
61  VideoReceiver* video_receiver_;
62};
63
64// Local implementation of RtpReceiverStatistics (defined by rtcp.h).
65// Used to pass statistics data from the RTP module to the RTCP module.
66class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
67 public:
68  explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
69     : rtp_receiver_(rtp_receiver) {
70  }
71
72  virtual void GetStatistics(uint8* fraction_lost,
73                             uint32* cumulative_lost,  // 24 bits valid.
74                             uint32* extended_high_sequence_number,
75                             uint32* jitter) OVERRIDE {
76    rtp_receiver_->GetStatistics(fraction_lost,
77                                 cumulative_lost,
78                                 extended_high_sequence_number,
79                                 jitter);
80  }
81
82 private:
83  RtpReceiver* rtp_receiver_;
84};
85
86VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
87                             const VideoReceiverConfig& video_config,
88                             PacedPacketSender* const packet_sender)
89      : cast_environment_(cast_environment),
90        codec_(video_config.codec),
91        target_delay_delta_(
92            base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
93        frame_delay_(base::TimeDelta::FromMilliseconds(
94            1000 / video_config.max_frame_rate)),
95        incoming_payload_callback_(new LocalRtpVideoData(this)),
96        incoming_payload_feedback_(new LocalRtpVideoFeedback(this)),
97        rtp_receiver_(cast_environment_->Clock(), NULL, &video_config,
98                      incoming_payload_callback_.get()),
99        rtp_video_receiver_statistics_(
100            new LocalRtpReceiverStatistics(&rtp_receiver_)),
101        time_incoming_packet_updated_(false),
102        incoming_rtp_timestamp_(0),
103        weak_factory_(this) {
104  int max_unacked_frames = video_config.rtp_max_delay_ms *
105      video_config.max_frame_rate / 1000;
106  DCHECK(max_unacked_frames) << "Invalid argument";
107
108  if (video_config.aes_iv_mask.size() == kAesKeySize &&
109      video_config.aes_key.size() == kAesKeySize) {
110    iv_mask_ = video_config.aes_iv_mask;
111    crypto::SymmetricKey* key = crypto::SymmetricKey::Import(
112        crypto::SymmetricKey::AES, video_config.aes_key);
113    decryptor_.reset(new crypto::Encryptor());
114    decryptor_->Init(key, crypto::Encryptor::CTR, std::string());
115  } else if (video_config.aes_iv_mask.size() != 0 ||
116             video_config.aes_key.size() != 0) {
117    DCHECK(false) << "Invalid crypto configuration";
118  }
119
120  framer_.reset(new Framer(cast_environment->Clock(),
121                           incoming_payload_feedback_.get(),
122                           video_config.incoming_ssrc,
123                           video_config.decoder_faster_than_max_frame_rate,
124                           max_unacked_frames));
125  if (!video_config.use_external_decoder) {
126    video_decoder_.reset(new VideoDecoder(video_config, cast_environment));
127  }
128
129  rtcp_.reset(
130      new Rtcp(cast_environment_,
131               NULL,
132               packet_sender,
133               NULL,
134               rtp_video_receiver_statistics_.get(),
135               video_config.rtcp_mode,
136               base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
137               video_config.feedback_ssrc,
138               video_config.incoming_ssrc,
139               video_config.rtcp_c_name));
140}
141
142VideoReceiver::~VideoReceiver() {}
143
144void VideoReceiver::InitializeTimers() {
145  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
146  ScheduleNextRtcpReport();
147  ScheduleNextCastMessage();
148}
149
150void VideoReceiver::GetRawVideoFrame(
151    const VideoFrameDecodedCallback& callback) {
152  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
153  GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame,
154                                  base::Unretained(this), callback));
155}
156
157// Called when we have a frame to decode.
158void VideoReceiver::DecodeVideoFrame(
159    const VideoFrameDecodedCallback& callback,
160    scoped_ptr<EncodedVideoFrame> encoded_frame,
161    const base::TimeTicks& render_time) {
162  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
163  // Hand the ownership of the encoded frame to the decode thread.
164  cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE,
165      base::Bind(&VideoReceiver::DecodeVideoFrameThread, base::Unretained(this),
166                 base::Passed(&encoded_frame), render_time, callback));
167}
168
169// Utility function to run the decoder on a designated decoding thread.
170void VideoReceiver::DecodeVideoFrameThread(
171    scoped_ptr<EncodedVideoFrame> encoded_frame,
172    const base::TimeTicks render_time,
173    const VideoFrameDecodedCallback& frame_decoded_callback) {
174  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER));
175  DCHECK(video_decoder_);
176
177  if (!(video_decoder_->DecodeVideoFrame(encoded_frame.get(), render_time,
178        frame_decoded_callback))) {
179    // This will happen if we decide to decode but not show a frame.
180    cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
181        base::Bind(&VideoReceiver::GetRawVideoFrame, base::Unretained(this),
182                   frame_decoded_callback));
183  }
184}
185
186bool VideoReceiver::DecryptVideoFrame(
187    scoped_ptr<EncodedVideoFrame>* video_frame) {
188  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
189  DCHECK(decryptor_) << "Invalid state";
190
191  if (!decryptor_->SetCounter(GetAesNonce((*video_frame)->frame_id,
192                                          iv_mask_))) {
193    NOTREACHED() << "Failed to set counter";
194    return false;
195  }
196  std::string decrypted_video_data;
197  if (!decryptor_->Decrypt((*video_frame)->data, &decrypted_video_data)) {
198    VLOG(1) << "Decryption error";
199    // Give up on this frame, release it from jitter buffer.
200    framer_->ReleaseFrame((*video_frame)->frame_id);
201    return false;
202  }
203  (*video_frame)->data.swap(decrypted_video_data);
204  return true;
205}
206
207// Called from the main cast thread.
208void VideoReceiver::GetEncodedVideoFrame(
209    const VideoFrameEncodedCallback& callback) {
210  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
211  scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
212  uint32 rtp_timestamp = 0;
213  bool next_frame = false;
214
215  if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
216                                     &next_frame)) {
217    // We have no video frames. Wait for new packet(s).
218    queued_encoded_callbacks_.push_back(callback);
219    return;
220  }
221
222  if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) {
223    // Logging already done.
224    queued_encoded_callbacks_.push_back(callback);
225    return;
226  }
227
228  base::TimeTicks render_time;
229  if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
230                            &render_time)) {
231    cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
232        base::Bind(callback, base::Passed(&encoded_frame), render_time));
233  } else {
234    // We have a video frame; however we are missing packets and we have time
235    // to wait for new packet(s).
236    queued_encoded_callbacks_.push_back(callback);
237  }
238}
239
240// Should we pull the encoded video frame from the framer? decided by if this is
241// the next frame or we are running out of time and have to pull the following
242// frame.
243// If the frame is too old to be rendered we set the don't show flag in the
244// video bitstream where possible.
245bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp,
246    bool next_frame, scoped_ptr<EncodedVideoFrame>* encoded_frame,
247    base::TimeTicks* render_time) {
248  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
249  base::TimeTicks now = cast_environment_->Clock()->NowTicks();
250  *render_time = GetRenderTime(now, rtp_timestamp);
251
252  // TODO(mikhal): Store actual render time and not diff.
253  cast_environment_->Logging()->InsertFrameEventWithDelay(kVideoRenderDelay,
254      rtp_timestamp, (*encoded_frame)->frame_id, now - *render_time);
255
256  // Minimum time before a frame is due to be rendered before we pull it for
257  // decode.
258  base::TimeDelta min_wait_delta = frame_delay_;
259  base::TimeDelta time_until_render = *render_time - now;
260  if (!next_frame && (time_until_render > min_wait_delta)) {
261    // Example:
262    // We have decoded frame 1 and we have received the complete frame 3, but
263    // not frame 2. If we still have time before frame 3 should be rendered we
264    // will wait for 2 to arrive, however if 2 never show up this timer will hit
265    // and we will pull out frame 3 for decoding and rendering.
266    base::TimeDelta time_until_release = time_until_render - min_wait_delta;
267    cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
268        base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
269        time_until_release);
270    VLOG(1) << "Wait before releasing frame "
271            << static_cast<int>((*encoded_frame)->frame_id)
272            << " time " << time_until_release.InMilliseconds();
273    return false;
274  }
275
276  base::TimeDelta dont_show_timeout_delta =
277      base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs);
278  if (codec_ == kVp8 && time_until_render < dont_show_timeout_delta) {
279    (*encoded_frame)->data[0] &= 0xef;
280    VLOG(1) << "Don't show frame "
281            << static_cast<int>((*encoded_frame)->frame_id)
282            << " time_until_render:" << time_until_render.InMilliseconds();
283  } else {
284    VLOG(1) << "Show frame "
285            << static_cast<int>((*encoded_frame)->frame_id)
286            << " time_until_render:" << time_until_render.InMilliseconds();
287  }
288  // We have a copy of the frame, release this one.
289  framer_->ReleaseFrame((*encoded_frame)->frame_id);
290  (*encoded_frame)->codec = codec_;
291  return true;
292}
293
294void VideoReceiver::PlayoutTimeout() {
295  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
296  if (queued_encoded_callbacks_.empty()) return;
297
298  uint32 rtp_timestamp = 0;
299  bool next_frame = false;
300  scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
301
302  if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
303                                     &next_frame)) {
304    // We have no video frames. Wait for new packet(s).
305    // Since the application can post multiple VideoFrameEncodedCallback and
306    // we only check the next frame to play out we might have multiple timeout
307    // events firing after each other; however this should be a rare event.
308    VLOG(1) << "Failed to retrieved a complete frame at this point in time";
309    return;
310  }
311  VLOG(1) << "PlayoutTimeout retrieved frame "
312          << static_cast<int>(encoded_frame->frame_id);
313
314  if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) {
315    // Logging already done.
316    return;
317  }
318
319  base::TimeTicks render_time;
320  if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
321                            &render_time)) {
322    if (!queued_encoded_callbacks_.empty()) {
323      VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
324      queued_encoded_callbacks_.pop_front();
325      cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
326          base::Bind(callback, base::Passed(&encoded_frame), render_time));
327    }
328  } else {
329    // We have a video frame; however we are missing packets and we have time
330    // to wait for new packet(s).
331  }
332}
333
334base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now,
335                                             uint32 rtp_timestamp) {
336  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
337  // Senders time in ms when this frame was captured.
338  // Note: the senders clock and our local clock might not be synced.
339  base::TimeTicks rtp_timestamp_in_ticks;
340
341  if (time_offset_.InMilliseconds() == 0) {
342    if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
343                                         incoming_rtp_timestamp_,
344                                         &rtp_timestamp_in_ticks)) {
345      // We have not received any RTCP to sync the stream play it out as soon as
346      // possible.
347      return now;
348    }
349    time_offset_ = time_incoming_packet_ - rtp_timestamp_in_ticks;
350  } else if (time_incoming_packet_updated_) {
351    if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
352                                        incoming_rtp_timestamp_,
353                                        &rtp_timestamp_in_ticks)) {
354      // Time to update the time_offset.
355      base::TimeDelta time_offset =
356          time_incoming_packet_ - rtp_timestamp_in_ticks;
357      time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset)
358          / kTimeOffsetFilter;
359    }
360  }
361  // Reset |time_incoming_packet_updated_| to enable a future measurement.
362  time_incoming_packet_updated_ = false;
363  if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
364                                       rtp_timestamp,
365                                       &rtp_timestamp_in_ticks)) {
366    // This can fail if we have not received any RTCP packets in a long time.
367    return now;
368  }
369  return (rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_);
370}
371
372void VideoReceiver::IncomingPacket(const uint8* packet, size_t length,
373                                   const base::Closure callback) {
374  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
375  if (Rtcp::IsRtcpPacket(packet, length)) {
376    rtcp_->IncomingRtcpPacket(packet, length);
377  } else {
378    rtp_receiver_.ReceivedPacket(packet, length);
379  }
380  cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
381}
382
383void VideoReceiver::IncomingParsedRtpPacket(const uint8* payload_data,
384                                            size_t payload_size,
385                                            const RtpCastHeader& rtp_header) {
386  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
387
388  base::TimeTicks now = cast_environment_->Clock()->NowTicks();
389  if (time_incoming_packet_.is_null() || now - time_incoming_packet_ >
390      base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
391    if (time_incoming_packet_.is_null()) InitializeTimers();
392    incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp;
393    time_incoming_packet_ = now;
394    time_incoming_packet_updated_ = true;
395  }
396
397  cast_environment_->Logging()->InsertPacketEvent(kPacketReceived,
398      rtp_header.webrtc.header.timestamp, rtp_header.frame_id,
399      rtp_header.packet_id, rtp_header.max_packet_id, payload_size);
400
401  bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header);
402
403  if (!complete) return;  // Video frame not complete; wait for more packets.
404  if (queued_encoded_callbacks_.empty()) return;  // No pending callback.
405
406  VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
407  queued_encoded_callbacks_.pop_front();
408  cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
409      base::Bind(&VideoReceiver::GetEncodedVideoFrame,
410          weak_factory_.GetWeakPtr(), callback));
411}
412
413// Send a cast feedback message. Actual message created in the framer (cast
414// message builder).
415void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
416  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
417  // TODO(pwestin): wire up log messages.
418  rtcp_->SendRtcpFromRtpReceiver(&cast_message, NULL);
419  time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks();
420}
421
422// Cast messages should be sent within a maximum interval. Schedule a call
423// if not triggered elsewhere, e.g. by the cast message_builder.
424void VideoReceiver::ScheduleNextCastMessage() {
425  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
426  base::TimeTicks send_time;
427  framer_->TimeToSendNextCastMessage(&send_time);
428
429  base::TimeDelta time_to_send = send_time -
430      cast_environment_->Clock()->NowTicks();
431  time_to_send = std::max(time_to_send,
432      base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
433  cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
434      base::Bind(&VideoReceiver::SendNextCastMessage,
435                 weak_factory_.GetWeakPtr()), time_to_send);
436}
437
438void VideoReceiver::SendNextCastMessage() {
439  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
440  framer_->SendCastMessage();  // Will only send a message if it is time.
441  ScheduleNextCastMessage();
442}
443
444// Schedule the next RTCP report to be sent back to the sender.
445void VideoReceiver::ScheduleNextRtcpReport() {
446  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
447  base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
448      cast_environment_->Clock()->NowTicks();
449
450  time_to_next = std::max(time_to_next,
451      base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
452
453  cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
454      base::Bind(&VideoReceiver::SendNextRtcpReport,
455                weak_factory_.GetWeakPtr()), time_to_next);
456}
457
458void VideoReceiver::SendNextRtcpReport() {
459  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
460  rtcp_->SendRtcpFromRtpReceiver(NULL, NULL);
461  ScheduleNextRtcpReport();
462}
463
464}  // namespace cast
465}  // namespace media
466