1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/session_info.h"
12
13#include "webrtc/base/logging.h"
14#include "webrtc/modules/video_coding/packet.h"
15
16namespace webrtc {
17
18namespace {
19
20uint16_t BufferToUWord16(const uint8_t* dataBuffer) {
21  return (dataBuffer[0] << 8) | dataBuffer[1];
22}
23
24}  // namespace
25
26VCMSessionInfo::VCMSessionInfo()
27    : session_nack_(false),
28      complete_(false),
29      decodable_(false),
30      frame_type_(kVideoFrameDelta),
31      packets_(),
32      empty_seq_num_low_(-1),
33      empty_seq_num_high_(-1),
34      first_packet_seq_num_(-1),
35      last_packet_seq_num_(-1) {}
36
37void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr,
38                                        const uint8_t* new_base_ptr) {
39  for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it)
40    if ((*it).dataPtr != NULL) {
41      assert(old_base_ptr != NULL && new_base_ptr != NULL);
42      (*it).dataPtr = new_base_ptr + ((*it).dataPtr - old_base_ptr);
43    }
44}
45
46int VCMSessionInfo::LowSequenceNumber() const {
47  if (packets_.empty())
48    return empty_seq_num_low_;
49  return packets_.front().seqNum;
50}
51
52int VCMSessionInfo::HighSequenceNumber() const {
53  if (packets_.empty())
54    return empty_seq_num_high_;
55  if (empty_seq_num_high_ == -1)
56    return packets_.back().seqNum;
57  return LatestSequenceNumber(packets_.back().seqNum, empty_seq_num_high_);
58}
59
60int VCMSessionInfo::PictureId() const {
61  if (packets_.empty())
62    return kNoPictureId;
63  if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) {
64    return packets_.front().codecSpecificHeader.codecHeader.VP8.pictureId;
65  } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) {
66    return packets_.front().codecSpecificHeader.codecHeader.VP9.picture_id;
67  } else {
68    return kNoPictureId;
69  }
70}
71
72int VCMSessionInfo::TemporalId() const {
73  if (packets_.empty())
74    return kNoTemporalIdx;
75  if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) {
76    return packets_.front().codecSpecificHeader.codecHeader.VP8.temporalIdx;
77  } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) {
78    return packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_idx;
79  } else {
80    return kNoTemporalIdx;
81  }
82}
83
84bool VCMSessionInfo::LayerSync() const {
85  if (packets_.empty())
86    return false;
87  if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) {
88    return packets_.front().codecSpecificHeader.codecHeader.VP8.layerSync;
89  } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) {
90    return packets_.front()
91        .codecSpecificHeader.codecHeader.VP9.temporal_up_switch;
92  } else {
93    return false;
94  }
95}
96
97int VCMSessionInfo::Tl0PicId() const {
98  if (packets_.empty())
99    return kNoTl0PicIdx;
100  if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) {
101    return packets_.front().codecSpecificHeader.codecHeader.VP8.tl0PicIdx;
102  } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) {
103    return packets_.front().codecSpecificHeader.codecHeader.VP9.tl0_pic_idx;
104  } else {
105    return kNoTl0PicIdx;
106  }
107}
108
109bool VCMSessionInfo::NonReference() const {
110  if (packets_.empty() ||
111      packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
112    return false;
113  return packets_.front().codecSpecificHeader.codecHeader.VP8.nonReference;
114}
115
116void VCMSessionInfo::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
117  if (packets_.empty() ||
118      packets_.front().codecSpecificHeader.codec != kRtpVideoVp9 ||
119      packets_.front().codecSpecificHeader.codecHeader.VP9.flexible_mode) {
120    return;
121  }
122  packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_idx =
123      gof_info.temporal_idx[idx];
124  packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_up_switch =
125      gof_info.temporal_up_switch[idx];
126  packets_.front().codecSpecificHeader.codecHeader.VP9.num_ref_pics =
127      gof_info.num_ref_pics[idx];
128  for (uint8_t i = 0; i < gof_info.num_ref_pics[idx]; ++i) {
129    packets_.front().codecSpecificHeader.codecHeader.VP9.pid_diff[i] =
130        gof_info.pid_diff[idx][i];
131  }
132}
133
134void VCMSessionInfo::Reset() {
135  session_nack_ = false;
136  complete_ = false;
137  decodable_ = false;
138  frame_type_ = kVideoFrameDelta;
139  packets_.clear();
140  empty_seq_num_low_ = -1;
141  empty_seq_num_high_ = -1;
142  first_packet_seq_num_ = -1;
143  last_packet_seq_num_ = -1;
144}
145
146size_t VCMSessionInfo::SessionLength() const {
147  size_t length = 0;
148  for (PacketIteratorConst it = packets_.begin(); it != packets_.end(); ++it)
149    length += (*it).sizeBytes;
150  return length;
151}
152
153int VCMSessionInfo::NumPackets() const {
154  return packets_.size();
155}
156
157size_t VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer,
158                                    PacketIterator packet_it) {
159  VCMPacket& packet = *packet_it;
160  PacketIterator it;
161
162  // Calculate the offset into the frame buffer for this packet.
163  size_t offset = 0;
164  for (it = packets_.begin(); it != packet_it; ++it)
165    offset += (*it).sizeBytes;
166
167  // Set the data pointer to pointing to the start of this packet in the
168  // frame buffer.
169  const uint8_t* packet_buffer = packet.dataPtr;
170  packet.dataPtr = frame_buffer + offset;
171
172  // We handle H.264 STAP-A packets in a special way as we need to remove the
173  // two length bytes between each NAL unit, and potentially add start codes.
174  // TODO(pbos): Remove H264 parsing from this step and use a fragmentation
175  // header supplied by the H264 depacketizer.
176  const size_t kH264NALHeaderLengthInBytes = 1;
177  const size_t kLengthFieldLength = 2;
178  if (packet.codecSpecificHeader.codec == kRtpVideoH264 &&
179      packet.codecSpecificHeader.codecHeader.H264.packetization_type ==
180          kH264StapA) {
181    size_t required_length = 0;
182    const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
183    while (nalu_ptr < packet_buffer + packet.sizeBytes) {
184      size_t length = BufferToUWord16(nalu_ptr);
185      required_length +=
186          length + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
187      nalu_ptr += kLengthFieldLength + length;
188    }
189    ShiftSubsequentPackets(packet_it, required_length);
190    nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
191    uint8_t* frame_buffer_ptr = frame_buffer + offset;
192    while (nalu_ptr < packet_buffer + packet.sizeBytes) {
193      size_t length = BufferToUWord16(nalu_ptr);
194      nalu_ptr += kLengthFieldLength;
195      frame_buffer_ptr += Insert(nalu_ptr, length, packet.insertStartCode,
196                                 const_cast<uint8_t*>(frame_buffer_ptr));
197      nalu_ptr += length;
198    }
199    packet.sizeBytes = required_length;
200    return packet.sizeBytes;
201  }
202  ShiftSubsequentPackets(
203      packet_it, packet.sizeBytes +
204                     (packet.insertStartCode ? kH264StartCodeLengthBytes : 0));
205
206  packet.sizeBytes =
207      Insert(packet_buffer, packet.sizeBytes, packet.insertStartCode,
208             const_cast<uint8_t*>(packet.dataPtr));
209  return packet.sizeBytes;
210}
211
212size_t VCMSessionInfo::Insert(const uint8_t* buffer,
213                              size_t length,
214                              bool insert_start_code,
215                              uint8_t* frame_buffer) {
216  if (insert_start_code) {
217    const unsigned char startCode[] = {0, 0, 0, 1};
218    memcpy(frame_buffer, startCode, kH264StartCodeLengthBytes);
219  }
220  memcpy(frame_buffer + (insert_start_code ? kH264StartCodeLengthBytes : 0),
221         buffer, length);
222  length += (insert_start_code ? kH264StartCodeLengthBytes : 0);
223
224  return length;
225}
226
227void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it,
228                                            int steps_to_shift) {
229  ++it;
230  if (it == packets_.end())
231    return;
232  uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr);
233  int shift_length = 0;
234  // Calculate the total move length and move the data pointers in advance.
235  for (; it != packets_.end(); ++it) {
236    shift_length += (*it).sizeBytes;
237    if ((*it).dataPtr != NULL)
238      (*it).dataPtr += steps_to_shift;
239  }
240  memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length);
241}
242
243void VCMSessionInfo::UpdateCompleteSession() {
244  if (HaveFirstPacket() && HaveLastPacket()) {
245    // Do we have all the packets in this session?
246    bool complete_session = true;
247    PacketIterator it = packets_.begin();
248    PacketIterator prev_it = it;
249    ++it;
250    for (; it != packets_.end(); ++it) {
251      if (!InSequence(it, prev_it)) {
252        complete_session = false;
253        break;
254      }
255      prev_it = it;
256    }
257    complete_ = complete_session;
258  }
259}
260
261void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) {
262  // Irrelevant if session is already complete or decodable
263  if (complete_ || decodable_)
264    return;
265  // TODO(agalusza): Account for bursty loss.
266  // TODO(agalusza): Refine these values to better approximate optimal ones.
267  // Do not decode frames if the RTT is lower than this.
268  const int64_t kRttThreshold = 100;
269  // Do not decode frames if the number of packets is between these two
270  // thresholds.
271  const float kLowPacketPercentageThreshold = 0.2f;
272  const float kHighPacketPercentageThreshold = 0.8f;
273  if (frame_data.rtt_ms < kRttThreshold || frame_type_ == kVideoFrameKey ||
274      !HaveFirstPacket() ||
275      (NumPackets() <= kHighPacketPercentageThreshold *
276                           frame_data.rolling_average_packets_per_frame &&
277       NumPackets() > kLowPacketPercentageThreshold *
278                          frame_data.rolling_average_packets_per_frame))
279    return;
280
281  decodable_ = true;
282}
283
284bool VCMSessionInfo::complete() const {
285  return complete_;
286}
287
288bool VCMSessionInfo::decodable() const {
289  return decodable_;
290}
291
292// Find the end of the NAL unit which the packet pointed to by |packet_it|
293// belongs to. Returns an iterator to the last packet of the frame if the end
294// of the NAL unit wasn't found.
295VCMSessionInfo::PacketIterator VCMSessionInfo::FindNaluEnd(
296    PacketIterator packet_it) const {
297  if ((*packet_it).completeNALU == kNaluEnd ||
298      (*packet_it).completeNALU == kNaluComplete) {
299    return packet_it;
300  }
301  // Find the end of the NAL unit.
302  for (; packet_it != packets_.end(); ++packet_it) {
303    if (((*packet_it).completeNALU == kNaluComplete &&
304         (*packet_it).sizeBytes > 0) ||
305        // Found next NALU.
306        (*packet_it).completeNALU == kNaluStart)
307      return --packet_it;
308    if ((*packet_it).completeNALU == kNaluEnd)
309      return packet_it;
310  }
311  // The end wasn't found.
312  return --packet_it;
313}
314
315size_t VCMSessionInfo::DeletePacketData(PacketIterator start,
316                                        PacketIterator end) {
317  size_t bytes_to_delete = 0;  // The number of bytes to delete.
318  PacketIterator packet_after_end = end;
319  ++packet_after_end;
320
321  // Get the number of bytes to delete.
322  // Clear the size of these packets.
323  for (PacketIterator it = start; it != packet_after_end; ++it) {
324    bytes_to_delete += (*it).sizeBytes;
325    (*it).sizeBytes = 0;
326    (*it).dataPtr = NULL;
327  }
328  if (bytes_to_delete > 0)
329    ShiftSubsequentPackets(end, -static_cast<int>(bytes_to_delete));
330  return bytes_to_delete;
331}
332
333size_t VCMSessionInfo::BuildVP8FragmentationHeader(
334    uint8_t* frame_buffer,
335    size_t frame_buffer_length,
336    RTPFragmentationHeader* fragmentation) {
337  size_t new_length = 0;
338  // Allocate space for max number of partitions
339  fragmentation->VerifyAndAllocateFragmentationHeader(kMaxVP8Partitions);
340  fragmentation->fragmentationVectorSize = 0;
341  memset(fragmentation->fragmentationLength, 0,
342         kMaxVP8Partitions * sizeof(size_t));
343  if (packets_.empty())
344    return new_length;
345  PacketIterator it = FindNextPartitionBeginning(packets_.begin());
346  while (it != packets_.end()) {
347    const int partition_id =
348        (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
349    PacketIterator partition_end = FindPartitionEnd(it);
350    fragmentation->fragmentationOffset[partition_id] =
351        (*it).dataPtr - frame_buffer;
352    assert(fragmentation->fragmentationOffset[partition_id] <
353           frame_buffer_length);
354    fragmentation->fragmentationLength[partition_id] =
355        (*partition_end).dataPtr + (*partition_end).sizeBytes - (*it).dataPtr;
356    assert(fragmentation->fragmentationLength[partition_id] <=
357           frame_buffer_length);
358    new_length += fragmentation->fragmentationLength[partition_id];
359    ++partition_end;
360    it = FindNextPartitionBeginning(partition_end);
361    if (partition_id + 1 > fragmentation->fragmentationVectorSize)
362      fragmentation->fragmentationVectorSize = partition_id + 1;
363  }
364  // Set all empty fragments to start where the previous fragment ends,
365  // and have zero length.
366  if (fragmentation->fragmentationLength[0] == 0)
367    fragmentation->fragmentationOffset[0] = 0;
368  for (int i = 1; i < fragmentation->fragmentationVectorSize; ++i) {
369    if (fragmentation->fragmentationLength[i] == 0)
370      fragmentation->fragmentationOffset[i] =
371          fragmentation->fragmentationOffset[i - 1] +
372          fragmentation->fragmentationLength[i - 1];
373    assert(i == 0 ||
374           fragmentation->fragmentationOffset[i] >=
375               fragmentation->fragmentationOffset[i - 1]);
376  }
377  assert(new_length <= frame_buffer_length);
378  return new_length;
379}
380
381VCMSessionInfo::PacketIterator VCMSessionInfo::FindNextPartitionBeginning(
382    PacketIterator it) const {
383  while (it != packets_.end()) {
384    if ((*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition) {
385      return it;
386    }
387    ++it;
388  }
389  return it;
390}
391
392VCMSessionInfo::PacketIterator VCMSessionInfo::FindPartitionEnd(
393    PacketIterator it) const {
394  assert((*it).codec == kVideoCodecVP8);
395  PacketIterator prev_it = it;
396  const int partition_id =
397      (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
398  while (it != packets_.end()) {
399    bool beginning =
400        (*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition;
401    int current_partition_id =
402        (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
403    bool packet_loss_found = (!beginning && !InSequence(it, prev_it));
404    if (packet_loss_found ||
405        (beginning && current_partition_id != partition_id)) {
406      // Missing packet, the previous packet was the last in sequence.
407      return prev_it;
408    }
409    prev_it = it;
410    ++it;
411  }
412  return prev_it;
413}
414
415bool VCMSessionInfo::InSequence(const PacketIterator& packet_it,
416                                const PacketIterator& prev_packet_it) {
417  // If the two iterators are pointing to the same packet they are considered
418  // to be in sequence.
419  return (packet_it == prev_packet_it ||
420          (static_cast<uint16_t>((*prev_packet_it).seqNum + 1) ==
421           (*packet_it).seqNum));
422}
423
424size_t VCMSessionInfo::MakeDecodable() {
425  size_t return_length = 0;
426  if (packets_.empty()) {
427    return 0;
428  }
429  PacketIterator it = packets_.begin();
430  // Make sure we remove the first NAL unit if it's not decodable.
431  if ((*it).completeNALU == kNaluIncomplete || (*it).completeNALU == kNaluEnd) {
432    PacketIterator nalu_end = FindNaluEnd(it);
433    return_length += DeletePacketData(it, nalu_end);
434    it = nalu_end;
435  }
436  PacketIterator prev_it = it;
437  // Take care of the rest of the NAL units.
438  for (; it != packets_.end(); ++it) {
439    bool start_of_nalu = ((*it).completeNALU == kNaluStart ||
440                          (*it).completeNALU == kNaluComplete);
441    if (!start_of_nalu && !InSequence(it, prev_it)) {
442      // Found a sequence number gap due to packet loss.
443      PacketIterator nalu_end = FindNaluEnd(it);
444      return_length += DeletePacketData(it, nalu_end);
445      it = nalu_end;
446    }
447    prev_it = it;
448  }
449  return return_length;
450}
451
452void VCMSessionInfo::SetNotDecodableIfIncomplete() {
453  // We don't need to check for completeness first because the two are
454  // orthogonal. If complete_ is true, decodable_ is irrelevant.
455  decodable_ = false;
456}
457
458bool VCMSessionInfo::HaveFirstPacket() const {
459  return !packets_.empty() && (first_packet_seq_num_ != -1);
460}
461
462bool VCMSessionInfo::HaveLastPacket() const {
463  return !packets_.empty() && (last_packet_seq_num_ != -1);
464}
465
466bool VCMSessionInfo::session_nack() const {
467  return session_nack_;
468}
469
470int VCMSessionInfo::InsertPacket(const VCMPacket& packet,
471                                 uint8_t* frame_buffer,
472                                 VCMDecodeErrorMode decode_error_mode,
473                                 const FrameData& frame_data) {
474  if (packet.frameType == kEmptyFrame) {
475    // Update sequence number of an empty packet.
476    // Only media packets are inserted into the packet list.
477    InformOfEmptyPacket(packet.seqNum);
478    return 0;
479  }
480
481  if (packets_.size() == kMaxPacketsInSession) {
482    LOG(LS_ERROR) << "Max number of packets per frame has been reached.";
483    return -1;
484  }
485
486  // Find the position of this packet in the packet list in sequence number
487  // order and insert it. Loop over the list in reverse order.
488  ReversePacketIterator rit = packets_.rbegin();
489  for (; rit != packets_.rend(); ++rit)
490    if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum)
491      break;
492
493  // Check for duplicate packets.
494  if (rit != packets_.rend() && (*rit).seqNum == packet.seqNum &&
495      (*rit).sizeBytes > 0)
496    return -2;
497
498  if (packet.codec == kVideoCodecH264) {
499    frame_type_ = packet.frameType;
500    if (packet.isFirstPacket &&
501        (first_packet_seq_num_ == -1 ||
502         IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum))) {
503      first_packet_seq_num_ = packet.seqNum;
504    }
505    if (packet.markerBit &&
506        (last_packet_seq_num_ == -1 ||
507         IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_))) {
508      last_packet_seq_num_ = packet.seqNum;
509    }
510  } else {
511    // Only insert media packets between first and last packets (when
512    // available).
513    // Placing check here, as to properly account for duplicate packets.
514    // Check if this is first packet (only valid for some codecs)
515    // Should only be set for one packet per session.
516    if (packet.isFirstPacket && first_packet_seq_num_ == -1) {
517      // The first packet in a frame signals the frame type.
518      frame_type_ = packet.frameType;
519      // Store the sequence number for the first packet.
520      first_packet_seq_num_ = static_cast<int>(packet.seqNum);
521    } else if (first_packet_seq_num_ != -1 &&
522               IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum)) {
523      LOG(LS_WARNING) << "Received packet with a sequence number which is out "
524                         "of frame boundaries";
525      return -3;
526    } else if (frame_type_ == kEmptyFrame && packet.frameType != kEmptyFrame) {
527      // Update the frame type with the type of the first media packet.
528      // TODO(mikhal): Can this trigger?
529      frame_type_ = packet.frameType;
530    }
531
532    // Track the marker bit, should only be set for one packet per session.
533    if (packet.markerBit && last_packet_seq_num_ == -1) {
534      last_packet_seq_num_ = static_cast<int>(packet.seqNum);
535    } else if (last_packet_seq_num_ != -1 &&
536               IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_)) {
537      LOG(LS_WARNING) << "Received packet with a sequence number which is out "
538                         "of frame boundaries";
539      return -3;
540    }
541  }
542
543  // The insert operation invalidates the iterator |rit|.
544  PacketIterator packet_list_it = packets_.insert(rit.base(), packet);
545
546  size_t returnLength = InsertBuffer(frame_buffer, packet_list_it);
547  UpdateCompleteSession();
548  if (decode_error_mode == kWithErrors)
549    decodable_ = true;
550  else if (decode_error_mode == kSelectiveErrors)
551    UpdateDecodableSession(frame_data);
552  return static_cast<int>(returnLength);
553}
554
555void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) {
556  // Empty packets may be FEC or filler packets. They are sequential and
557  // follow the data packets, therefore, we should only keep track of the high
558  // and low sequence numbers and may assume that the packets in between are
559  // empty packets belonging to the same frame (timestamp).
560  if (empty_seq_num_high_ == -1)
561    empty_seq_num_high_ = seq_num;
562  else
563    empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_);
564  if (empty_seq_num_low_ == -1 ||
565      IsNewerSequenceNumber(empty_seq_num_low_, seq_num))
566    empty_seq_num_low_ = seq_num;
567}
568
569}  // namespace webrtc
570