1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/main/source/session_info.h"
12
13#include "webrtc/modules/video_coding/main/source/packet.h"
14#include "webrtc/system_wrappers/interface/logging.h"
15
16namespace webrtc {
17namespace {
18// Used in determining whether a frame is decodable.
19enum {kRttThreshold = 100};  // Not decodable if Rtt is lower than this.
20
21// Do not decode frames if the number of packets is between these two
22// thresholds.
23static const float kLowPacketPercentageThreshold = 0.2f;
24static const float kHighPacketPercentageThreshold = 0.8f;
25
26uint16_t BufferToUWord16(const uint8_t* dataBuffer) {
27  return (dataBuffer[0] << 8) | dataBuffer[1];
28}
29}  // namespace
30
31VCMSessionInfo::VCMSessionInfo()
32    : session_nack_(false),
33      complete_(false),
34      decodable_(false),
35      frame_type_(kVideoFrameDelta),
36      packets_(),
37      empty_seq_num_low_(-1),
38      empty_seq_num_high_(-1),
39      first_packet_seq_num_(-1),
40      last_packet_seq_num_(-1) {
41}
42
43void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr,
44                                        const uint8_t* new_base_ptr) {
45  for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it)
46    if ((*it).dataPtr != NULL) {
47      assert(old_base_ptr != NULL && new_base_ptr != NULL);
48      (*it).dataPtr = new_base_ptr + ((*it).dataPtr - old_base_ptr);
49    }
50}
51
52int VCMSessionInfo::LowSequenceNumber() const {
53  if (packets_.empty())
54    return empty_seq_num_low_;
55  return packets_.front().seqNum;
56}
57
58int VCMSessionInfo::HighSequenceNumber() const {
59  if (packets_.empty())
60    return empty_seq_num_high_;
61  if (empty_seq_num_high_ == -1)
62    return packets_.back().seqNum;
63  return LatestSequenceNumber(packets_.back().seqNum, empty_seq_num_high_);
64}
65
66int VCMSessionInfo::PictureId() const {
67  if (packets_.empty() ||
68      packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
69    return kNoPictureId;
70  return packets_.front().codecSpecificHeader.codecHeader.VP8.pictureId;
71}
72
73int VCMSessionInfo::TemporalId() const {
74  if (packets_.empty() ||
75      packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
76    return kNoTemporalIdx;
77  return packets_.front().codecSpecificHeader.codecHeader.VP8.temporalIdx;
78}
79
80bool VCMSessionInfo::LayerSync() const {
81  if (packets_.empty() ||
82        packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
83    return false;
84  return packets_.front().codecSpecificHeader.codecHeader.VP8.layerSync;
85}
86
87int VCMSessionInfo::Tl0PicId() const {
88  if (packets_.empty() ||
89      packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
90    return kNoTl0PicIdx;
91  return packets_.front().codecSpecificHeader.codecHeader.VP8.tl0PicIdx;
92}
93
94bool VCMSessionInfo::NonReference() const {
95  if (packets_.empty() ||
96      packets_.front().codecSpecificHeader.codec != kRtpVideoVp8)
97    return false;
98  return packets_.front().codecSpecificHeader.codecHeader.VP8.nonReference;
99}
100
101void VCMSessionInfo::Reset() {
102  session_nack_ = false;
103  complete_ = false;
104  decodable_ = false;
105  frame_type_ = kVideoFrameDelta;
106  packets_.clear();
107  empty_seq_num_low_ = -1;
108  empty_seq_num_high_ = -1;
109  first_packet_seq_num_ = -1;
110  last_packet_seq_num_ = -1;
111}
112
113int VCMSessionInfo::SessionLength() const {
114  int length = 0;
115  for (PacketIteratorConst it = packets_.begin(); it != packets_.end(); ++it)
116    length += (*it).sizeBytes;
117  return length;
118}
119
120int VCMSessionInfo::NumPackets() const {
121  return packets_.size();
122}
123
124int VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer,
125                                 PacketIterator packet_it) {
126  VCMPacket& packet = *packet_it;
127  PacketIterator it;
128
129  // Calculate the offset into the frame buffer for this packet.
130  int offset = 0;
131  for (it = packets_.begin(); it != packet_it; ++it)
132    offset += (*it).sizeBytes;
133
134  // Set the data pointer to pointing to the start of this packet in the
135  // frame buffer.
136  const uint8_t* packet_buffer = packet.dataPtr;
137  packet.dataPtr = frame_buffer + offset;
138
139  // We handle H.264 STAP-A packets in a special way as we need to remove the
140  // two length bytes between each NAL unit, and potentially add start codes.
141  const size_t kH264NALHeaderLengthInBytes = 1;
142  const size_t kLengthFieldLength = 2;
143  if (packet.codecSpecificHeader.codec == kRtpVideoH264 &&
144      packet.codecSpecificHeader.codecHeader.H264.stap_a) {
145    size_t required_length = 0;
146    const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
147    while (nalu_ptr < packet_buffer + packet.sizeBytes) {
148      uint32_t length = BufferToUWord16(nalu_ptr);
149      required_length +=
150          length + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
151      nalu_ptr += kLengthFieldLength + length;
152    }
153    ShiftSubsequentPackets(packet_it, required_length);
154    nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
155    uint8_t* frame_buffer_ptr = frame_buffer + offset;
156    while (nalu_ptr < packet_buffer + packet.sizeBytes) {
157      uint32_t length = BufferToUWord16(nalu_ptr);
158      nalu_ptr += kLengthFieldLength;
159      frame_buffer_ptr += Insert(nalu_ptr,
160                                 length,
161                                 packet.insertStartCode,
162                                 const_cast<uint8_t*>(frame_buffer_ptr));
163      nalu_ptr += length;
164    }
165    packet.sizeBytes = required_length;
166    return packet.sizeBytes;
167  }
168  ShiftSubsequentPackets(
169      packet_it,
170      packet.sizeBytes +
171          (packet.insertStartCode ? kH264StartCodeLengthBytes : 0));
172
173  packet.sizeBytes = Insert(packet_buffer,
174                            packet.sizeBytes,
175                            packet.insertStartCode,
176                            const_cast<uint8_t*>(packet.dataPtr));
177  return packet.sizeBytes;
178}
179
180size_t VCMSessionInfo::Insert(const uint8_t* buffer,
181                              size_t length,
182                              bool insert_start_code,
183                              uint8_t* frame_buffer) {
184  if (insert_start_code) {
185    const unsigned char startCode[] = {0, 0, 0, 1};
186    memcpy(frame_buffer, startCode, kH264StartCodeLengthBytes);
187  }
188  memcpy(frame_buffer + (insert_start_code ? kH264StartCodeLengthBytes : 0),
189         buffer,
190         length);
191  length += (insert_start_code ? kH264StartCodeLengthBytes : 0);
192
193  return length;
194}
195
196void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it,
197                                            int steps_to_shift) {
198  ++it;
199  if (it == packets_.end())
200    return;
201  uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr);
202  int shift_length = 0;
203  // Calculate the total move length and move the data pointers in advance.
204  for (; it != packets_.end(); ++it) {
205    shift_length += (*it).sizeBytes;
206    if ((*it).dataPtr != NULL)
207      (*it).dataPtr += steps_to_shift;
208  }
209  memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length);
210}
211
212void VCMSessionInfo::UpdateCompleteSession() {
213  if (HaveFirstPacket() && HaveLastPacket()) {
214    // Do we have all the packets in this session?
215    bool complete_session = true;
216    PacketIterator it = packets_.begin();
217    PacketIterator prev_it = it;
218    ++it;
219    for (; it != packets_.end(); ++it) {
220      if (!InSequence(it, prev_it)) {
221        complete_session = false;
222        break;
223      }
224      prev_it = it;
225    }
226    complete_ = complete_session;
227  }
228}
229
230void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) {
231  // Irrelevant if session is already complete or decodable
232  if (complete_ || decodable_)
233    return;
234  // TODO(agalusza): Account for bursty loss.
235  // TODO(agalusza): Refine these values to better approximate optimal ones.
236  if (frame_data.rtt_ms < kRttThreshold
237      || frame_type_ == kVideoFrameKey
238      || !HaveFirstPacket()
239      || (NumPackets() <= kHighPacketPercentageThreshold
240                          * frame_data.rolling_average_packets_per_frame
241          && NumPackets() > kLowPacketPercentageThreshold
242                            * frame_data.rolling_average_packets_per_frame))
243    return;
244
245  decodable_ = true;
246}
247
248bool VCMSessionInfo::complete() const {
249  return complete_;
250}
251
252bool VCMSessionInfo::decodable() const {
253  return decodable_;
254}
255
256// Find the end of the NAL unit which the packet pointed to by |packet_it|
257// belongs to. Returns an iterator to the last packet of the frame if the end
258// of the NAL unit wasn't found.
259VCMSessionInfo::PacketIterator VCMSessionInfo::FindNaluEnd(
260    PacketIterator packet_it) const {
261  if ((*packet_it).completeNALU == kNaluEnd ||
262      (*packet_it).completeNALU == kNaluComplete) {
263    return packet_it;
264  }
265  // Find the end of the NAL unit.
266  for (; packet_it != packets_.end(); ++packet_it) {
267    if (((*packet_it).completeNALU == kNaluComplete &&
268        (*packet_it).sizeBytes > 0) ||
269        // Found next NALU.
270        (*packet_it).completeNALU == kNaluStart)
271      return --packet_it;
272    if ((*packet_it).completeNALU == kNaluEnd)
273      return packet_it;
274  }
275  // The end wasn't found.
276  return --packet_it;
277}
278
279int VCMSessionInfo::DeletePacketData(PacketIterator start,
280                                     PacketIterator end) {
281  int bytes_to_delete = 0;  // The number of bytes to delete.
282  PacketIterator packet_after_end = end;
283  ++packet_after_end;
284
285  // Get the number of bytes to delete.
286  // Clear the size of these packets.
287  for (PacketIterator it = start; it != packet_after_end; ++it) {
288    bytes_to_delete += (*it).sizeBytes;
289    (*it).sizeBytes = 0;
290    (*it).dataPtr = NULL;
291  }
292  if (bytes_to_delete > 0)
293    ShiftSubsequentPackets(end, -bytes_to_delete);
294  return bytes_to_delete;
295}
296
297int VCMSessionInfo::BuildVP8FragmentationHeader(
298    uint8_t* frame_buffer,
299    int frame_buffer_length,
300    RTPFragmentationHeader* fragmentation) {
301  int new_length = 0;
302  // Allocate space for max number of partitions
303  fragmentation->VerifyAndAllocateFragmentationHeader(kMaxVP8Partitions);
304  fragmentation->fragmentationVectorSize = 0;
305  memset(fragmentation->fragmentationLength, 0,
306         kMaxVP8Partitions * sizeof(uint32_t));
307  if (packets_.empty())
308      return new_length;
309  PacketIterator it = FindNextPartitionBeginning(packets_.begin());
310  while (it != packets_.end()) {
311    const int partition_id =
312        (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
313    PacketIterator partition_end = FindPartitionEnd(it);
314    fragmentation->fragmentationOffset[partition_id] =
315        (*it).dataPtr - frame_buffer;
316    assert(fragmentation->fragmentationOffset[partition_id] <
317           static_cast<uint32_t>(frame_buffer_length));
318    fragmentation->fragmentationLength[partition_id] =
319        (*partition_end).dataPtr + (*partition_end).sizeBytes - (*it).dataPtr;
320    assert(fragmentation->fragmentationLength[partition_id] <=
321           static_cast<uint32_t>(frame_buffer_length));
322    new_length += fragmentation->fragmentationLength[partition_id];
323    ++partition_end;
324    it = FindNextPartitionBeginning(partition_end);
325    if (partition_id + 1 > fragmentation->fragmentationVectorSize)
326      fragmentation->fragmentationVectorSize = partition_id + 1;
327  }
328  // Set all empty fragments to start where the previous fragment ends,
329  // and have zero length.
330  if (fragmentation->fragmentationLength[0] == 0)
331      fragmentation->fragmentationOffset[0] = 0;
332  for (int i = 1; i < fragmentation->fragmentationVectorSize; ++i) {
333    if (fragmentation->fragmentationLength[i] == 0)
334      fragmentation->fragmentationOffset[i] =
335          fragmentation->fragmentationOffset[i - 1] +
336          fragmentation->fragmentationLength[i - 1];
337    assert(i == 0 ||
338           fragmentation->fragmentationOffset[i] >=
339           fragmentation->fragmentationOffset[i - 1]);
340  }
341  assert(new_length <= frame_buffer_length);
342  return new_length;
343}
344
345VCMSessionInfo::PacketIterator VCMSessionInfo::FindNextPartitionBeginning(
346    PacketIterator it) const {
347  while (it != packets_.end()) {
348    if ((*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition) {
349      return it;
350    }
351    ++it;
352  }
353  return it;
354}
355
356VCMSessionInfo::PacketIterator VCMSessionInfo::FindPartitionEnd(
357    PacketIterator it) const {
358  assert((*it).codec == kVideoCodecVP8);
359  PacketIterator prev_it = it;
360  const int partition_id =
361      (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
362  while (it != packets_.end()) {
363    bool beginning =
364        (*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition;
365    int current_partition_id =
366        (*it).codecSpecificHeader.codecHeader.VP8.partitionId;
367    bool packet_loss_found = (!beginning && !InSequence(it, prev_it));
368    if (packet_loss_found ||
369        (beginning && current_partition_id != partition_id)) {
370      // Missing packet, the previous packet was the last in sequence.
371      return prev_it;
372    }
373    prev_it = it;
374    ++it;
375  }
376  return prev_it;
377}
378
379bool VCMSessionInfo::InSequence(const PacketIterator& packet_it,
380                                const PacketIterator& prev_packet_it) {
381  // If the two iterators are pointing to the same packet they are considered
382  // to be in sequence.
383  return (packet_it == prev_packet_it ||
384      (static_cast<uint16_t>((*prev_packet_it).seqNum + 1) ==
385          (*packet_it).seqNum));
386}
387
388int VCMSessionInfo::MakeDecodable() {
389  int return_length = 0;
390  if (packets_.empty()) {
391    return 0;
392  }
393  PacketIterator it = packets_.begin();
394  // Make sure we remove the first NAL unit if it's not decodable.
395  if ((*it).completeNALU == kNaluIncomplete ||
396      (*it).completeNALU == kNaluEnd) {
397    PacketIterator nalu_end = FindNaluEnd(it);
398    return_length += DeletePacketData(it, nalu_end);
399    it = nalu_end;
400  }
401  PacketIterator prev_it = it;
402  // Take care of the rest of the NAL units.
403  for (; it != packets_.end(); ++it) {
404    bool start_of_nalu = ((*it).completeNALU == kNaluStart ||
405        (*it).completeNALU == kNaluComplete);
406    if (!start_of_nalu && !InSequence(it, prev_it)) {
407      // Found a sequence number gap due to packet loss.
408      PacketIterator nalu_end = FindNaluEnd(it);
409      return_length += DeletePacketData(it, nalu_end);
410      it = nalu_end;
411    }
412    prev_it = it;
413  }
414  return return_length;
415}
416
417void VCMSessionInfo::SetNotDecodableIfIncomplete() {
418  // We don't need to check for completeness first because the two are
419  // orthogonal. If complete_ is true, decodable_ is irrelevant.
420  decodable_ = false;
421}
422
423bool
424VCMSessionInfo::HaveFirstPacket() const {
425  return !packets_.empty() && (first_packet_seq_num_ != -1);
426}
427
428bool
429VCMSessionInfo::HaveLastPacket() const {
430  return !packets_.empty() && (last_packet_seq_num_ != -1);
431}
432
433bool
434VCMSessionInfo::session_nack() const {
435  return session_nack_;
436}
437
438int VCMSessionInfo::InsertPacket(const VCMPacket& packet,
439                                 uint8_t* frame_buffer,
440                                 VCMDecodeErrorMode decode_error_mode,
441                                 const FrameData& frame_data) {
442  if (packet.frameType == kFrameEmpty) {
443    // Update sequence number of an empty packet.
444    // Only media packets are inserted into the packet list.
445    InformOfEmptyPacket(packet.seqNum);
446    return 0;
447  }
448
449  if (packets_.size() == kMaxPacketsInSession) {
450    LOG(LS_ERROR) << "Max number of packets per frame has been reached.";
451    return -1;
452  }
453
454  // Find the position of this packet in the packet list in sequence number
455  // order and insert it. Loop over the list in reverse order.
456  ReversePacketIterator rit = packets_.rbegin();
457  for (; rit != packets_.rend(); ++rit)
458    if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum)
459      break;
460
461  // Check for duplicate packets.
462  if (rit != packets_.rend() &&
463      (*rit).seqNum == packet.seqNum && (*rit).sizeBytes > 0)
464    return -2;
465
466  if (packet.codec == kVideoCodecH264) {
467    frame_type_ = packet.frameType;
468    if (packet.isFirstPacket &&
469        (first_packet_seq_num_ == -1 ||
470         IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum))) {
471      first_packet_seq_num_ = packet.seqNum;
472    }
473    if (packet.markerBit &&
474        (last_packet_seq_num_ == -1 ||
475         IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_))) {
476      last_packet_seq_num_ = packet.seqNum;
477    }
478  } else {
479    // Only insert media packets between first and last packets (when
480    // available).
481    // Placing check here, as to properly account for duplicate packets.
482    // Check if this is first packet (only valid for some codecs)
483    // Should only be set for one packet per session.
484    if (packet.isFirstPacket && first_packet_seq_num_ == -1) {
485      // The first packet in a frame signals the frame type.
486      frame_type_ = packet.frameType;
487      // Store the sequence number for the first packet.
488      first_packet_seq_num_ = static_cast<int>(packet.seqNum);
489    } else if (first_packet_seq_num_ != -1 &&
490               !IsNewerSequenceNumber(packet.seqNum, first_packet_seq_num_)) {
491      LOG(LS_WARNING) << "Received packet with a sequence number which is out "
492                         "of frame boundaries";
493      return -3;
494    } else if (frame_type_ == kFrameEmpty && packet.frameType != kFrameEmpty) {
495      // Update the frame type with the type of the first media packet.
496      // TODO(mikhal): Can this trigger?
497      frame_type_ = packet.frameType;
498    }
499
500    // Track the marker bit, should only be set for one packet per session.
501    if (packet.markerBit && last_packet_seq_num_ == -1) {
502      last_packet_seq_num_ = static_cast<int>(packet.seqNum);
503    } else if (last_packet_seq_num_ != -1 &&
504               IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_)) {
505      LOG(LS_WARNING) << "Received packet with a sequence number which is out "
506                         "of frame boundaries";
507      return -3;
508    }
509  }
510
511  // The insert operation invalidates the iterator |rit|.
512  PacketIterator packet_list_it = packets_.insert(rit.base(), packet);
513
514  int returnLength = InsertBuffer(frame_buffer, packet_list_it);
515  UpdateCompleteSession();
516  if (decode_error_mode == kWithErrors)
517    decodable_ = true;
518  else if (decode_error_mode == kSelectiveErrors)
519    UpdateDecodableSession(frame_data);
520  return returnLength;
521}
522
523void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) {
524  // Empty packets may be FEC or filler packets. They are sequential and
525  // follow the data packets, therefore, we should only keep track of the high
526  // and low sequence numbers and may assume that the packets in between are
527  // empty packets belonging to the same frame (timestamp).
528  if (empty_seq_num_high_ == -1)
529    empty_seq_num_high_ = seq_num;
530  else
531    empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_);
532  if (empty_seq_num_low_ == -1 || IsNewerSequenceNumber(empty_seq_num_low_,
533                                                        seq_num))
534    empty_seq_num_low_ = seq_num;
535}
536
537}  // namespace webrtc
538