1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/video_coding/main/source/session_info.h" 12 13#include "webrtc/modules/video_coding/main/source/packet.h" 14#include "webrtc/system_wrappers/interface/logging.h" 15 16namespace webrtc { 17namespace { 18// Used in determining whether a frame is decodable. 19enum {kRttThreshold = 100}; // Not decodable if Rtt is lower than this. 20 21// Do not decode frames if the number of packets is between these two 22// thresholds. 23static const float kLowPacketPercentageThreshold = 0.2f; 24static const float kHighPacketPercentageThreshold = 0.8f; 25 26uint16_t BufferToUWord16(const uint8_t* dataBuffer) { 27 return (dataBuffer[0] << 8) | dataBuffer[1]; 28} 29} // namespace 30 31VCMSessionInfo::VCMSessionInfo() 32 : session_nack_(false), 33 complete_(false), 34 decodable_(false), 35 frame_type_(kVideoFrameDelta), 36 packets_(), 37 empty_seq_num_low_(-1), 38 empty_seq_num_high_(-1), 39 first_packet_seq_num_(-1), 40 last_packet_seq_num_(-1) { 41} 42 43void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr, 44 const uint8_t* new_base_ptr) { 45 for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it) 46 if ((*it).dataPtr != NULL) { 47 assert(old_base_ptr != NULL && new_base_ptr != NULL); 48 (*it).dataPtr = new_base_ptr + ((*it).dataPtr - old_base_ptr); 49 } 50} 51 52int VCMSessionInfo::LowSequenceNumber() const { 53 if (packets_.empty()) 54 return empty_seq_num_low_; 55 return packets_.front().seqNum; 56} 57 58int VCMSessionInfo::HighSequenceNumber() const { 59 if (packets_.empty()) 60 return empty_seq_num_high_; 61 if (empty_seq_num_high_ == -1) 62 return packets_.back().seqNum; 63 return LatestSequenceNumber(packets_.back().seqNum, empty_seq_num_high_); 64} 65 66int VCMSessionInfo::PictureId() const { 67 if (packets_.empty() || 68 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 69 return kNoPictureId; 70 return packets_.front().codecSpecificHeader.codecHeader.VP8.pictureId; 71} 72 73int VCMSessionInfo::TemporalId() const { 74 if (packets_.empty() || 75 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 76 return kNoTemporalIdx; 77 return packets_.front().codecSpecificHeader.codecHeader.VP8.temporalIdx; 78} 79 80bool VCMSessionInfo::LayerSync() const { 81 if (packets_.empty() || 82 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 83 return false; 84 return packets_.front().codecSpecificHeader.codecHeader.VP8.layerSync; 85} 86 87int VCMSessionInfo::Tl0PicId() const { 88 if (packets_.empty() || 89 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 90 return kNoTl0PicIdx; 91 return packets_.front().codecSpecificHeader.codecHeader.VP8.tl0PicIdx; 92} 93 94bool VCMSessionInfo::NonReference() const { 95 if (packets_.empty() || 96 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 97 return false; 98 return packets_.front().codecSpecificHeader.codecHeader.VP8.nonReference; 99} 100 101void VCMSessionInfo::Reset() { 102 session_nack_ = false; 103 complete_ = false; 104 decodable_ = false; 105 frame_type_ = kVideoFrameDelta; 106 packets_.clear(); 107 empty_seq_num_low_ = -1; 108 empty_seq_num_high_ = -1; 109 first_packet_seq_num_ = -1; 110 last_packet_seq_num_ = -1; 111} 112 113int VCMSessionInfo::SessionLength() const { 114 int length = 0; 115 for (PacketIteratorConst it = packets_.begin(); it != packets_.end(); ++it) 116 length += (*it).sizeBytes; 117 return length; 118} 119 120int VCMSessionInfo::NumPackets() const { 121 return packets_.size(); 122} 123 124int VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer, 125 PacketIterator packet_it) { 126 VCMPacket& packet = *packet_it; 127 PacketIterator it; 128 129 // Calculate the offset into the frame buffer for this packet. 130 int offset = 0; 131 for (it = packets_.begin(); it != packet_it; ++it) 132 offset += (*it).sizeBytes; 133 134 // Set the data pointer to pointing to the start of this packet in the 135 // frame buffer. 136 const uint8_t* packet_buffer = packet.dataPtr; 137 packet.dataPtr = frame_buffer + offset; 138 139 // We handle H.264 STAP-A packets in a special way as we need to remove the 140 // two length bytes between each NAL unit, and potentially add start codes. 141 const size_t kH264NALHeaderLengthInBytes = 1; 142 const size_t kLengthFieldLength = 2; 143 if (packet.codecSpecificHeader.codec == kRtpVideoH264 && 144 packet.codecSpecificHeader.codecHeader.H264.stap_a) { 145 size_t required_length = 0; 146 const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes; 147 while (nalu_ptr < packet_buffer + packet.sizeBytes) { 148 uint32_t length = BufferToUWord16(nalu_ptr); 149 required_length += 150 length + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); 151 nalu_ptr += kLengthFieldLength + length; 152 } 153 ShiftSubsequentPackets(packet_it, required_length); 154 nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes; 155 uint8_t* frame_buffer_ptr = frame_buffer + offset; 156 while (nalu_ptr < packet_buffer + packet.sizeBytes) { 157 uint32_t length = BufferToUWord16(nalu_ptr); 158 nalu_ptr += kLengthFieldLength; 159 frame_buffer_ptr += Insert(nalu_ptr, 160 length, 161 packet.insertStartCode, 162 const_cast<uint8_t*>(frame_buffer_ptr)); 163 nalu_ptr += length; 164 } 165 packet.sizeBytes = required_length; 166 return packet.sizeBytes; 167 } 168 ShiftSubsequentPackets( 169 packet_it, 170 packet.sizeBytes + 171 (packet.insertStartCode ? kH264StartCodeLengthBytes : 0)); 172 173 packet.sizeBytes = Insert(packet_buffer, 174 packet.sizeBytes, 175 packet.insertStartCode, 176 const_cast<uint8_t*>(packet.dataPtr)); 177 return packet.sizeBytes; 178} 179 180size_t VCMSessionInfo::Insert(const uint8_t* buffer, 181 size_t length, 182 bool insert_start_code, 183 uint8_t* frame_buffer) { 184 if (insert_start_code) { 185 const unsigned char startCode[] = {0, 0, 0, 1}; 186 memcpy(frame_buffer, startCode, kH264StartCodeLengthBytes); 187 } 188 memcpy(frame_buffer + (insert_start_code ? kH264StartCodeLengthBytes : 0), 189 buffer, 190 length); 191 length += (insert_start_code ? kH264StartCodeLengthBytes : 0); 192 193 return length; 194} 195 196void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it, 197 int steps_to_shift) { 198 ++it; 199 if (it == packets_.end()) 200 return; 201 uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr); 202 int shift_length = 0; 203 // Calculate the total move length and move the data pointers in advance. 204 for (; it != packets_.end(); ++it) { 205 shift_length += (*it).sizeBytes; 206 if ((*it).dataPtr != NULL) 207 (*it).dataPtr += steps_to_shift; 208 } 209 memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length); 210} 211 212void VCMSessionInfo::UpdateCompleteSession() { 213 if (HaveFirstPacket() && HaveLastPacket()) { 214 // Do we have all the packets in this session? 215 bool complete_session = true; 216 PacketIterator it = packets_.begin(); 217 PacketIterator prev_it = it; 218 ++it; 219 for (; it != packets_.end(); ++it) { 220 if (!InSequence(it, prev_it)) { 221 complete_session = false; 222 break; 223 } 224 prev_it = it; 225 } 226 complete_ = complete_session; 227 } 228} 229 230void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) { 231 // Irrelevant if session is already complete or decodable 232 if (complete_ || decodable_) 233 return; 234 // TODO(agalusza): Account for bursty loss. 235 // TODO(agalusza): Refine these values to better approximate optimal ones. 236 if (frame_data.rtt_ms < kRttThreshold 237 || frame_type_ == kVideoFrameKey 238 || !HaveFirstPacket() 239 || (NumPackets() <= kHighPacketPercentageThreshold 240 * frame_data.rolling_average_packets_per_frame 241 && NumPackets() > kLowPacketPercentageThreshold 242 * frame_data.rolling_average_packets_per_frame)) 243 return; 244 245 decodable_ = true; 246} 247 248bool VCMSessionInfo::complete() const { 249 return complete_; 250} 251 252bool VCMSessionInfo::decodable() const { 253 return decodable_; 254} 255 256// Find the end of the NAL unit which the packet pointed to by |packet_it| 257// belongs to. Returns an iterator to the last packet of the frame if the end 258// of the NAL unit wasn't found. 259VCMSessionInfo::PacketIterator VCMSessionInfo::FindNaluEnd( 260 PacketIterator packet_it) const { 261 if ((*packet_it).completeNALU == kNaluEnd || 262 (*packet_it).completeNALU == kNaluComplete) { 263 return packet_it; 264 } 265 // Find the end of the NAL unit. 266 for (; packet_it != packets_.end(); ++packet_it) { 267 if (((*packet_it).completeNALU == kNaluComplete && 268 (*packet_it).sizeBytes > 0) || 269 // Found next NALU. 270 (*packet_it).completeNALU == kNaluStart) 271 return --packet_it; 272 if ((*packet_it).completeNALU == kNaluEnd) 273 return packet_it; 274 } 275 // The end wasn't found. 276 return --packet_it; 277} 278 279int VCMSessionInfo::DeletePacketData(PacketIterator start, 280 PacketIterator end) { 281 int bytes_to_delete = 0; // The number of bytes to delete. 282 PacketIterator packet_after_end = end; 283 ++packet_after_end; 284 285 // Get the number of bytes to delete. 286 // Clear the size of these packets. 287 for (PacketIterator it = start; it != packet_after_end; ++it) { 288 bytes_to_delete += (*it).sizeBytes; 289 (*it).sizeBytes = 0; 290 (*it).dataPtr = NULL; 291 } 292 if (bytes_to_delete > 0) 293 ShiftSubsequentPackets(end, -bytes_to_delete); 294 return bytes_to_delete; 295} 296 297int VCMSessionInfo::BuildVP8FragmentationHeader( 298 uint8_t* frame_buffer, 299 int frame_buffer_length, 300 RTPFragmentationHeader* fragmentation) { 301 int new_length = 0; 302 // Allocate space for max number of partitions 303 fragmentation->VerifyAndAllocateFragmentationHeader(kMaxVP8Partitions); 304 fragmentation->fragmentationVectorSize = 0; 305 memset(fragmentation->fragmentationLength, 0, 306 kMaxVP8Partitions * sizeof(uint32_t)); 307 if (packets_.empty()) 308 return new_length; 309 PacketIterator it = FindNextPartitionBeginning(packets_.begin()); 310 while (it != packets_.end()) { 311 const int partition_id = 312 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 313 PacketIterator partition_end = FindPartitionEnd(it); 314 fragmentation->fragmentationOffset[partition_id] = 315 (*it).dataPtr - frame_buffer; 316 assert(fragmentation->fragmentationOffset[partition_id] < 317 static_cast<uint32_t>(frame_buffer_length)); 318 fragmentation->fragmentationLength[partition_id] = 319 (*partition_end).dataPtr + (*partition_end).sizeBytes - (*it).dataPtr; 320 assert(fragmentation->fragmentationLength[partition_id] <= 321 static_cast<uint32_t>(frame_buffer_length)); 322 new_length += fragmentation->fragmentationLength[partition_id]; 323 ++partition_end; 324 it = FindNextPartitionBeginning(partition_end); 325 if (partition_id + 1 > fragmentation->fragmentationVectorSize) 326 fragmentation->fragmentationVectorSize = partition_id + 1; 327 } 328 // Set all empty fragments to start where the previous fragment ends, 329 // and have zero length. 330 if (fragmentation->fragmentationLength[0] == 0) 331 fragmentation->fragmentationOffset[0] = 0; 332 for (int i = 1; i < fragmentation->fragmentationVectorSize; ++i) { 333 if (fragmentation->fragmentationLength[i] == 0) 334 fragmentation->fragmentationOffset[i] = 335 fragmentation->fragmentationOffset[i - 1] + 336 fragmentation->fragmentationLength[i - 1]; 337 assert(i == 0 || 338 fragmentation->fragmentationOffset[i] >= 339 fragmentation->fragmentationOffset[i - 1]); 340 } 341 assert(new_length <= frame_buffer_length); 342 return new_length; 343} 344 345VCMSessionInfo::PacketIterator VCMSessionInfo::FindNextPartitionBeginning( 346 PacketIterator it) const { 347 while (it != packets_.end()) { 348 if ((*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition) { 349 return it; 350 } 351 ++it; 352 } 353 return it; 354} 355 356VCMSessionInfo::PacketIterator VCMSessionInfo::FindPartitionEnd( 357 PacketIterator it) const { 358 assert((*it).codec == kVideoCodecVP8); 359 PacketIterator prev_it = it; 360 const int partition_id = 361 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 362 while (it != packets_.end()) { 363 bool beginning = 364 (*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition; 365 int current_partition_id = 366 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 367 bool packet_loss_found = (!beginning && !InSequence(it, prev_it)); 368 if (packet_loss_found || 369 (beginning && current_partition_id != partition_id)) { 370 // Missing packet, the previous packet was the last in sequence. 371 return prev_it; 372 } 373 prev_it = it; 374 ++it; 375 } 376 return prev_it; 377} 378 379bool VCMSessionInfo::InSequence(const PacketIterator& packet_it, 380 const PacketIterator& prev_packet_it) { 381 // If the two iterators are pointing to the same packet they are considered 382 // to be in sequence. 383 return (packet_it == prev_packet_it || 384 (static_cast<uint16_t>((*prev_packet_it).seqNum + 1) == 385 (*packet_it).seqNum)); 386} 387 388int VCMSessionInfo::MakeDecodable() { 389 int return_length = 0; 390 if (packets_.empty()) { 391 return 0; 392 } 393 PacketIterator it = packets_.begin(); 394 // Make sure we remove the first NAL unit if it's not decodable. 395 if ((*it).completeNALU == kNaluIncomplete || 396 (*it).completeNALU == kNaluEnd) { 397 PacketIterator nalu_end = FindNaluEnd(it); 398 return_length += DeletePacketData(it, nalu_end); 399 it = nalu_end; 400 } 401 PacketIterator prev_it = it; 402 // Take care of the rest of the NAL units. 403 for (; it != packets_.end(); ++it) { 404 bool start_of_nalu = ((*it).completeNALU == kNaluStart || 405 (*it).completeNALU == kNaluComplete); 406 if (!start_of_nalu && !InSequence(it, prev_it)) { 407 // Found a sequence number gap due to packet loss. 408 PacketIterator nalu_end = FindNaluEnd(it); 409 return_length += DeletePacketData(it, nalu_end); 410 it = nalu_end; 411 } 412 prev_it = it; 413 } 414 return return_length; 415} 416 417void VCMSessionInfo::SetNotDecodableIfIncomplete() { 418 // We don't need to check for completeness first because the two are 419 // orthogonal. If complete_ is true, decodable_ is irrelevant. 420 decodable_ = false; 421} 422 423bool 424VCMSessionInfo::HaveFirstPacket() const { 425 return !packets_.empty() && (first_packet_seq_num_ != -1); 426} 427 428bool 429VCMSessionInfo::HaveLastPacket() const { 430 return !packets_.empty() && (last_packet_seq_num_ != -1); 431} 432 433bool 434VCMSessionInfo::session_nack() const { 435 return session_nack_; 436} 437 438int VCMSessionInfo::InsertPacket(const VCMPacket& packet, 439 uint8_t* frame_buffer, 440 VCMDecodeErrorMode decode_error_mode, 441 const FrameData& frame_data) { 442 if (packet.frameType == kFrameEmpty) { 443 // Update sequence number of an empty packet. 444 // Only media packets are inserted into the packet list. 445 InformOfEmptyPacket(packet.seqNum); 446 return 0; 447 } 448 449 if (packets_.size() == kMaxPacketsInSession) { 450 LOG(LS_ERROR) << "Max number of packets per frame has been reached."; 451 return -1; 452 } 453 454 // Find the position of this packet in the packet list in sequence number 455 // order and insert it. Loop over the list in reverse order. 456 ReversePacketIterator rit = packets_.rbegin(); 457 for (; rit != packets_.rend(); ++rit) 458 if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum) 459 break; 460 461 // Check for duplicate packets. 462 if (rit != packets_.rend() && 463 (*rit).seqNum == packet.seqNum && (*rit).sizeBytes > 0) 464 return -2; 465 466 if (packet.codec == kVideoCodecH264) { 467 frame_type_ = packet.frameType; 468 if (packet.isFirstPacket && 469 (first_packet_seq_num_ == -1 || 470 IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum))) { 471 first_packet_seq_num_ = packet.seqNum; 472 } 473 if (packet.markerBit && 474 (last_packet_seq_num_ == -1 || 475 IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_))) { 476 last_packet_seq_num_ = packet.seqNum; 477 } 478 } else { 479 // Only insert media packets between first and last packets (when 480 // available). 481 // Placing check here, as to properly account for duplicate packets. 482 // Check if this is first packet (only valid for some codecs) 483 // Should only be set for one packet per session. 484 if (packet.isFirstPacket && first_packet_seq_num_ == -1) { 485 // The first packet in a frame signals the frame type. 486 frame_type_ = packet.frameType; 487 // Store the sequence number for the first packet. 488 first_packet_seq_num_ = static_cast<int>(packet.seqNum); 489 } else if (first_packet_seq_num_ != -1 && 490 !IsNewerSequenceNumber(packet.seqNum, first_packet_seq_num_)) { 491 LOG(LS_WARNING) << "Received packet with a sequence number which is out " 492 "of frame boundaries"; 493 return -3; 494 } else if (frame_type_ == kFrameEmpty && packet.frameType != kFrameEmpty) { 495 // Update the frame type with the type of the first media packet. 496 // TODO(mikhal): Can this trigger? 497 frame_type_ = packet.frameType; 498 } 499 500 // Track the marker bit, should only be set for one packet per session. 501 if (packet.markerBit && last_packet_seq_num_ == -1) { 502 last_packet_seq_num_ = static_cast<int>(packet.seqNum); 503 } else if (last_packet_seq_num_ != -1 && 504 IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_)) { 505 LOG(LS_WARNING) << "Received packet with a sequence number which is out " 506 "of frame boundaries"; 507 return -3; 508 } 509 } 510 511 // The insert operation invalidates the iterator |rit|. 512 PacketIterator packet_list_it = packets_.insert(rit.base(), packet); 513 514 int returnLength = InsertBuffer(frame_buffer, packet_list_it); 515 UpdateCompleteSession(); 516 if (decode_error_mode == kWithErrors) 517 decodable_ = true; 518 else if (decode_error_mode == kSelectiveErrors) 519 UpdateDecodableSession(frame_data); 520 return returnLength; 521} 522 523void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) { 524 // Empty packets may be FEC or filler packets. They are sequential and 525 // follow the data packets, therefore, we should only keep track of the high 526 // and low sequence numbers and may assume that the packets in between are 527 // empty packets belonging to the same frame (timestamp). 528 if (empty_seq_num_high_ == -1) 529 empty_seq_num_high_ = seq_num; 530 else 531 empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_); 532 if (empty_seq_num_low_ == -1 || IsNewerSequenceNumber(empty_seq_num_low_, 533 seq_num)) 534 empty_seq_num_low_ = seq_num; 535} 536 537} // namespace webrtc 538