1/*
2 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/test/rtp_file_reader.h"
12
13#include <stdio.h>
14
15#include <map>
16#include <string>
17#include <vector>
18
19#include "webrtc/base/checks.h"
20#include "webrtc/base/format_macros.h"
21#include "webrtc/base/scoped_ptr.h"
22#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
23
24namespace webrtc {
25namespace test {
26
27static const size_t kFirstLineLength = 40;
28static uint16_t kPacketHeaderSize = 8;
29
30#if 1
31# define DEBUG_LOG(text)
32# define DEBUG_LOG1(text, arg)
33#else
34# define DEBUG_LOG(text) (printf(text "\n"))
35# define DEBUG_LOG1(text, arg) (printf(text "\n", arg))
36#endif
37
38#define TRY(expr)                                      \
39  do {                                                 \
40    if (!(expr)) {                                     \
41      DEBUG_LOG1("FAIL at " __FILE__ ":%d", __LINE__); \
42      return false;                                    \
43    }                                                  \
44  } while (0)
45
46bool ReadUint32(uint32_t* out, FILE* file) {
47  *out = 0;
48  for (size_t i = 0; i < 4; ++i) {
49    *out <<= 8;
50    uint8_t tmp;
51    if (fread(&tmp, 1, sizeof(uint8_t), file) != sizeof(uint8_t))
52      return false;
53    *out |= tmp;
54  }
55  return true;
56}
57
58bool ReadUint16(uint16_t* out, FILE* file) {
59  *out = 0;
60  for (size_t i = 0; i < 2; ++i) {
61    *out <<= 8;
62    uint8_t tmp;
63    if (fread(&tmp, 1, sizeof(uint8_t), file) != sizeof(uint8_t))
64      return false;
65    *out |= tmp;
66  }
67  return true;
68}
69
70class RtpFileReaderImpl : public RtpFileReader {
71 public:
72  virtual bool Init(const std::string& filename,
73                    const std::set<uint32_t>& ssrc_filter) = 0;
74};
75
76class InterleavedRtpFileReader : public RtpFileReaderImpl {
77 public:
78  virtual ~InterleavedRtpFileReader() {
79    if (file_ != NULL) {
80      fclose(file_);
81      file_ = NULL;
82    }
83  }
84
85  virtual bool Init(const std::string& filename,
86                    const std::set<uint32_t>& ssrc_filter) {
87    file_ = fopen(filename.c_str(), "rb");
88    if (file_ == NULL) {
89      printf("ERROR: Can't open file: %s\n", filename.c_str());
90      return false;
91    }
92    return true;
93  }
94  virtual bool NextPacket(RtpPacket* packet) {
95    assert(file_ != NULL);
96    packet->length = RtpPacket::kMaxPacketBufferSize;
97    uint32_t len = 0;
98    TRY(ReadUint32(&len, file_));
99    if (packet->length < len) {
100      FATAL() << "Packet is too large to fit: " << len << " bytes vs "
101              << packet->length
102              << " bytes allocated. Consider increasing the buffer "
103                 "size";
104    }
105    if (fread(packet->data, 1, len, file_) != len)
106      return false;
107
108    packet->length = len;
109    packet->original_length = len;
110    packet->time_ms = time_ms_;
111    time_ms_ += 5;
112    return true;
113  }
114
115 private:
116  FILE* file_ = NULL;
117  int64_t time_ms_ = 0;
118};
119
120// Read RTP packets from file in rtpdump format, as documented at:
121// http://www.cs.columbia.edu/irt/software/rtptools/
122class RtpDumpReader : public RtpFileReaderImpl {
123 public:
124  RtpDumpReader() : file_(NULL) {}
125  virtual ~RtpDumpReader() {
126    if (file_ != NULL) {
127      fclose(file_);
128      file_ = NULL;
129    }
130  }
131
132  bool Init(const std::string& filename,
133            const std::set<uint32_t>& ssrc_filter) {
134    file_ = fopen(filename.c_str(), "rb");
135    if (file_ == NULL) {
136      printf("ERROR: Can't open file: %s\n", filename.c_str());
137      return false;
138    }
139
140    char firstline[kFirstLineLength + 1] = {0};
141    if (fgets(firstline, kFirstLineLength, file_) == NULL) {
142      DEBUG_LOG("ERROR: Can't read from file\n");
143      return false;
144    }
145    if (strncmp(firstline, "#!rtpplay", 9) == 0) {
146      if (strncmp(firstline, "#!rtpplay1.0", 12) != 0) {
147        DEBUG_LOG("ERROR: wrong rtpplay version, must be 1.0\n");
148        return false;
149      }
150    } else if (strncmp(firstline, "#!RTPencode", 11) == 0) {
151      if (strncmp(firstline, "#!RTPencode1.0", 14) != 0) {
152        DEBUG_LOG("ERROR: wrong RTPencode version, must be 1.0\n");
153        return false;
154      }
155    } else {
156      DEBUG_LOG("ERROR: wrong file format of input file\n");
157      return false;
158    }
159
160    uint32_t start_sec;
161    uint32_t start_usec;
162    uint32_t source;
163    uint16_t port;
164    uint16_t padding;
165    TRY(ReadUint32(&start_sec, file_));
166    TRY(ReadUint32(&start_usec, file_));
167    TRY(ReadUint32(&source, file_));
168    TRY(ReadUint16(&port, file_));
169    TRY(ReadUint16(&padding, file_));
170
171    return true;
172  }
173
174  bool NextPacket(RtpPacket* packet) override {
175    uint8_t* rtp_data = packet->data;
176    packet->length = RtpPacket::kMaxPacketBufferSize;
177
178    uint16_t len;
179    uint16_t plen;
180    uint32_t offset;
181    TRY(ReadUint16(&len, file_));
182    TRY(ReadUint16(&plen, file_));
183    TRY(ReadUint32(&offset, file_));
184
185    // Use 'len' here because a 'plen' of 0 specifies rtcp.
186    len -= kPacketHeaderSize;
187    if (packet->length < len) {
188      FATAL() << "Packet is too large to fit: " << len << " bytes vs "
189              << packet->length
190              << " bytes allocated. Consider increasing the buffer "
191                 "size";
192    }
193    if (fread(rtp_data, 1, len, file_) != len) {
194      return false;
195    }
196
197    packet->length = len;
198    packet->original_length = plen;
199    packet->time_ms = offset;
200    return true;
201  }
202
203 private:
204  FILE* file_;
205
206  RTC_DISALLOW_COPY_AND_ASSIGN(RtpDumpReader);
207};
208
209enum {
210  kResultFail = -1,
211  kResultSuccess = 0,
212  kResultSkip = 1,
213
214  kPcapVersionMajor = 2,
215  kPcapVersionMinor = 4,
216  kLinktypeNull = 0,
217  kLinktypeEthernet = 1,
218  kBsdNullLoopback1 = 0x00000002,
219  kBsdNullLoopback2 = 0x02000000,
220  kEthernetIIHeaderMacSkip = 12,
221  kEthertypeIp = 0x0800,
222  kIpVersion4 = 4,
223  kMinIpHeaderLength = 20,
224  kFragmentOffsetClear = 0x0000,
225  kFragmentOffsetDoNotFragment = 0x4000,
226  kProtocolTcp = 0x06,
227  kProtocolUdp = 0x11,
228  kUdpHeaderLength = 8,
229  kMaxReadBufferSize = 4096
230};
231
232const uint32_t kPcapBOMSwapOrder = 0xd4c3b2a1UL;
233const uint32_t kPcapBOMNoSwapOrder = 0xa1b2c3d4UL;
234
235#define TRY_PCAP(expr)                                 \
236  do {                                                 \
237    int r = (expr);                                    \
238    if (r == kResultFail) {                            \
239      DEBUG_LOG1("FAIL at " __FILE__ ":%d", __LINE__); \
240      return kResultFail;                              \
241    } else if (r == kResultSkip) {                     \
242      return kResultSkip;                              \
243    }                                                  \
244  } while (0)
245
246// Read RTP packets from file in tcpdump/libpcap format, as documented at:
247// http://wiki.wireshark.org/Development/LibpcapFileFormat
248class PcapReader : public RtpFileReaderImpl {
249 public:
250  PcapReader()
251    : file_(NULL),
252      swap_pcap_byte_order_(false),
253#ifdef WEBRTC_ARCH_BIG_ENDIAN
254      swap_network_byte_order_(false),
255#else
256      swap_network_byte_order_(true),
257#endif
258      read_buffer_(),
259      packets_by_ssrc_(),
260      packets_(),
261      next_packet_it_() {
262  }
263
264  virtual ~PcapReader() {
265    if (file_ != NULL) {
266      fclose(file_);
267      file_ = NULL;
268    }
269  }
270
271  bool Init(const std::string& filename,
272            const std::set<uint32_t>& ssrc_filter) override {
273    return Initialize(filename, ssrc_filter) == kResultSuccess;
274  }
275
276  int Initialize(const std::string& filename,
277                 const std::set<uint32_t>& ssrc_filter) {
278    file_ = fopen(filename.c_str(), "rb");
279    if (file_ == NULL) {
280      printf("ERROR: Can't open file: %s\n", filename.c_str());
281      return kResultFail;
282    }
283
284    if (ReadGlobalHeader() < 0) {
285      return kResultFail;
286    }
287
288    int total_packet_count = 0;
289    uint32_t stream_start_ms = 0;
290    int32_t next_packet_pos = ftell(file_);
291    for (;;) {
292      TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET));
293      int result = ReadPacket(&next_packet_pos, stream_start_ms,
294                              ++total_packet_count, ssrc_filter);
295      if (result == kResultFail) {
296        break;
297      } else if (result == kResultSuccess && packets_.size() == 1) {
298        assert(stream_start_ms == 0);
299        PacketIterator it = packets_.begin();
300        stream_start_ms = it->time_offset_ms;
301        it->time_offset_ms = 0;
302      }
303    }
304
305    if (feof(file_) == 0) {
306      printf("Failed reading file!\n");
307      return kResultFail;
308    }
309
310    printf("Total packets in file: %d\n", total_packet_count);
311    printf("Total RTP/RTCP packets: %" PRIuS "\n", packets_.size());
312
313    for (SsrcMapIterator mit = packets_by_ssrc_.begin();
314        mit != packets_by_ssrc_.end(); ++mit) {
315      uint32_t ssrc = mit->first;
316      const std::vector<uint32_t>& packet_indices = mit->second;
317      uint8_t pt = packets_[packet_indices[0]].rtp_header.payloadType;
318      printf("SSRC: %08x, %" PRIuS " packets, pt=%d\n", ssrc,
319             packet_indices.size(), pt);
320    }
321
322    // TODO(solenberg): Better validation of identified SSRC streams.
323    //
324    // Since we're dealing with raw network data here, we will wrongly identify
325    // some packets as RTP. When these packets are consumed by RtpPlayer, they
326    // are unlikely to cause issues as they will ultimately be filtered out by
327    // the RtpRtcp module. However, we should really do better filtering here,
328    // which we can accomplish in a number of ways, e.g.:
329    //
330    // - Verify that the time stamps and sequence numbers for RTP packets are
331    //   both increasing/decreasing. If they move in different directions, the
332    //   SSRC is likely bogus and can be dropped. (Normally they should be inc-
333    //   reasing but we must allow packet reordering).
334    // - If RTP sequence number is not changing, drop the stream.
335    // - Can also use srcip:port->dstip:port pairs, assuming few SSRC collisions
336    //   for up/down streams.
337
338    next_packet_it_ = packets_.begin();
339    return kResultSuccess;
340  }
341
342  bool NextPacket(RtpPacket* packet) override {
343    uint32_t length = RtpPacket::kMaxPacketBufferSize;
344    if (NextPcap(packet->data, &length, &packet->time_ms) != kResultSuccess)
345      return false;
346    packet->length = static_cast<size_t>(length);
347    packet->original_length = packet->length;
348    return true;
349  }
350
351  virtual int NextPcap(uint8_t* data, uint32_t* length, uint32_t* time_ms) {
352    assert(data);
353    assert(length);
354    assert(time_ms);
355
356    if (next_packet_it_ == packets_.end()) {
357      return -1;
358    }
359    if (*length < next_packet_it_->payload_length) {
360      return -1;
361    }
362    TRY_PCAP(fseek(file_, next_packet_it_->pos_in_file, SEEK_SET));
363    TRY_PCAP(Read(data, next_packet_it_->payload_length));
364    *length = next_packet_it_->payload_length;
365    *time_ms = next_packet_it_->time_offset_ms;
366    next_packet_it_++;
367
368    return 0;
369  }
370
371 private:
372  // A marker of an RTP packet within the file.
373  struct RtpPacketMarker {
374    uint32_t packet_number;   // One-based index (like in WireShark)
375    uint32_t time_offset_ms;
376    uint32_t source_ip;
377    uint32_t dest_ip;
378    uint16_t source_port;
379    uint16_t dest_port;
380    RTPHeader rtp_header;
381    int32_t pos_in_file;      // Byte offset of payload from start of file.
382    uint32_t payload_length;
383  };
384
385  typedef std::vector<RtpPacketMarker>::iterator PacketIterator;
386  typedef std::map<uint32_t, std::vector<uint32_t> > SsrcMap;
387  typedef std::map<uint32_t, std::vector<uint32_t> >::iterator SsrcMapIterator;
388
389  int ReadGlobalHeader() {
390    uint32_t magic;
391    TRY_PCAP(Read(&magic, false));
392    if (magic == kPcapBOMSwapOrder) {
393      swap_pcap_byte_order_ = true;
394    } else if (magic == kPcapBOMNoSwapOrder) {
395      swap_pcap_byte_order_ = false;
396    } else {
397      return kResultFail;
398    }
399
400    uint16_t version_major;
401    uint16_t version_minor;
402    TRY_PCAP(Read(&version_major, false));
403    TRY_PCAP(Read(&version_minor, false));
404    if (version_major != kPcapVersionMajor ||
405        version_minor != kPcapVersionMinor) {
406      return kResultFail;
407    }
408
409    int32_t this_zone;  // GMT to local correction.
410    uint32_t sigfigs;   // Accuracy of timestamps.
411    uint32_t snaplen;   // Max length of captured packets, in octets.
412    uint32_t network;   // Data link type.
413    TRY_PCAP(Read(&this_zone, false));
414    TRY_PCAP(Read(&sigfigs, false));
415    TRY_PCAP(Read(&snaplen, false));
416    TRY_PCAP(Read(&network, false));
417
418    // Accept only LINKTYPE_NULL and LINKTYPE_ETHERNET.
419    // See: http://www.tcpdump.org/linktypes.html
420    if (network != kLinktypeNull && network != kLinktypeEthernet) {
421      return kResultFail;
422    }
423
424    return kResultSuccess;
425  }
426
427  int ReadPacket(int32_t* next_packet_pos,
428                 uint32_t stream_start_ms,
429                 uint32_t number,
430                 const std::set<uint32_t>& ssrc_filter) {
431    assert(next_packet_pos);
432
433    uint32_t ts_sec;    // Timestamp seconds.
434    uint32_t ts_usec;   // Timestamp microseconds.
435    uint32_t incl_len;  // Number of octets of packet saved in file.
436    uint32_t orig_len;  // Actual length of packet.
437    TRY_PCAP(Read(&ts_sec, false));
438    TRY_PCAP(Read(&ts_usec, false));
439    TRY_PCAP(Read(&incl_len, false));
440    TRY_PCAP(Read(&orig_len, false));
441
442    *next_packet_pos = ftell(file_) + incl_len;
443
444    RtpPacketMarker marker = {0};
445    marker.packet_number = number;
446    marker.time_offset_ms = CalcTimeDelta(ts_sec, ts_usec, stream_start_ms);
447    TRY_PCAP(ReadPacketHeader(&marker));
448    marker.pos_in_file = ftell(file_);
449
450    if (marker.payload_length > sizeof(read_buffer_)) {
451      printf("Packet too large!\n");
452      return kResultFail;
453    }
454    TRY_PCAP(Read(read_buffer_, marker.payload_length));
455
456    RtpUtility::RtpHeaderParser rtp_parser(read_buffer_, marker.payload_length);
457    if (rtp_parser.RTCP()) {
458      rtp_parser.ParseRtcp(&marker.rtp_header);
459      packets_.push_back(marker);
460    } else {
461      if (!rtp_parser.Parse(&marker.rtp_header, nullptr)) {
462        DEBUG_LOG("Not recognized as RTP/RTCP");
463        return kResultSkip;
464      }
465
466      uint32_t ssrc = marker.rtp_header.ssrc;
467      if (ssrc_filter.empty() || ssrc_filter.find(ssrc) != ssrc_filter.end()) {
468        packets_by_ssrc_[ssrc].push_back(
469            static_cast<uint32_t>(packets_.size()));
470        packets_.push_back(marker);
471      } else {
472        return kResultSkip;
473      }
474    }
475
476    return kResultSuccess;
477  }
478
479  int ReadPacketHeader(RtpPacketMarker* marker) {
480    int32_t file_pos = ftell(file_);
481
482    // Check for BSD null/loopback frame header. The header is just 4 bytes in
483    // native byte order, so we check for both versions as we don't care about
484    // the header as such and will likely fail reading the IP header if this is
485    // something else than null/loopback.
486    uint32_t protocol;
487    TRY_PCAP(Read(&protocol, true));
488    if (protocol == kBsdNullLoopback1 || protocol == kBsdNullLoopback2) {
489      int result = ReadXxpIpHeader(marker);
490      DEBUG_LOG("Recognized loopback frame");
491      if (result != kResultSkip) {
492        return result;
493      }
494    }
495
496    TRY_PCAP(fseek(file_, file_pos, SEEK_SET));
497
498    // Check for Ethernet II, IP frame header.
499    uint16_t type;
500    TRY_PCAP(Skip(kEthernetIIHeaderMacSkip));  // Source+destination MAC.
501    TRY_PCAP(Read(&type, true));
502    if (type == kEthertypeIp) {
503      int result = ReadXxpIpHeader(marker);
504      DEBUG_LOG("Recognized ethernet 2 frame");
505      if (result != kResultSkip) {
506        return result;
507      }
508    }
509
510    return kResultSkip;
511  }
512
513  uint32_t CalcTimeDelta(uint32_t ts_sec, uint32_t ts_usec, uint32_t start_ms) {
514    // Round to nearest ms.
515    uint64_t t2_ms = ((static_cast<uint64_t>(ts_sec) * 1000000) + ts_usec +
516        500) / 1000;
517    uint64_t t1_ms = static_cast<uint64_t>(start_ms);
518    if (t2_ms < t1_ms) {
519      return 0;
520    } else {
521      return t2_ms - t1_ms;
522    }
523  }
524
525  int ReadXxpIpHeader(RtpPacketMarker* marker) {
526    assert(marker);
527
528    uint16_t version;
529    uint16_t length;
530    uint16_t id;
531    uint16_t fragment;
532    uint16_t protocol;
533    uint16_t checksum;
534    TRY_PCAP(Read(&version, true));
535    TRY_PCAP(Read(&length, true));
536    TRY_PCAP(Read(&id, true));
537    TRY_PCAP(Read(&fragment, true));
538    TRY_PCAP(Read(&protocol, true));
539    TRY_PCAP(Read(&checksum, true));
540    TRY_PCAP(Read(&marker->source_ip, true));
541    TRY_PCAP(Read(&marker->dest_ip, true));
542
543    if (((version >> 12) & 0x000f) != kIpVersion4) {
544      DEBUG_LOG("IP header is not IPv4");
545      return kResultSkip;
546    }
547
548    if (fragment != kFragmentOffsetClear &&
549        fragment != kFragmentOffsetDoNotFragment) {
550      DEBUG_LOG("IP fragments cannot be handled");
551      return kResultSkip;
552    }
553
554    // Skip remaining fields of IP header.
555    uint16_t header_length = (version & 0x0f00) >> (8 - 2);
556    assert(header_length >= kMinIpHeaderLength);
557    TRY_PCAP(Skip(header_length - kMinIpHeaderLength));
558
559    protocol = protocol & 0x00ff;
560    if (protocol == kProtocolTcp) {
561      DEBUG_LOG("TCP packets are not handled");
562      return kResultSkip;
563    } else if (protocol == kProtocolUdp) {
564      uint16_t length;
565      uint16_t checksum;
566      TRY_PCAP(Read(&marker->source_port, true));
567      TRY_PCAP(Read(&marker->dest_port, true));
568      TRY_PCAP(Read(&length, true));
569      TRY_PCAP(Read(&checksum, true));
570      marker->payload_length = length - kUdpHeaderLength;
571    } else {
572      DEBUG_LOG("Unknown transport (expected UDP or TCP)");
573      return kResultSkip;
574    }
575
576    return kResultSuccess;
577  }
578
579  int Read(uint32_t* out, bool expect_network_order) {
580    uint32_t tmp = 0;
581    if (fread(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) {
582      return kResultFail;
583    }
584    if ((!expect_network_order && swap_pcap_byte_order_) ||
585        (expect_network_order && swap_network_byte_order_)) {
586      tmp = ((tmp >> 24) & 0x000000ff) | (tmp << 24) |
587          ((tmp >> 8) & 0x0000ff00) | ((tmp << 8) & 0x00ff0000);
588    }
589    *out = tmp;
590    return kResultSuccess;
591  }
592
593  int Read(uint16_t* out, bool expect_network_order) {
594    uint16_t tmp = 0;
595    if (fread(&tmp, 1, sizeof(uint16_t), file_) != sizeof(uint16_t)) {
596      return kResultFail;
597    }
598    if ((!expect_network_order && swap_pcap_byte_order_) ||
599        (expect_network_order && swap_network_byte_order_)) {
600      tmp = ((tmp >> 8) & 0x00ff) | (tmp << 8);
601    }
602    *out = tmp;
603    return kResultSuccess;
604  }
605
606  int Read(uint8_t* out, uint32_t count) {
607    if (fread(out, 1, count, file_) != count) {
608      return kResultFail;
609    }
610    return kResultSuccess;
611  }
612
613  int Read(int32_t* out, bool expect_network_order) {
614    int32_t tmp = 0;
615    if (fread(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) {
616      return kResultFail;
617    }
618    if ((!expect_network_order && swap_pcap_byte_order_) ||
619        (expect_network_order && swap_network_byte_order_)) {
620      tmp = ((tmp >> 24) & 0x000000ff) | (tmp << 24) |
621          ((tmp >> 8) & 0x0000ff00) | ((tmp << 8) & 0x00ff0000);
622    }
623    *out = tmp;
624    return kResultSuccess;
625  }
626
627  int Skip(uint32_t length) {
628    if (fseek(file_, length, SEEK_CUR) != 0) {
629      return kResultFail;
630    }
631    return kResultSuccess;
632  }
633
634  FILE* file_;
635  bool swap_pcap_byte_order_;
636  const bool swap_network_byte_order_;
637  uint8_t read_buffer_[kMaxReadBufferSize];
638
639  SsrcMap packets_by_ssrc_;
640  std::vector<RtpPacketMarker> packets_;
641  PacketIterator next_packet_it_;
642
643  RTC_DISALLOW_COPY_AND_ASSIGN(PcapReader);
644};
645
646RtpFileReader* RtpFileReader::Create(FileFormat format,
647                                     const std::string& filename,
648                                     const std::set<uint32_t>& ssrc_filter) {
649  RtpFileReaderImpl* reader = NULL;
650  switch (format) {
651    case kPcap:
652      reader = new PcapReader();
653      break;
654    case kRtpDump:
655      reader = new RtpDumpReader();
656      break;
657    case kLengthPacketInterleaved:
658      reader = new InterleavedRtpFileReader();
659      break;
660  }
661  if (!reader->Init(filename, ssrc_filter)) {
662    delete reader;
663    return NULL;
664  }
665  return reader;
666}
667
668RtpFileReader* RtpFileReader::Create(FileFormat format,
669                                     const std::string& filename) {
670  return RtpFileReader::Create(format, filename, std::set<uint32_t>());
671}
672
673}  // namespace test
674}  // namespace webrtc
675