1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
12#define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
13
14#include <map>
15#include <vector>
16
17#include "webrtc/video_engine/include/vie_codec.h"
18#include "webrtc/video_engine/include/vie_image_process.h"
19#include "webrtc/video_engine/test/auto_test/interface/vie_autotest_defines.h"
20#include "webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h"
21
22class FrameDropDetector;
23struct NetworkParameters;
24class TbInterfaces;
25
26// Initializes the Video engine and its components, runs video playback using
27// for KAutoTestSleepTimeMs milliseconds, then shuts down everything.
28// The bit rate and packet loss parameters should be configured so that
29// frames are dropped, in order to test the frame drop detection that is
30// performed by the FrameDropDetector class.
31void TestFullStack(const TbInterfaces& interfaces,
32                   int capture_id,
33                   int video_channel,
34                   int width,
35                   int height,
36                   int bit_rate_kbps,
37                   const NetworkParameters& network,
38                   FrameDropDetector* frame_drop_detector,
39                   ViEToFileRenderer* remote_file_renderer,
40                   ViEToFileRenderer* local_file_renderer);
41
42// A frame in a video file. The four different points in the stack when
43// register the frame state are (in time order): created, transmitted, decoded,
44// rendered.
45class Frame {
46 public:
47  Frame(int number, unsigned int timestamp)
48    : number_(number),
49      frame_timestamp_(timestamp),
50      created_timestamp_in_us_(-1),
51      sent_timestamp_in_us_(-1),
52      received_timestamp_in_us_(-1),
53      decoded_timestamp_in_us_(-1),
54      rendered_timestamp_in_us_(-1),
55      dropped_at_send(false),
56      dropped_at_receive(false),
57      dropped_at_decode(false),
58      dropped_at_render(false) {}
59
60  // Frame number, starting at 0.
61  int number_;
62
63  // Frame timestamp, that is used by Video Engine and RTP headers and set when
64  // the frame is sent into the stack.
65  unsigned int frame_timestamp_;
66
67  // Timestamps for our measurements of when the frame is in different states.
68  int64_t created_timestamp_in_us_;
69  int64_t sent_timestamp_in_us_;
70  int64_t received_timestamp_in_us_;
71  int64_t decoded_timestamp_in_us_;
72  int64_t rendered_timestamp_in_us_;
73
74  // Where the frame was dropped (more than one may be true).
75  bool dropped_at_send;
76  bool dropped_at_receive;
77  bool dropped_at_decode;
78  bool dropped_at_render;
79};
80
81// Fixes the output file by copying the last successful frame into the place
82// where the dropped frame would be, for all dropped frames (if any).
83// This method will not be able to fix data for the first frame if that is
84// dropped, since there'll be no previous frame to copy. This case should never
85// happen because of encoder frame dropping at least.
86// Parameters:
87//    output_file            The output file to modify (pad with frame copies
88//                           for all dropped frames)
89//    frame_length_in_bytes  Byte length of each frame.
90//    frames                 A vector of all Frame objects. Must be sorted by
91//                           frame number. If empty this method will do nothing.
92void FixOutputFileForComparison(const std::string& output_file,
93                                int frame_length_in_bytes,
94                                const std::vector<Frame*>& frames);
95
96// Handles statistics about dropped frames. Frames travel through the stack
97// with different timestamps. The frames created and sent to the encoder have
98// one timestamp on the sending side while the decoded/rendered frames have
99// another timestamp on the receiving side. The difference between these
100// timestamps is fixed, which we can use to identify the frames when they
101// arrive, since the FrameDropDetector class gets data reported from both sides.
102// The four different points in the stack when this class examines the frame
103// states are (in time order): created, sent, received, decoded, rendered.
104//
105// The flow can be visualized like this:
106//
107//         Created        Sent        Received               Decoded   Rendered
108// +-------+  |  +-------+ | +---------+ | +------+  +-------+  |  +--------+
109// |Capture|  |  |Encoder| | |  Ext.   | | |Jitter|  |Decoder|  |  |  Ext.  |
110// | device|---->|       |-->|transport|-->|buffer|->|       |---->|renderer|
111// +-------+     +-------+   +---------+   +------+  +-------+     +--------+
112//
113// This class has no intention of being thread-safe.
114class FrameDropDetector {
115 public:
116  enum State {
117    // A frame being created, i.e. sent to the encoder; the first step of
118    // a frame's life cycle. This timestamp becomes the frame timestamp in the
119    // Frame objects.
120    kCreated,
121    // A frame being sent in external transport (to the simulated network). This
122    // timestamp differs from the one in the Created state by a constant diff.
123    kSent,
124    // A frame being received in external transport (from the simulated
125    // network). This timestamp differs from the one in the Created state by a
126    // constant diff.
127    kReceived,
128    // A frame that has been decoded in the decoder. This timestamp differs
129    // from the one in the Created state by a constant diff.
130    kDecoded,
131    // A frame that has been rendered; the last step of a frame's life cycle.
132    // This timestamp differs from the one in the Created state by a constant
133    // diff.
134    kRendered
135  };
136
137  FrameDropDetector()
138      : dirty_(true),
139        dropped_frames_at_send_(0),
140        dropped_frames_at_receive_(0),
141        dropped_frames_at_decode_(0),
142        dropped_frames_at_render_(0),
143        num_created_frames_(0),
144        num_sent_frames_(0),
145        num_received_frames_(0),
146        num_decoded_frames_(0),
147        num_rendered_frames_(0),
148        timestamp_diff_(0) {}
149
150  // Reports a frame has reached a state in the frame life cycle.
151  void ReportFrameState(State state, unsigned int timestamp,
152                        int64_t report_time_us);
153
154  // Uses all the gathered timestamp information to calculate which frames have
155  // been dropped during the test and where they were dropped. Not until
156  // this method has been executed, the Frame objects will have all fields
157  // filled with the proper timestamp information.
158  void CalculateResults();
159
160  // Calculates the number of frames have been registered as dropped at the
161  // specified state of the frame life cycle.
162  // CalculateResults() must be called before calling this method.
163  int GetNumberOfFramesDroppedAt(State state);
164
165  // Gets a vector of all the created frames.
166  // CalculateResults() must be called before calling this method to have all
167  // fields of the Frame objects to represent the current state.
168  const std::vector<Frame*>& GetAllFrames();
169
170  // Prints a detailed report about all the different frame states and which
171  // ones are detected as dropped, using ViETest::Log. Also prints
172  // perf-formatted output and adds |test_label| as a modifier to the perf
173  // output.
174  // CalculateResults() must be called before calling this method.
175  void PrintReport(const std::string& test_label);
176
177  // Prints all the timestamp maps. Mainly used for debugging purposes to find
178  // missing timestamps.
179  void PrintDebugDump();
180 private:
181  // Will be false until CalculateResults() is called. Switches to true
182  // as soon as new timestamps are reported using ReportFrameState().
183  bool dirty_;
184
185  // Map of frame creation timestamps to all Frame objects.
186  std::map<unsigned int, Frame*> created_frames_;
187
188  // Maps converted frame timestamps (differ from creation timestamp) to the
189  // time they arrived in the different states of the frame's life cycle.
190  std::map<unsigned int, int64_t> sent_frames_;
191  std::map<unsigned int, int64_t> received_frames_;
192  std::map<unsigned int, int64_t> decoded_frames_;
193  std::map<unsigned int, int64_t> rendered_frames_;
194
195  // A vector with the frames sorted in their created order.
196  std::vector<Frame*> created_frames_vector_;
197
198  // Statistics.
199  int dropped_frames_at_send_;
200  int dropped_frames_at_receive_;
201  int dropped_frames_at_decode_;
202  int dropped_frames_at_render_;
203
204  int num_created_frames_;
205  int num_sent_frames_;
206  int num_received_frames_;
207  int num_decoded_frames_;
208  int num_rendered_frames_;
209
210  // The constant diff between the created and transmitted frames, since their
211  // timestamps are converted.
212  unsigned int timestamp_diff_;
213};
214
215// Tracks which frames are received on the remote side and reports back to the
216// FrameDropDetector class when they are rendered.
217class FrameDropMonitoringRemoteFileRenderer : public ViEToFileRenderer {
218 public:
219  explicit FrameDropMonitoringRemoteFileRenderer(
220      FrameDropDetector* frame_drop_detector)
221      : frame_drop_detector_(frame_drop_detector) {}
222  virtual ~FrameDropMonitoringRemoteFileRenderer() {}
223
224  // Implementation of ExternalRenderer:
225  int FrameSizeChange(unsigned int width, unsigned int height,
226                      unsigned int number_of_streams) OVERRIDE;
227  int DeliverFrame(unsigned char* buffer, int buffer_size,
228                   uint32_t time_stamp,
229                   int64_t ntp_time_ms,
230                   int64_t render_time,
231                   void* handle) OVERRIDE;
232 private:
233  FrameDropDetector* frame_drop_detector_;
234};
235
236#endif  // WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
237