1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/video/overuse_frame_detector.h"
12
13#include <assert.h>
14#include <math.h>
15
16#include <algorithm>
17#include <list>
18#include <map>
19
20#include "webrtc/base/checks.h"
21#include "webrtc/base/exp_filter.h"
22#include "webrtc/base/logging.h"
23#include "webrtc/system_wrappers/include/clock.h"
24
25namespace webrtc {
26
27namespace {
28const int64_t kProcessIntervalMs = 5000;
29
30// Delay between consecutive rampups. (Used for quick recovery.)
31const int kQuickRampUpDelayMs = 10 * 1000;
32// Delay between rampup attempts. Initially uses standard, scales up to max.
33const int kStandardRampUpDelayMs = 40 * 1000;
34const int kMaxRampUpDelayMs = 240 * 1000;
35// Expontential back-off factor, to prevent annoying up-down behaviour.
36const double kRampUpBackoffFactor = 2.0;
37
38// Max number of overuses detected before always applying the rampup delay.
39const int kMaxOverusesBeforeApplyRampupDelay = 4;
40
41// The maximum exponent to use in VCMExpFilter.
42const float kSampleDiffMs = 33.0f;
43const float kMaxExp = 7.0f;
44
45}  // namespace
46
47// Class for calculating the processing usage on the send-side (the average
48// processing time of a frame divided by the average time difference between
49// captured frames).
50class OveruseFrameDetector::SendProcessingUsage {
51 public:
52  explicit SendProcessingUsage(const CpuOveruseOptions& options)
53      : kWeightFactorFrameDiff(0.998f),
54        kWeightFactorProcessing(0.995f),
55        kInitialSampleDiffMs(40.0f),
56        kMaxSampleDiffMs(45.0f),
57        count_(0),
58        options_(options),
59        filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)),
60        filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) {
61    Reset();
62  }
63  ~SendProcessingUsage() {}
64
65  void Reset() {
66    count_ = 0;
67    filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
68    filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
69    filtered_processing_ms_->Reset(kWeightFactorProcessing);
70    filtered_processing_ms_->Apply(1.0f, InitialProcessingMs());
71  }
72
73  void AddCaptureSample(float sample_ms) {
74    float exp = sample_ms / kSampleDiffMs;
75    exp = std::min(exp, kMaxExp);
76    filtered_frame_diff_ms_->Apply(exp, sample_ms);
77  }
78
79  void AddSample(float processing_ms, int64_t diff_last_sample_ms) {
80    ++count_;
81    float exp = diff_last_sample_ms / kSampleDiffMs;
82    exp = std::min(exp, kMaxExp);
83    filtered_processing_ms_->Apply(exp, processing_ms);
84  }
85
86  int Value() const {
87    if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
88      return static_cast<int>(InitialUsageInPercent() + 0.5f);
89    }
90    float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f);
91    frame_diff_ms = std::min(frame_diff_ms, kMaxSampleDiffMs);
92    float encode_usage_percent =
93        100.0f * filtered_processing_ms_->filtered() / frame_diff_ms;
94    return static_cast<int>(encode_usage_percent + 0.5);
95  }
96
97 private:
98  float InitialUsageInPercent() const {
99    // Start in between the underuse and overuse threshold.
100    return (options_.low_encode_usage_threshold_percent +
101            options_.high_encode_usage_threshold_percent) / 2.0f;
102  }
103
104  float InitialProcessingMs() const {
105    return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
106  }
107
108  const float kWeightFactorFrameDiff;
109  const float kWeightFactorProcessing;
110  const float kInitialSampleDiffMs;
111  const float kMaxSampleDiffMs;
112  uint64_t count_;
113  const CpuOveruseOptions options_;
114  rtc::scoped_ptr<rtc::ExpFilter> filtered_processing_ms_;
115  rtc::scoped_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
116};
117
118// Class for calculating the processing time of frames.
119class OveruseFrameDetector::FrameQueue {
120 public:
121  FrameQueue() : last_processing_time_ms_(-1) {}
122  ~FrameQueue() {}
123
124  // Called when a frame is captured.
125  // Starts the measuring of the processing time of the frame.
126  void Start(int64_t capture_time, int64_t now) {
127    const size_t kMaxSize = 90;  // Allows for processing time of 1.5s at 60fps.
128    if (frame_times_.size() > kMaxSize) {
129      LOG(LS_WARNING) << "Max size reached, removed oldest frame.";
130      frame_times_.erase(frame_times_.begin());
131    }
132    if (frame_times_.find(capture_time) != frame_times_.end()) {
133      // Frame should not exist.
134      assert(false);
135      return;
136    }
137    frame_times_[capture_time] = now;
138  }
139
140  // Called when the processing of a frame has finished.
141  // Returns the processing time of the frame.
142  int End(int64_t capture_time, int64_t now) {
143    std::map<int64_t, int64_t>::iterator it = frame_times_.find(capture_time);
144    if (it == frame_times_.end()) {
145      return -1;
146    }
147    // Remove any old frames up to current.
148    // Old frames have been skipped by the capture process thread.
149    // TODO(asapersson): Consider measuring time from first frame in list.
150    last_processing_time_ms_ = now - (*it).second;
151    frame_times_.erase(frame_times_.begin(), ++it);
152    return last_processing_time_ms_;
153  }
154
155  void Reset() { frame_times_.clear(); }
156  int NumFrames() const { return static_cast<int>(frame_times_.size()); }
157  int last_processing_time_ms() const { return last_processing_time_ms_; }
158
159 private:
160  // Captured frames mapped by the capture time.
161  std::map<int64_t, int64_t> frame_times_;
162  int last_processing_time_ms_;
163};
164
165
166OveruseFrameDetector::OveruseFrameDetector(
167    Clock* clock,
168    const CpuOveruseOptions& options,
169    CpuOveruseObserver* observer,
170    CpuOveruseMetricsObserver* metrics_observer)
171    : options_(options),
172      observer_(observer),
173      metrics_observer_(metrics_observer),
174      clock_(clock),
175      num_process_times_(0),
176      last_capture_time_(0),
177      num_pixels_(0),
178      next_process_time_(clock_->TimeInMilliseconds()),
179      last_overuse_time_(0),
180      checks_above_threshold_(0),
181      num_overuse_detections_(0),
182      last_rampup_time_(0),
183      in_quick_rampup_(false),
184      current_rampup_delay_ms_(kStandardRampUpDelayMs),
185      last_sample_time_ms_(0),
186      usage_(new SendProcessingUsage(options)),
187      frame_queue_(new FrameQueue()) {
188  RTC_DCHECK(metrics_observer != nullptr);
189  // Make sure stats are initially up-to-date. This simplifies unit testing
190  // since we don't have to trigger an update using one of the methods which
191  // would also alter the overuse state.
192  UpdateCpuOveruseMetrics();
193  processing_thread_.DetachFromThread();
194}
195
196OveruseFrameDetector::~OveruseFrameDetector() {
197}
198
199int OveruseFrameDetector::LastProcessingTimeMs() const {
200  rtc::CritScope cs(&crit_);
201  return frame_queue_->last_processing_time_ms();
202}
203
204int OveruseFrameDetector::FramesInQueue() const {
205  rtc::CritScope cs(&crit_);
206  return frame_queue_->NumFrames();
207}
208
209void OveruseFrameDetector::UpdateCpuOveruseMetrics() {
210  metrics_.encode_usage_percent = usage_->Value();
211
212  metrics_observer_->CpuOveruseMetricsUpdated(metrics_);
213}
214
215int64_t OveruseFrameDetector::TimeUntilNextProcess() {
216  RTC_DCHECK(processing_thread_.CalledOnValidThread());
217  return next_process_time_ - clock_->TimeInMilliseconds();
218}
219
220bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
221  if (num_pixels != num_pixels_) {
222    return true;
223  }
224  return false;
225}
226
227bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const {
228  if (last_capture_time_ == 0) {
229    return false;
230  }
231  return (now - last_capture_time_) > options_.frame_timeout_interval_ms;
232}
233
234void OveruseFrameDetector::ResetAll(int num_pixels) {
235  num_pixels_ = num_pixels;
236  usage_->Reset();
237  frame_queue_->Reset();
238  last_capture_time_ = 0;
239  num_process_times_ = 0;
240  UpdateCpuOveruseMetrics();
241}
242
243void OveruseFrameDetector::FrameCaptured(int width,
244                                         int height,
245                                         int64_t capture_time_ms) {
246  rtc::CritScope cs(&crit_);
247
248  int64_t now = clock_->TimeInMilliseconds();
249  if (FrameSizeChanged(width * height) || FrameTimeoutDetected(now)) {
250    ResetAll(width * height);
251  }
252
253  if (last_capture_time_ != 0)
254    usage_->AddCaptureSample(now - last_capture_time_);
255
256  last_capture_time_ = now;
257
258  frame_queue_->Start(capture_time_ms, now);
259}
260
261void OveruseFrameDetector::FrameSent(int64_t capture_time_ms) {
262  rtc::CritScope cs(&crit_);
263  int delay_ms = frame_queue_->End(capture_time_ms,
264                                   clock_->TimeInMilliseconds());
265  if (delay_ms > 0) {
266    AddProcessingTime(delay_ms);
267  }
268}
269
270void OveruseFrameDetector::AddProcessingTime(int elapsed_ms) {
271  int64_t now = clock_->TimeInMilliseconds();
272  if (last_sample_time_ms_ != 0) {
273    int64_t diff_ms = now - last_sample_time_ms_;
274    usage_->AddSample(elapsed_ms, diff_ms);
275  }
276  last_sample_time_ms_ = now;
277  UpdateCpuOveruseMetrics();
278}
279
280int32_t OveruseFrameDetector::Process() {
281  RTC_DCHECK(processing_thread_.CalledOnValidThread());
282
283  int64_t now = clock_->TimeInMilliseconds();
284
285  // Used to protect against Process() being called too often.
286  if (now < next_process_time_)
287    return 0;
288
289  next_process_time_ = now + kProcessIntervalMs;
290
291  CpuOveruseMetrics current_metrics;
292  {
293    rtc::CritScope cs(&crit_);
294    ++num_process_times_;
295
296    current_metrics = metrics_;
297    if (num_process_times_ <= options_.min_process_count)
298      return 0;
299  }
300
301  if (IsOverusing(current_metrics)) {
302    // If the last thing we did was going up, and now have to back down, we need
303    // to check if this peak was short. If so we should back off to avoid going
304    // back and forth between this load, the system doesn't seem to handle it.
305    bool check_for_backoff = last_rampup_time_ > last_overuse_time_;
306    if (check_for_backoff) {
307      if (now - last_rampup_time_ < kStandardRampUpDelayMs ||
308          num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) {
309        // Going up was not ok for very long, back off.
310        current_rampup_delay_ms_ *= kRampUpBackoffFactor;
311        if (current_rampup_delay_ms_ > kMaxRampUpDelayMs)
312          current_rampup_delay_ms_ = kMaxRampUpDelayMs;
313      } else {
314        // Not currently backing off, reset rampup delay.
315        current_rampup_delay_ms_ = kStandardRampUpDelayMs;
316      }
317    }
318
319    last_overuse_time_ = now;
320    in_quick_rampup_ = false;
321    checks_above_threshold_ = 0;
322    ++num_overuse_detections_;
323
324    if (observer_ != NULL)
325      observer_->OveruseDetected();
326  } else if (IsUnderusing(current_metrics, now)) {
327    last_rampup_time_ = now;
328    in_quick_rampup_ = true;
329
330    if (observer_ != NULL)
331      observer_->NormalUsage();
332  }
333
334  int rampup_delay =
335      in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
336
337  LOG(LS_VERBOSE) << " Frame stats: "
338                  << " encode usage " << current_metrics.encode_usage_percent
339                  << " overuse detections " << num_overuse_detections_
340                  << " rampup delay " << rampup_delay;
341
342  return 0;
343}
344
345bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) {
346  if (metrics.encode_usage_percent >=
347      options_.high_encode_usage_threshold_percent) {
348    ++checks_above_threshold_;
349  } else {
350    checks_above_threshold_ = 0;
351  }
352  return checks_above_threshold_ >= options_.high_threshold_consecutive_count;
353}
354
355bool OveruseFrameDetector::IsUnderusing(const CpuOveruseMetrics& metrics,
356                                        int64_t time_now) {
357  int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
358  if (time_now < last_rampup_time_ + delay)
359    return false;
360
361  return metrics.encode_usage_percent <
362         options_.low_encode_usage_threshold_percent;
363}
364}  // namespace webrtc
365