1/*
2 * libjingle
3 * Copyright 2010 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
29#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
30
31#include "talk/media/base/videocommon.h"
32#include "webrtc/base/common.h"  // For ASSERT
33#include "webrtc/base/criticalsection.h"
34#include "webrtc/base/scoped_ptr.h"
35#include "webrtc/base/sigslot.h"
36
37namespace cricket {
38
39class VideoFrame;
40
41// VideoAdapter adapts an input video frame to an output frame based on the
42// specified input and output formats. The adaptation includes dropping frames
43// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
44class VideoAdapter {
45 public:
46  VideoAdapter();
47  virtual ~VideoAdapter();
48
49  virtual void SetInputFormat(const VideoFormat& format);
50  void SetOutputFormat(const VideoFormat& format);
51  // Constrain output resolution to this many pixels overall
52  void SetOutputNumPixels(int num_pixels);
53  int GetOutputNumPixels() const;
54
55  const VideoFormat& input_format();
56  // Returns true if the adapter will always return zero size from
57  // AdaptFrameResolution.
58  bool drops_all_frames() const;
59  const VideoFormat& output_format();
60
61  // Return the adapted resolution given the input resolution. The returned
62  // resolution will be 0x0 if the frame should be dropped.
63  VideoFormat AdaptFrameResolution(int in_width, int in_height);
64
65  void set_scale_third(bool enable);
66  bool scale_third() const { return scale_third_; }
67
68  int adaptation_changes() const { return adaption_changes_; }
69
70 protected:
71  float FindClosestScale(int width, int height, int target_num_pixels);
72  float FindClosestViewScale(int width, int height, int target_num_pixels);
73  float FindLowerScale(int width, int height, int target_num_pixels);
74
75 private:
76  const float* GetViewScaleFactors() const;
77  float FindScale(const float* scale_factors,
78                  const float upbias, int width, int height,
79                  int target_num_pixels);
80
81  VideoFormat input_format_;
82  VideoFormat output_format_;
83  int output_num_pixels_;
84  bool scale_third_;  // True if adapter allows scaling to 1/3 and 2/3.
85  int frames_in_;  // Number of input frames.
86  int frames_out_;  // Number of output frames.
87  int frames_scaled_;  // Number of frames scaled.
88  int adaption_changes_;  // Number of changes in scale factor.
89  size_t previous_width_;  // Previous adapter output width.
90  size_t previous_height_;  // Previous adapter output height.
91  int64_t interval_next_frame_;
92  // The critical section to protect the above variables.
93  rtc::CriticalSection critical_section_;
94
95  RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
96};
97
98// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
99// the format request from the server, the resolution request from the encoder,
100// and the CPU load.
101class CoordinatedVideoAdapter
102    : public VideoAdapter, public sigslot::has_slots<>  {
103 public:
104  enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
105  enum AdaptReasonEnum {
106    ADAPTREASON_NONE = 0,
107    ADAPTREASON_CPU = 1,
108    ADAPTREASON_BANDWIDTH = 2,
109    ADAPTREASON_VIEW = 4
110  };
111  typedef int AdaptReason;
112
113  CoordinatedVideoAdapter();
114  virtual ~CoordinatedVideoAdapter() {}
115
116  virtual void SetInputFormat(const VideoFormat& format);
117
118  // Enable or disable video adaptation due to the change of the CPU load.
119  void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
120  bool cpu_adaptation() const { return cpu_adaptation_; }
121  // Enable or disable smoothing when doing CPU adaptation. When smoothing is
122  // enabled, system CPU load is tracked using an exponential weighted
123  // average.
124  void set_cpu_smoothing(bool enable);
125  bool cpu_smoothing() const { return cpu_smoothing_; }
126  // Enable or disable video adaptation due to the change of the GD
127  void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
128  bool gd_adaptation() const { return gd_adaptation_; }
129  // Enable or disable video adaptation due to the change of the View
130  void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
131  bool view_adaptation() const { return view_adaptation_; }
132  // Enable or disable video adaptation to fast switch View
133  void set_view_switch(bool enable) { view_switch_ = enable; }
134  bool view_switch() const { return view_switch_; }
135
136  CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
137    return adapt_reason_;
138  }
139
140  // When the video is decreased, set the waiting time for CPU adaptation to
141  // decrease video again.
142  void set_cpu_load_min_samples(int cpu_load_min_samples);
143  int cpu_load_min_samples() const { return cpu_load_min_samples_; }
144  // CPU system load high threshold for reducing resolution.  e.g. 0.85f
145  void set_high_system_threshold(float high_system_threshold);
146  float high_system_threshold() const { return high_system_threshold_; }
147  // CPU system load low threshold for increasing resolution.  e.g. 0.70f
148  void set_low_system_threshold(float low_system_threshold);
149  float low_system_threshold() const { return low_system_threshold_; }
150  // CPU process load threshold for reducing resolution.  e.g. 0.10f
151  void set_process_threshold(float process_threshold);
152  float process_threshold() const { return process_threshold_; }
153
154  // Handle the format request from the server via Jingle update message.
155  void OnOutputFormatRequest(const VideoFormat& format);
156  // Handle the resolution request from the encoder due to bandwidth changes.
157  void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
158  // Handle the resolution request for CPU overuse.
159  void OnCpuResolutionRequest(AdaptRequest request);
160  // Handle the CPU load provided by a CPU monitor.
161  void OnCpuLoadUpdated(int current_cpus, int max_cpus,
162                        float process_load, float system_load);
163
164  sigslot::signal0<> SignalCpuAdaptationUnable;
165
166 private:
167  // Adapt to the minimum of the formats the server requests, the CPU wants, and
168  // the encoder wants. Returns true if resolution changed.
169  bool AdaptToMinimumFormat(int* new_width, int* new_height);
170  bool IsMinimumFormat(int pixels);
171  void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
172                      int* num_pixels);
173  CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
174    int current_cpus, int max_cpus,
175    float process_load, float system_load);
176
177  bool cpu_adaptation_;  // True if cpu adaptation is enabled.
178  bool cpu_smoothing_;  // True if cpu smoothing is enabled (with adaptation).
179  bool gd_adaptation_;  // True if gd adaptation is enabled.
180  bool view_adaptation_;  // True if view adaptation is enabled.
181  bool view_switch_;  // True if view switch is enabled.
182  int cpu_downgrade_count_;
183  int cpu_load_min_samples_;
184  int cpu_load_num_samples_;
185  // cpu system load thresholds relative to max cpus.
186  float high_system_threshold_;
187  float low_system_threshold_;
188  // cpu process load thresholds relative to current cpus.
189  float process_threshold_;
190  // Video formats that the server view requests, the CPU wants, and the encoder
191  // wants respectively. The adapted output format is the minimum of these.
192  int view_desired_num_pixels_;
193  int64_t view_desired_interval_;
194  int encoder_desired_num_pixels_;
195  int cpu_desired_num_pixels_;
196  CoordinatedVideoAdapter::AdaptReason adapt_reason_;
197  // The critical section to protect handling requests.
198  rtc::CriticalSection request_critical_section_;
199
200  // The weighted average of cpu load over time. It's always updated (if cpu
201  // adaptation is on), but only used if cpu_smoothing_ is set.
202  float system_load_average_;
203
204  RTC_DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
205};
206
207}  // namespace cricket
208
209#endif  // TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
210