1// libjingle
2// Copyright 2010 Google Inc.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//  1. Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//  2. Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//  3. The name of the author may not be used to endorse or promote products
13//     derived from this software without specific prior written permission.
14//
15// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
27#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
28
29#include "talk/media/base/videocommon.h"
30#include "webrtc/base/common.h"  // For ASSERT
31#include "webrtc/base/criticalsection.h"
32#include "webrtc/base/scoped_ptr.h"
33#include "webrtc/base/sigslot.h"
34
35namespace cricket {
36
37class VideoFrame;
38
39// VideoAdapter adapts an input video frame to an output frame based on the
40// specified input and output formats. The adaptation includes dropping frames
41// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
42class VideoAdapter {
43 public:
44  VideoAdapter();
45  virtual ~VideoAdapter();
46
47  virtual void SetInputFormat(const VideoFormat& format);
48  void SetOutputFormat(const VideoFormat& format);
49  // Constrain output resolution to this many pixels overall
50  void SetOutputNumPixels(int num_pixels);
51  int GetOutputNumPixels() const;
52
53  const VideoFormat& input_format();
54  // Returns true if the adapter is dropping frames in calls to AdaptFrame.
55  bool drops_all_frames() const;
56  const VideoFormat& output_format();
57  // If the parameter black is true, the adapted frames will be black.
58  void SetBlackOutput(bool black);
59
60  // Adapt the input frame from the input format to the output format. Return
61  // true and set the output frame to NULL if the input frame is dropped. Return
62  // true and set the out frame to output_frame_ if the input frame is adapted
63  // successfully. Return false otherwise.
64  // Note that, if no adaptation is required, |out_frame| will refer directly
65  // in_frame. If a copy is always required, the caller must do an explicit
66  // copy.
67  // If a copy has taken place, |output_frame_| is owned by the VideoAdapter
68  // and will remain usable until the adapter is destroyed or AdaptFrame is
69  // called again.
70  bool AdaptFrame(VideoFrame* in_frame, VideoFrame** out_frame);
71
72  void set_scale_third(bool enable);
73  bool scale_third() const { return scale_third_; }
74
75  int adaptation_changes() const { return adaption_changes_; }
76
77 protected:
78  float FindClosestScale(int width, int height, int target_num_pixels);
79  float FindClosestViewScale(int width, int height, int target_num_pixels);
80  float FindLowerScale(int width, int height, int target_num_pixels);
81
82 private:
83  const float* GetViewScaleFactors() const;
84  float FindScale(const float* scale_factors,
85                  const float upbias, int width, int height,
86                  int target_num_pixels);
87  bool StretchToOutputFrame(const VideoFrame* in_frame);
88
89  VideoFormat input_format_;
90  VideoFormat output_format_;
91  int output_num_pixels_;
92  bool scale_third_;  // True if adapter allows scaling to 1/3 and 2/3.
93  int frames_in_;  // Number of input frames.
94  int frames_out_;  // Number of output frames.
95  int frames_scaled_;  // Number of frames scaled.
96  int adaption_changes_;  // Number of changes in scale factor.
97  size_t previous_width_;  // Previous adapter output width.
98  size_t previous_height_;  // Previous adapter output height.
99  bool black_output_;  // Flag to tell if we need to black output_frame_.
100  bool is_black_;  // Flag to tell if output_frame_ is currently black.
101  int64 interval_next_frame_;
102  rtc::scoped_ptr<VideoFrame> output_frame_;
103  // The critical section to protect the above variables.
104  rtc::CriticalSection critical_section_;
105
106  DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
107};
108
109// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
110// the format request from the server, the resolution request from the encoder,
111// and the CPU load.
112class CoordinatedVideoAdapter
113    : public VideoAdapter, public sigslot::has_slots<>  {
114 public:
115  enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
116  enum AdaptReasonEnum {
117    ADAPTREASON_NONE = 0,
118    ADAPTREASON_CPU = 1,
119    ADAPTREASON_BANDWIDTH = 2,
120    ADAPTREASON_VIEW = 4
121  };
122  typedef int AdaptReason;
123
124  CoordinatedVideoAdapter();
125  virtual ~CoordinatedVideoAdapter() {}
126
127  virtual void SetInputFormat(const VideoFormat& format);
128
129  // Enable or disable video adaptation due to the change of the CPU load.
130  void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
131  bool cpu_adaptation() const { return cpu_adaptation_; }
132  // Enable or disable smoothing when doing CPU adaptation. When smoothing is
133  // enabled, system CPU load is tracked using an exponential weighted
134  // average.
135  void set_cpu_smoothing(bool enable);
136  bool cpu_smoothing() const { return cpu_smoothing_; }
137  // Enable or disable video adaptation due to the change of the GD
138  void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
139  bool gd_adaptation() const { return gd_adaptation_; }
140  // Enable or disable video adaptation due to the change of the View
141  void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
142  bool view_adaptation() const { return view_adaptation_; }
143  // Enable or disable video adaptation to fast switch View
144  void set_view_switch(bool enable) { view_switch_ = enable; }
145  bool view_switch() const { return view_switch_; }
146
147  CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
148    return adapt_reason_;
149  }
150
151  // When the video is decreased, set the waiting time for CPU adaptation to
152  // decrease video again.
153  void set_cpu_load_min_samples(int cpu_load_min_samples);
154  int cpu_load_min_samples() const { return cpu_load_min_samples_; }
155  // CPU system load high threshold for reducing resolution.  e.g. 0.85f
156  void set_high_system_threshold(float high_system_threshold);
157  float high_system_threshold() const { return high_system_threshold_; }
158  // CPU system load low threshold for increasing resolution.  e.g. 0.70f
159  void set_low_system_threshold(float low_system_threshold);
160  float low_system_threshold() const { return low_system_threshold_; }
161  // CPU process load threshold for reducing resolution.  e.g. 0.10f
162  void set_process_threshold(float process_threshold);
163  float process_threshold() const { return process_threshold_; }
164
165  // Handle the format request from the server via Jingle update message.
166  void OnOutputFormatRequest(const VideoFormat& format);
167  // Handle the resolution request from the encoder due to bandwidth changes.
168  void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
169  // Handle the resolution request for CPU overuse.
170  void OnCpuResolutionRequest(AdaptRequest request);
171  // Handle the CPU load provided by a CPU monitor.
172  void OnCpuLoadUpdated(int current_cpus, int max_cpus,
173                        float process_load, float system_load);
174
175  sigslot::signal0<> SignalCpuAdaptationUnable;
176
177 private:
178  // Adapt to the minimum of the formats the server requests, the CPU wants, and
179  // the encoder wants. Returns true if resolution changed.
180  bool AdaptToMinimumFormat(int* new_width, int* new_height);
181  bool IsMinimumFormat(int pixels);
182  void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
183                      int* num_pixels);
184  CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
185    int current_cpus, int max_cpus,
186    float process_load, float system_load);
187
188  bool cpu_adaptation_;  // True if cpu adaptation is enabled.
189  bool cpu_smoothing_;  // True if cpu smoothing is enabled (with adaptation).
190  bool gd_adaptation_;  // True if gd adaptation is enabled.
191  bool view_adaptation_;  // True if view adaptation is enabled.
192  bool view_switch_;  // True if view switch is enabled.
193  int cpu_downgrade_count_;
194  int cpu_load_min_samples_;
195  int cpu_load_num_samples_;
196  // cpu system load thresholds relative to max cpus.
197  float high_system_threshold_;
198  float low_system_threshold_;
199  // cpu process load thresholds relative to current cpus.
200  float process_threshold_;
201  // Video formats that the server view requests, the CPU wants, and the encoder
202  // wants respectively. The adapted output format is the minimum of these.
203  int view_desired_num_pixels_;
204  int64 view_desired_interval_;
205  int encoder_desired_num_pixels_;
206  int cpu_desired_num_pixels_;
207  CoordinatedVideoAdapter::AdaptReason adapt_reason_;
208  // The critical section to protect handling requests.
209  rtc::CriticalSection request_critical_section_;
210
211  // The weighted average of cpu load over time. It's always updated (if cpu
212  // adaptation is on), but only used if cpu_smoothing_ is set.
213  float system_load_average_;
214
215  DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
216};
217
218}  // namespace cricket
219
220#endif  // TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
221