1/*
2 * libjingle
3 * Copyright 2004 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_
29#define TALK_MEDIA_WEBRTCVIDEOENGINE_H_
30
31#include <map>
32#include <vector>
33
34#include "talk/base/scoped_ptr.h"
35#include "talk/media/base/codec.h"
36#include "talk/media/base/videocommon.h"
37#include "talk/media/webrtc/webrtccommon.h"
38#include "talk/media/webrtc/webrtcexport.h"
39#include "talk/media/webrtc/webrtcvideoencoderfactory.h"
40#include "talk/session/media/channel.h"
41#include "webrtc/video_engine/include/vie_base.h"
42
43#if !defined(LIBPEERCONNECTION_LIB) && \
44    !defined(LIBPEERCONNECTION_IMPLEMENTATION)
45#error "Bogus include."
46#endif
47
48namespace webrtc {
49class VideoCaptureModule;
50class VideoDecoder;
51class VideoEncoder;
52class VideoRender;
53class ViEExternalCapture;
54class ViERTP_RTCP;
55}
56
57namespace talk_base {
58class CpuMonitor;
59}  // namespace talk_base
60
61namespace cricket {
62
63class VideoCapturer;
64class VideoFrame;
65class VideoProcessor;
66class VideoRenderer;
67class ViETraceWrapper;
68class ViEWrapper;
69class VoiceMediaChannel;
70class WebRtcDecoderObserver;
71class WebRtcEncoderObserver;
72class WebRtcLocalStreamInfo;
73class WebRtcRenderAdapter;
74class WebRtcVideoChannelRecvInfo;
75class WebRtcVideoChannelSendInfo;
76class WebRtcVideoDecoderFactory;
77class WebRtcVideoEncoderFactory;
78class WebRtcVideoMediaChannel;
79class WebRtcVoiceEngine;
80
81struct CapturedFrame;
82struct Device;
83
84class WebRtcVideoEngine : public sigslot::has_slots<>,
85                          public webrtc::TraceCallback,
86                          public WebRtcVideoEncoderFactory::Observer {
87 public:
88  // Creates the WebRtcVideoEngine with internal VideoCaptureModule.
89  WebRtcVideoEngine();
90  // For testing purposes. Allows the WebRtcVoiceEngine,
91  // ViEWrapper and CpuMonitor to be mocks.
92  // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented.
93  WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
94                    ViEWrapper* vie_wrapper,
95                    talk_base::CpuMonitor* cpu_monitor);
96  WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
97                    ViEWrapper* vie_wrapper,
98                    ViETraceWrapper* tracing,
99                    talk_base::CpuMonitor* cpu_monitor);
100  ~WebRtcVideoEngine();
101
102  // Basic video engine implementation.
103  bool Init(talk_base::Thread* worker_thread);
104  void Terminate();
105
106  int GetCapabilities();
107  bool SetOptions(int options);
108  bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
109
110  WebRtcVideoMediaChannel* CreateChannel(VoiceMediaChannel* voice_channel);
111
112  const std::vector<VideoCodec>& codecs() const;
113  const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
114  void SetLogging(int min_sev, const char* filter);
115
116  bool SetLocalRenderer(VideoRenderer* renderer);
117  sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange;
118
119  // Set the VoiceEngine for A/V sync. This can only be called before Init.
120  bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine);
121  // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
122  // not take the ownership of |decoder_factory|. The caller needs to make sure
123  // that |decoder_factory| outlives the video engine.
124  void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
125  // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
126  // not take the ownership of |encoder_factory|. The caller needs to make sure
127  // that |encoder_factory| outlives the video engine.
128  void SetExternalEncoderFactory(WebRtcVideoEncoderFactory* encoder_factory);
129  // Enable the render module with timing control.
130  bool EnableTimedRender();
131
132  // Returns an external decoder for the given codec type. The return value
133  // can be NULL if decoder factory is not given or it does not support the
134  // codec type. The caller takes the ownership of the returned object.
135  webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type);
136  // Releases the decoder instance created by CreateExternalDecoder().
137  void DestroyExternalDecoder(webrtc::VideoDecoder* decoder);
138
139  // Returns an external encoder for the given codec type. The return value
140  // can be NULL if encoder factory is not given or it does not support the
141  // codec type. The caller takes the ownership of the returned object.
142  webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type);
143  // Releases the encoder instance created by CreateExternalEncoder().
144  void DestroyExternalEncoder(webrtc::VideoEncoder* encoder);
145
146  // Returns true if the codec type is supported by the external encoder.
147  bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const;
148
149  // Functions called by WebRtcVideoMediaChannel.
150  talk_base::Thread* worker_thread() { return worker_thread_; }
151  ViEWrapper* vie() { return vie_wrapper_.get(); }
152  const VideoFormat& default_codec_format() const {
153    return default_codec_format_;
154  }
155  int GetLastEngineError();
156  bool FindCodec(const VideoCodec& in);
157  bool CanSendCodec(const VideoCodec& in, const VideoCodec& current,
158                    VideoCodec* out);
159  void RegisterChannel(WebRtcVideoMediaChannel* channel);
160  void UnregisterChannel(WebRtcVideoMediaChannel* channel);
161  bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec,
162                                    webrtc::VideoCodec* out_codec);
163  // Check whether the supplied trace should be ignored.
164  bool ShouldIgnoreTrace(const std::string& trace);
165  int GetNumOfChannels();
166
167  VideoFormat GetStartCaptureFormat() const { return default_codec_format_; }
168
169  talk_base::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); }
170
171 protected:
172  // When a video processor registers with the engine.
173  // SignalMediaFrame will be invoked for every video frame.
174  // See videoprocessor.h for param reference.
175  sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame;
176
177 private:
178  typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels;
179  struct VideoCodecPref {
180    const char* name;
181    int payload_type;
182    int pref;
183  };
184
185  static const VideoCodecPref kVideoCodecPrefs[];
186  static const VideoFormatPod kVideoFormats[];
187  static const VideoFormatPod kDefaultVideoFormat;
188
189  void Construct(ViEWrapper* vie_wrapper,
190                 ViETraceWrapper* tracing,
191                 WebRtcVoiceEngine* voice_engine,
192                 talk_base::CpuMonitor* cpu_monitor);
193  bool SetDefaultCodec(const VideoCodec& codec);
194  bool RebuildCodecList(const VideoCodec& max_codec);
195  void SetTraceFilter(int filter);
196  void SetTraceOptions(const std::string& options);
197  bool InitVideoEngine();
198
199  // webrtc::TraceCallback implementation.
200  virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
201
202  // WebRtcVideoEncoderFactory::Observer implementation.
203  virtual void OnCodecsAvailable();
204
205  talk_base::Thread* worker_thread_;
206  talk_base::scoped_ptr<ViEWrapper> vie_wrapper_;
207  bool vie_wrapper_base_initialized_;
208  talk_base::scoped_ptr<ViETraceWrapper> tracing_;
209  WebRtcVoiceEngine* voice_engine_;
210  talk_base::scoped_ptr<webrtc::VideoRender> render_module_;
211  WebRtcVideoEncoderFactory* encoder_factory_;
212  WebRtcVideoDecoderFactory* decoder_factory_;
213  std::vector<VideoCodec> video_codecs_;
214  std::vector<RtpHeaderExtension> rtp_header_extensions_;
215  VideoFormat default_codec_format_;
216
217  bool initialized_;
218  talk_base::CriticalSection channels_crit_;
219  VideoChannels channels_;
220
221  bool capture_started_;
222  int local_renderer_w_;
223  int local_renderer_h_;
224  VideoRenderer* local_renderer_;
225
226  // Critical section to protect the media processor register/unregister
227  // while processing a frame
228  talk_base::CriticalSection signal_media_critical_;
229
230  talk_base::scoped_ptr<talk_base::CpuMonitor> cpu_monitor_;
231};
232
233class WebRtcVideoMediaChannel : public talk_base::MessageHandler,
234                                public VideoMediaChannel,
235                                public webrtc::Transport {
236 public:
237  WebRtcVideoMediaChannel(WebRtcVideoEngine* engine,
238                          VoiceMediaChannel* voice_channel);
239  ~WebRtcVideoMediaChannel();
240  bool Init();
241
242  WebRtcVideoEngine* engine() { return engine_; }
243  VoiceMediaChannel* voice_channel() { return voice_channel_; }
244  int video_channel() const { return vie_channel_; }
245  bool sending() const { return sending_; }
246
247  // VideoMediaChannel implementation
248  virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs);
249  virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs);
250  virtual bool GetSendCodec(VideoCodec* send_codec);
251  virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format);
252  virtual bool SetRender(bool render);
253  virtual bool SetSend(bool send);
254
255  virtual bool AddSendStream(const StreamParams& sp);
256  virtual bool RemoveSendStream(uint32 ssrc);
257  virtual bool AddRecvStream(const StreamParams& sp);
258  virtual bool RemoveRecvStream(uint32 ssrc);
259  virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer);
260  virtual bool GetStats(VideoMediaInfo* info);
261  virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer);
262  virtual bool SendIntraFrame();
263  virtual bool RequestIntraFrame();
264
265  virtual void OnPacketReceived(talk_base::Buffer* packet);
266  virtual void OnRtcpReceived(talk_base::Buffer* packet);
267  virtual void OnReadyToSend(bool ready);
268  virtual bool MuteStream(uint32 ssrc, bool on);
269  virtual bool SetRecvRtpHeaderExtensions(
270      const std::vector<RtpHeaderExtension>& extensions);
271  virtual bool SetSendRtpHeaderExtensions(
272      const std::vector<RtpHeaderExtension>& extensions);
273  virtual bool SetSendBandwidth(bool autobw, int bps);
274  virtual bool SetOptions(const VideoOptions &options);
275  virtual bool GetOptions(VideoOptions *options) const {
276    *options = options_;
277    return true;
278  }
279  virtual void SetInterface(NetworkInterface* iface);
280  virtual void UpdateAspectRatio(int ratio_w, int ratio_h);
281
282  // Public functions for use by tests and other specialized code.
283  uint32 send_ssrc() const { return 0; }
284  bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
285  void SendFrame(VideoCapturer* capturer, const VideoFrame* frame);
286  bool SendFrame(WebRtcVideoChannelSendInfo* channel_info,
287                 const VideoFrame* frame, bool is_screencast);
288
289  void AdaptAndSendFrame(VideoCapturer* capturer, const VideoFrame* frame);
290
291  // Thunk functions for use with HybridVideoEngine
292  void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) {
293    SendFrame(0u, frame, capturer->IsScreencast());
294  }
295  void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) {
296  }
297
298  virtual void OnMessage(talk_base::Message* msg);
299
300 protected:
301  int GetLastEngineError() { return engine()->GetLastEngineError(); }
302  virtual int SendPacket(int channel, const void* data, int len);
303  virtual int SendRTCPPacket(int channel, const void* data, int len);
304
305 private:
306  typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap;
307  typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap;
308  typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int);
309
310  enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV };
311
312  // Creates and initializes a ViE channel. When successful |channel_id| will
313  // contain the new channel's ID. If |receiving| is true |ssrc| is the
314  // remote ssrc. If |sending| is true the ssrc is local ssrc. If both
315  // |receiving| and |sending| is true the ssrc must be 0 and the channel will
316  // be created as a default channel. The ssrc must be different for receive
317  // channels and it must be different for send channels. If the same SSRC is
318  // being used for creating channel more than once, this function will fail
319  // returning false.
320  bool CreateChannel(uint32 ssrc_key, MediaDirection direction,
321                     int* channel_id);
322  bool ConfigureChannel(int channel_id, MediaDirection direction,
323                        uint32 ssrc_key);
324  bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key);
325  bool ConfigureSending(int channel_id, uint32 local_ssrc_key);
326  bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type,
327                  bool nack_enabled);
328  bool SetSendCodec(const webrtc::VideoCodec& codec, int min_bitrate,
329                    int start_bitrate, int max_bitrate);
330  bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel,
331                    const webrtc::VideoCodec& codec, int min_bitrate,
332                    int start_bitrate, int max_bitrate);
333  void LogSendCodecChange(const std::string& reason);
334  // Prepares the channel with channel id |info->channel_id()| to receive all
335  // codecs in |receive_codecs_| and start receive packets.
336  bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info);
337  // Returns the channel number that receives the stream with SSRC |ssrc|.
338  int GetRecvChannelNum(uint32 ssrc);
339  // Given captured video frame size, checks if we need to reset vie send codec.
340  // |reset| is set to whether resetting has happened on vie or not.
341  // Returns false on error.
342  bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel,
343                              int new_width, int new_height, bool is_screencast,
344                              bool* reset);
345  // Checks the current bitrate estimate and modifies the start bitrate
346  // accordingly.
347  void MaybeChangeStartBitrate(int channel_id, webrtc::VideoCodec* video_codec);
348  // Helper function for starting the sending of media on all channels or
349  // |channel_id|. Note that these two function do not change |sending_|.
350  bool StartSend();
351  bool StartSend(WebRtcVideoChannelSendInfo* send_channel);
352  // Helper function for stop the sending of media on all channels or
353  // |channel_id|. Note that these two function do not change |sending_|.
354  bool StopSend();
355  bool StopSend(WebRtcVideoChannelSendInfo* send_channel);
356  bool SendIntraFrame(int channel_id);
357
358  // Send with one local SSRC. Normal case.
359  bool IsOneSsrcStream(const StreamParams& sp);
360
361  bool HasReadySendChannels();
362
363  // Send channel key returns the key corresponding to the provided local SSRC
364  // in |key|. The return value is true upon success.
365  // If the local ssrc correspond to that of the default channel the key is 0.
366  // For all other channels the returned key will be the same as the local ssrc.
367  bool GetSendChannelKey(uint32 local_ssrc, uint32* key);
368  WebRtcVideoChannelSendInfo* GetSendChannel(VideoCapturer* video_capturer);
369  WebRtcVideoChannelSendInfo* GetSendChannel(uint32 local_ssrc);
370  // Creates a new unique key that can be used for inserting a new send channel
371  // into |send_channels_|
372  bool CreateSendChannelKey(uint32 local_ssrc, uint32* key);
373
374  bool IsDefaultChannel(int channel_id) const {
375    return channel_id == vie_channel_;
376  }
377  uint32 GetDefaultChannelSsrc();
378
379  bool DeleteSendChannel(uint32 ssrc_key);
380
381  bool InConferenceMode() const {
382    return options_.conference_mode.GetWithDefaultIfUnset(false);
383  }
384  bool RemoveCapturer(uint32 ssrc);
385
386
387  talk_base::MessageQueue* worker_thread() { return engine_->worker_thread(); }
388  void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate);
389  void FlushBlackFrame(uint32 ssrc, int64 timestamp);
390
391  void SetNetworkTransmissionState(bool is_transmitting);
392
393  bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
394                          const RtpHeaderExtension* extension);
395  bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
396                          const std::vector<RtpHeaderExtension>& extensions,
397                          const char header_extension_uri[]);
398
399  // Signal when cpu adaptation has no further scope to adapt.
400  void OnCpuAdaptationUnable();
401
402  // Global state.
403  WebRtcVideoEngine* engine_;
404  VoiceMediaChannel* voice_channel_;
405  int vie_channel_;
406  bool nack_enabled_;
407  // Receiver Estimated Max Bitrate
408  bool remb_enabled_;
409  VideoOptions options_;
410
411  // Global recv side state.
412  // Note the default channel (vie_channel_), i.e. the send channel
413  // corresponding to all the receive channels (this must be done for REMB to
414  // work properly), resides in both recv_channels_ and send_channels_ with the
415  // ssrc key 0.
416  RecvChannelMap recv_channels_;  // Contains all receive channels.
417  std::vector<webrtc::VideoCodec> receive_codecs_;
418  bool render_started_;
419  uint32 first_receive_ssrc_;
420  std::vector<RtpHeaderExtension> receive_extensions_;
421
422  // Global send side state.
423  SendChannelMap send_channels_;
424  talk_base::scoped_ptr<webrtc::VideoCodec> send_codec_;
425  int send_red_type_;
426  int send_fec_type_;
427  int send_min_bitrate_;
428  int send_start_bitrate_;
429  int send_max_bitrate_;
430  bool sending_;
431  std::vector<RtpHeaderExtension> send_extensions_;
432
433  // The aspect ratio that the channel desires. 0 means there is no desired
434  // aspect ratio
435  int ratio_w_;
436  int ratio_h_;
437};
438
439}  // namespace cricket
440
441#endif  // TALK_MEDIA_WEBRTCVIDEOENGINE_H_
442