1/*
2 * libjingle
3 * Copyright 2014 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
29#define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
30
31#include <map>
32#include <string>
33#include <vector>
34
35#include "talk/media/base/mediaengine.h"
36#include "talk/media/webrtc/webrtcvideochannelfactory.h"
37#include "talk/media/webrtc/webrtcvideodecoderfactory.h"
38#include "talk/media/webrtc/webrtcvideoencoderfactory.h"
39#include "webrtc/base/cpumonitor.h"
40#include "webrtc/base/scoped_ptr.h"
41#include "webrtc/base/thread_annotations.h"
42#include "webrtc/common_video/interface/i420_video_frame.h"
43#include "webrtc/transport.h"
44#include "webrtc/video_receive_stream.h"
45#include "webrtc/video_renderer.h"
46#include "webrtc/video_send_stream.h"
47
48namespace webrtc {
49class Call;
50class VideoCaptureModule;
51class VideoDecoder;
52class VideoEncoder;
53class VideoRender;
54class VideoSendStreamInput;
55class VideoReceiveStream;
56}
57
58namespace rtc {
59class CpuMonitor;
60class Thread;
61}  // namespace rtc
62
63namespace cricket {
64
65class VideoCapturer;
66class VideoFrame;
67class VideoProcessor;
68class VideoRenderer;
69class VoiceMediaChannel;
70class WebRtcDecoderObserver;
71class WebRtcEncoderObserver;
72class WebRtcLocalStreamInfo;
73class WebRtcRenderAdapter;
74class WebRtcVideoChannel2;
75class WebRtcVideoChannelRecvInfo;
76class WebRtcVideoChannelSendInfo;
77class WebRtcVoiceEngine;
78
79struct CapturedFrame;
80struct Device;
81
82class WebRtcVideoEngine2;
83class WebRtcVideoChannel2;
84class WebRtcVideoRenderer;
85
86class UnsignalledSsrcHandler {
87 public:
88  enum Action {
89    kDropPacket,
90    kDeliverPacket,
91  };
92  virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine,
93                                   uint32_t ssrc) = 0;
94};
95
96// TODO(pbos): Remove, use external handlers only.
97class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
98 public:
99  DefaultUnsignalledSsrcHandler();
100  virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine,
101                                   uint32_t ssrc) OVERRIDE;
102
103  VideoRenderer* GetDefaultRenderer() const;
104  void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer);
105
106 private:
107  uint32_t default_recv_ssrc_;
108  VideoRenderer* default_renderer_;
109};
110
111class WebRtcVideoEncoderFactory2 {
112 public:
113  virtual ~WebRtcVideoEncoderFactory2();
114  virtual std::vector<webrtc::VideoStream> CreateVideoStreams(
115      const VideoCodec& codec,
116      const VideoOptions& options,
117      size_t num_streams);
118
119  virtual webrtc::VideoEncoder* CreateVideoEncoder(
120      const VideoCodec& codec,
121      const VideoOptions& options);
122
123  virtual void* CreateVideoEncoderSettings(const VideoCodec& codec,
124                                           const VideoOptions& options);
125
126  virtual void DestroyVideoEncoderSettings(const VideoCodec& codec,
127                                           void* encoder_settings);
128
129  virtual bool SupportsCodec(const cricket::VideoCodec& codec);
130};
131
132// WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667).
133class WebRtcVideoEngine2 : public sigslot::has_slots<>,
134                           public WebRtcVideoEncoderFactory::Observer {
135 public:
136  // Creates the WebRtcVideoEngine2 with internal VideoCaptureModule.
137  WebRtcVideoEngine2();
138  virtual ~WebRtcVideoEngine2();
139
140  // Use a custom WebRtcVideoChannelFactory (for testing purposes).
141  void SetChannelFactory(WebRtcVideoChannelFactory* channel_factory);
142
143  // Basic video engine implementation.
144  bool Init(rtc::Thread* worker_thread);
145  void Terminate();
146
147  int GetCapabilities();
148  bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
149  VideoEncoderConfig GetDefaultEncoderConfig() const;
150
151  WebRtcVideoChannel2* CreateChannel(VoiceMediaChannel* voice_channel);
152
153  const std::vector<VideoCodec>& codecs() const;
154  const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
155  void SetLogging(int min_sev, const char* filter);
156
157  // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
158  // not take the ownership of |decoder_factory|. The caller needs to make sure
159  // that |decoder_factory| outlives the video engine.
160  void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
161  // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
162  // not take the ownership of |encoder_factory|. The caller needs to make sure
163  // that |encoder_factory| outlives the video engine.
164  virtual void SetExternalEncoderFactory(
165      WebRtcVideoEncoderFactory* encoder_factory);
166
167  bool EnableTimedRender();
168  // This is currently ignored.
169  sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange;
170
171  // Set the VoiceEngine for A/V sync. This can only be called before Init.
172  bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine);
173
174  bool FindCodec(const VideoCodec& in);
175  bool CanSendCodec(const VideoCodec& in,
176                    const VideoCodec& current,
177                    VideoCodec* out);
178  // Check whether the supplied trace should be ignored.
179  bool ShouldIgnoreTrace(const std::string& trace);
180
181  VideoFormat GetStartCaptureFormat() const { return default_codec_format_; }
182
183  rtc::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); }
184
185  virtual WebRtcVideoEncoderFactory2* GetVideoEncoderFactory();
186
187 private:
188  virtual void OnCodecsAvailable() OVERRIDE;
189
190  rtc::Thread* worker_thread_;
191  WebRtcVoiceEngine* voice_engine_;
192  std::vector<VideoCodec> video_codecs_;
193  std::vector<RtpHeaderExtension> rtp_header_extensions_;
194  VideoFormat default_codec_format_;
195
196  bool initialized_;
197
198  // Critical section to protect the media processor register/unregister
199  // while processing a frame
200  rtc::CriticalSection signal_media_critical_;
201
202  rtc::scoped_ptr<rtc::CpuMonitor> cpu_monitor_;
203  WebRtcVideoChannelFactory* channel_factory_;
204  WebRtcVideoEncoderFactory2 default_video_encoder_factory_;
205
206  WebRtcVideoDecoderFactory* external_decoder_factory_;
207  WebRtcVideoEncoderFactory* external_encoder_factory_;
208};
209
210class WebRtcVideoChannel2 : public rtc::MessageHandler,
211                            public VideoMediaChannel,
212                            public webrtc::newapi::Transport {
213 public:
214  WebRtcVideoChannel2(WebRtcVideoEngine2* engine,
215                      VoiceMediaChannel* voice_channel,
216                      WebRtcVideoEncoderFactory2* encoder_factory);
217  // For testing purposes insert a pre-constructed call to verify that
218  // WebRtcVideoChannel2 calls the correct corresponding methods.
219  WebRtcVideoChannel2(webrtc::Call* call,
220                      WebRtcVideoEngine2* engine,
221                      WebRtcVideoEncoderFactory2* encoder_factory);
222  ~WebRtcVideoChannel2();
223  bool Init();
224
225  // VideoMediaChannel implementation
226  virtual bool SetRecvCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE;
227  virtual bool SetSendCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE;
228  virtual bool GetSendCodec(VideoCodec* send_codec) OVERRIDE;
229  virtual bool SetSendStreamFormat(uint32 ssrc,
230                                   const VideoFormat& format) OVERRIDE;
231  virtual bool SetRender(bool render) OVERRIDE;
232  virtual bool SetSend(bool send) OVERRIDE;
233
234  virtual bool AddSendStream(const StreamParams& sp) OVERRIDE;
235  virtual bool RemoveSendStream(uint32 ssrc) OVERRIDE;
236  virtual bool AddRecvStream(const StreamParams& sp) OVERRIDE;
237  virtual bool RemoveRecvStream(uint32 ssrc) OVERRIDE;
238  virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) OVERRIDE;
239  virtual bool GetStats(const StatsOptions& options,
240                        VideoMediaInfo* info) OVERRIDE;
241  virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer) OVERRIDE;
242  virtual bool SendIntraFrame() OVERRIDE;
243  virtual bool RequestIntraFrame() OVERRIDE;
244
245  virtual void OnPacketReceived(rtc::Buffer* packet,
246                                const rtc::PacketTime& packet_time)
247      OVERRIDE;
248  virtual void OnRtcpReceived(rtc::Buffer* packet,
249                              const rtc::PacketTime& packet_time)
250      OVERRIDE;
251  virtual void OnReadyToSend(bool ready) OVERRIDE;
252  virtual bool MuteStream(uint32 ssrc, bool mute) OVERRIDE;
253
254  // Set send/receive RTP header extensions. This must be done before creating
255  // streams as it only has effect on future streams.
256  virtual bool SetRecvRtpHeaderExtensions(
257      const std::vector<RtpHeaderExtension>& extensions) OVERRIDE;
258  virtual bool SetSendRtpHeaderExtensions(
259      const std::vector<RtpHeaderExtension>& extensions) OVERRIDE;
260  virtual bool SetStartSendBandwidth(int bps) OVERRIDE;
261  virtual bool SetMaxSendBandwidth(int bps) OVERRIDE;
262  virtual bool SetOptions(const VideoOptions& options) OVERRIDE;
263  virtual bool GetOptions(VideoOptions* options) const OVERRIDE {
264    *options = options_;
265    return true;
266  }
267  virtual void SetInterface(NetworkInterface* iface) OVERRIDE;
268  virtual void UpdateAspectRatio(int ratio_w, int ratio_h) OVERRIDE;
269
270  virtual void OnMessage(rtc::Message* msg) OVERRIDE;
271
272  // Implemented for VideoMediaChannelTest.
273  bool sending() const { return sending_; }
274  uint32 GetDefaultSendChannelSsrc() { return default_send_ssrc_; }
275  bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
276
277 private:
278  void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config,
279                            const StreamParams& sp) const;
280
281  struct VideoCodecSettings {
282    VideoCodecSettings();
283
284    VideoCodec codec;
285    webrtc::FecConfig fec;
286    int rtx_payload_type;
287  };
288
289  // Wrapper for the sender part, this is where the capturer is connected and
290  // frames are then converted from cricket frames to webrtc frames.
291  class WebRtcVideoSendStream : public sigslot::has_slots<> {
292   public:
293    WebRtcVideoSendStream(
294        webrtc::Call* call,
295        WebRtcVideoEncoderFactory2* encoder_factory,
296        const VideoOptions& options,
297        const Settable<VideoCodecSettings>& codec_settings,
298        const StreamParams& sp,
299        const std::vector<webrtc::RtpExtension>& rtp_extensions);
300
301    ~WebRtcVideoSendStream();
302    void SetOptions(const VideoOptions& options);
303    void SetCodec(const VideoCodecSettings& codec);
304    void SetRtpExtensions(
305        const std::vector<webrtc::RtpExtension>& rtp_extensions);
306
307    void InputFrame(VideoCapturer* capturer, const VideoFrame* frame);
308    bool SetCapturer(VideoCapturer* capturer);
309    bool SetVideoFormat(const VideoFormat& format);
310    void MuteStream(bool mute);
311    bool DisconnectCapturer();
312
313    void Start();
314    void Stop();
315
316    VideoSenderInfo GetVideoSenderInfo();
317
318   private:
319    // Parameters needed to reconstruct the underlying stream.
320    // webrtc::VideoSendStream doesn't support setting a lot of options on the
321    // fly, so when those need to be changed we tear down and reconstruct with
322    // similar parameters depending on which options changed etc.
323    struct VideoSendStreamParameters {
324      VideoSendStreamParameters(
325          const webrtc::VideoSendStream::Config& config,
326          const VideoOptions& options,
327          const Settable<VideoCodecSettings>& codec_settings);
328      webrtc::VideoSendStream::Config config;
329      VideoOptions options;
330      Settable<VideoCodecSettings> codec_settings;
331      // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
332      // typically changes when setting a new resolution or reconfiguring
333      // bitrates.
334      webrtc::VideoEncoderConfig encoder_config;
335    };
336
337    void SetCodecAndOptions(const VideoCodecSettings& codec,
338                            const VideoOptions& options)
339        EXCLUSIVE_LOCKS_REQUIRED(lock_);
340    void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_);
341    // When |override_max| is false constrain width/height to codec dimensions.
342    void SetDimensions(int width, int height, bool override_max)
343        EXCLUSIVE_LOCKS_REQUIRED(lock_);
344
345    webrtc::Call* const call_;
346    WebRtcVideoEncoderFactory2* const encoder_factory_;
347
348    rtc::CriticalSection lock_;
349    webrtc::VideoSendStream* stream_ GUARDED_BY(lock_);
350    VideoSendStreamParameters parameters_ GUARDED_BY(lock_);
351
352    VideoCapturer* capturer_ GUARDED_BY(lock_);
353    bool sending_ GUARDED_BY(lock_);
354    bool muted_ GUARDED_BY(lock_);
355    VideoFormat format_ GUARDED_BY(lock_);
356
357    rtc::CriticalSection frame_lock_;
358    webrtc::I420VideoFrame video_frame_ GUARDED_BY(frame_lock_);
359  };
360
361  // Wrapper for the receiver part, contains configs etc. that are needed to
362  // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper
363  // between webrtc::VideoRenderer and cricket::VideoRenderer.
364  class WebRtcVideoReceiveStream : public webrtc::VideoRenderer {
365   public:
366    WebRtcVideoReceiveStream(
367        webrtc::Call*,
368        const webrtc::VideoReceiveStream::Config& config,
369        const std::vector<VideoCodecSettings>& recv_codecs);
370    ~WebRtcVideoReceiveStream();
371
372    void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
373    void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions);
374
375    virtual void RenderFrame(const webrtc::I420VideoFrame& frame,
376                             int time_to_render_ms) OVERRIDE;
377
378    void SetRenderer(cricket::VideoRenderer* renderer);
379    cricket::VideoRenderer* GetRenderer();
380
381    VideoReceiverInfo GetVideoReceiverInfo();
382
383   private:
384    void SetSize(int width, int height);
385    void RecreateWebRtcStream();
386
387    webrtc::Call* const call_;
388
389    webrtc::VideoReceiveStream* stream_;
390    webrtc::VideoReceiveStream::Config config_;
391
392    rtc::CriticalSection renderer_lock_;
393    cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_);
394    int last_width_ GUARDED_BY(renderer_lock_);
395    int last_height_ GUARDED_BY(renderer_lock_);
396  };
397
398  void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine);
399  void SetDefaultOptions();
400
401  virtual bool SendRtp(const uint8_t* data, size_t len) OVERRIDE;
402  virtual bool SendRtcp(const uint8_t* data, size_t len) OVERRIDE;
403
404  void StartAllSendStreams();
405  void StopAllSendStreams();
406
407  static std::vector<VideoCodecSettings> MapCodecs(
408      const std::vector<VideoCodec>& codecs);
409  std::vector<VideoCodecSettings> FilterSupportedCodecs(
410      const std::vector<VideoCodecSettings>& mapped_codecs);
411
412  void FillSenderStats(VideoMediaInfo* info);
413  void FillReceiverStats(VideoMediaInfo* info);
414  void FillBandwidthEstimationStats(VideoMediaInfo* info);
415
416  uint32_t rtcp_receiver_report_ssrc_;
417  bool sending_;
418  rtc::scoped_ptr<webrtc::Call> call_;
419  uint32_t default_send_ssrc_;
420
421  DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
422  UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
423
424  // Using primary-ssrc (first ssrc) as key.
425  std::map<uint32, WebRtcVideoSendStream*> send_streams_;
426  std::map<uint32, WebRtcVideoReceiveStream*> receive_streams_;
427
428  Settable<VideoCodecSettings> send_codec_;
429  std::vector<webrtc::RtpExtension> send_rtp_extensions_;
430
431  WebRtcVideoEncoderFactory2* const encoder_factory_;
432  std::vector<VideoCodecSettings> recv_codecs_;
433  std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
434  VideoOptions options_;
435};
436
437}  // namespace cricket
438
439#endif  // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
440