1/*
2 * libjingle
3 * Copyright 2014 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
29#define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
30
31#include <map>
32#include <string>
33#include <vector>
34
35#include "talk/media/base/mediaengine.h"
36#include "talk/media/webrtc/webrtcvideochannelfactory.h"
37#include "talk/media/webrtc/webrtcvideodecoderfactory.h"
38#include "talk/media/webrtc/webrtcvideoencoderfactory.h"
39#include "webrtc/base/criticalsection.h"
40#include "webrtc/base/scoped_ptr.h"
41#include "webrtc/base/thread_annotations.h"
42#include "webrtc/base/thread_checker.h"
43#include "webrtc/call.h"
44#include "webrtc/transport.h"
45#include "webrtc/video_frame.h"
46#include "webrtc/video_receive_stream.h"
47#include "webrtc/video_renderer.h"
48#include "webrtc/video_send_stream.h"
49
50namespace webrtc {
51class VideoDecoder;
52class VideoEncoder;
53}
54
55namespace rtc {
56class Thread;
57}  // namespace rtc
58
59namespace cricket {
60
61class VideoCapturer;
62class VideoFrame;
63class VideoProcessor;
64class VideoRenderer;
65class VoiceMediaChannel;
66class WebRtcDecoderObserver;
67class WebRtcEncoderObserver;
68class WebRtcLocalStreamInfo;
69class WebRtcRenderAdapter;
70class WebRtcVideoChannelRecvInfo;
71class WebRtcVideoChannelSendInfo;
72class WebRtcVoiceEngine;
73class WebRtcVoiceMediaChannel;
74
75struct CapturedFrame;
76struct Device;
77
78// Exposed here for unittests.
79std::vector<VideoCodec> DefaultVideoCodecList();
80
81class UnsignalledSsrcHandler {
82 public:
83  enum Action {
84    kDropPacket,
85    kDeliverPacket,
86  };
87  virtual Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel,
88                                   uint32_t ssrc) = 0;
89};
90
91// TODO(pbos): Remove, use external handlers only.
92class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
93 public:
94  DefaultUnsignalledSsrcHandler();
95  Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel,
96                           uint32_t ssrc) override;
97
98  VideoRenderer* GetDefaultRenderer() const;
99  void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer);
100
101 private:
102  uint32_t default_recv_ssrc_;
103  VideoRenderer* default_renderer_;
104};
105
106// WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667).
107class WebRtcVideoEngine2 {
108 public:
109  WebRtcVideoEngine2();
110  ~WebRtcVideoEngine2();
111
112  // Basic video engine implementation.
113  void Init();
114
115  WebRtcVideoChannel2* CreateChannel(webrtc::Call* call,
116                                     const VideoOptions& options);
117
118  const std::vector<VideoCodec>& codecs() const;
119  RtpCapabilities GetCapabilities() const;
120
121  // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
122  // not take the ownership of |decoder_factory|. The caller needs to make sure
123  // that |decoder_factory| outlives the video engine.
124  void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
125  // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
126  // not take the ownership of |encoder_factory|. The caller needs to make sure
127  // that |encoder_factory| outlives the video engine.
128  virtual void SetExternalEncoderFactory(
129      WebRtcVideoEncoderFactory* encoder_factory);
130
131  bool EnableTimedRender();
132
133  bool FindCodec(const VideoCodec& in);
134  // Check whether the supplied trace should be ignored.
135  bool ShouldIgnoreTrace(const std::string& trace);
136
137 private:
138  std::vector<VideoCodec> GetSupportedCodecs() const;
139
140  std::vector<VideoCodec> video_codecs_;
141
142  bool initialized_;
143
144  WebRtcVideoDecoderFactory* external_decoder_factory_;
145  WebRtcVideoEncoderFactory* external_encoder_factory_;
146  rtc::scoped_ptr<WebRtcVideoEncoderFactory> simulcast_encoder_factory_;
147};
148
149class WebRtcVideoChannel2 : public rtc::MessageHandler,
150                            public VideoMediaChannel,
151                            public webrtc::Transport,
152                            public webrtc::LoadObserver {
153 public:
154  WebRtcVideoChannel2(webrtc::Call* call,
155                      const VideoOptions& options,
156                      const std::vector<VideoCodec>& recv_codecs,
157                      WebRtcVideoEncoderFactory* external_encoder_factory,
158                      WebRtcVideoDecoderFactory* external_decoder_factory);
159  ~WebRtcVideoChannel2() override;
160
161  // VideoMediaChannel implementation
162  bool SetSendParameters(const VideoSendParameters& params) override;
163  bool SetRecvParameters(const VideoRecvParameters& params) override;
164  bool GetSendCodec(VideoCodec* send_codec) override;
165  bool SetSendStreamFormat(uint32_t ssrc, const VideoFormat& format) override;
166  bool SetSend(bool send) override;
167  bool SetVideoSend(uint32_t ssrc,
168                    bool mute,
169                    const VideoOptions* options) override;
170  bool AddSendStream(const StreamParams& sp) override;
171  bool RemoveSendStream(uint32_t ssrc) override;
172  bool AddRecvStream(const StreamParams& sp) override;
173  bool AddRecvStream(const StreamParams& sp, bool default_stream);
174  bool RemoveRecvStream(uint32_t ssrc) override;
175  bool SetRenderer(uint32_t ssrc, VideoRenderer* renderer) override;
176  bool GetStats(VideoMediaInfo* info) override;
177  bool SetCapturer(uint32_t ssrc, VideoCapturer* capturer) override;
178  bool SendIntraFrame() override;
179  bool RequestIntraFrame() override;
180
181  void OnPacketReceived(rtc::Buffer* packet,
182                        const rtc::PacketTime& packet_time) override;
183  void OnRtcpReceived(rtc::Buffer* packet,
184                      const rtc::PacketTime& packet_time) override;
185  void OnReadyToSend(bool ready) override;
186  void SetInterface(NetworkInterface* iface) override;
187  void UpdateAspectRatio(int ratio_w, int ratio_h) override;
188
189  void OnMessage(rtc::Message* msg) override;
190
191  void OnLoadUpdate(Load load) override;
192
193  // Implemented for VideoMediaChannelTest.
194  bool sending() const { return sending_; }
195  uint32_t GetDefaultSendChannelSsrc() { return default_send_ssrc_; }
196  bool GetRenderer(uint32_t ssrc, VideoRenderer** renderer);
197
198 private:
199  bool MuteStream(uint32_t ssrc, bool mute);
200  class WebRtcVideoReceiveStream;
201
202  bool SetSendCodecs(const std::vector<VideoCodec>& codecs);
203  bool SetSendRtpHeaderExtensions(
204      const std::vector<RtpHeaderExtension>& extensions);
205  bool SetMaxSendBandwidth(int bps);
206  bool SetOptions(const VideoOptions& options);
207  bool SetRecvCodecs(const std::vector<VideoCodec>& codecs);
208  bool SetRecvRtpHeaderExtensions(
209      const std::vector<RtpHeaderExtension>& extensions);
210
211  void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config,
212                            const StreamParams& sp) const;
213  bool CodecIsExternallySupported(const std::string& name) const;
214  bool ValidateSendSsrcAvailability(const StreamParams& sp) const
215      EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
216  bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
217      EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
218  void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
219      EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
220
221  struct VideoCodecSettings {
222    VideoCodecSettings();
223
224    bool operator==(const VideoCodecSettings& other) const;
225    bool operator!=(const VideoCodecSettings& other) const;
226
227    VideoCodec codec;
228    webrtc::FecConfig fec;
229    int rtx_payload_type;
230  };
231
232  static std::string CodecSettingsVectorToString(
233      const std::vector<VideoCodecSettings>& codecs);
234
235  // Wrapper for the sender part, this is where the capturer is connected and
236  // frames are then converted from cricket frames to webrtc frames.
237  class WebRtcVideoSendStream : public sigslot::has_slots<> {
238   public:
239    WebRtcVideoSendStream(
240        webrtc::Call* call,
241        const StreamParams& sp,
242        const webrtc::VideoSendStream::Config& config,
243        WebRtcVideoEncoderFactory* external_encoder_factory,
244        const VideoOptions& options,
245        int max_bitrate_bps,
246        const rtc::Optional<VideoCodecSettings>& codec_settings,
247        const std::vector<webrtc::RtpExtension>& rtp_extensions,
248        const VideoSendParameters& send_params);
249    ~WebRtcVideoSendStream();
250
251    void SetOptions(const VideoOptions& options);
252    void SetCodec(const VideoCodecSettings& codec);
253    void SetRtpExtensions(
254        const std::vector<webrtc::RtpExtension>& rtp_extensions);
255    // TODO(deadbeef): Move logic from SetCodec/SetRtpExtensions/etc.
256    // into this method. Currently this method only sets the RTCP mode.
257    void SetSendParameters(const VideoSendParameters& send_params);
258
259    void InputFrame(VideoCapturer* capturer, const VideoFrame* frame);
260    bool SetCapturer(VideoCapturer* capturer);
261    bool SetVideoFormat(const VideoFormat& format);
262    void MuteStream(bool mute);
263    bool DisconnectCapturer();
264
265    void SetApplyRotation(bool apply_rotation);
266
267    void Start();
268    void Stop();
269
270    const std::vector<uint32_t>& GetSsrcs() const;
271    VideoSenderInfo GetVideoSenderInfo();
272    void FillBandwidthEstimationInfo(BandwidthEstimationInfo* bwe_info);
273
274    void SetMaxBitrateBps(int max_bitrate_bps);
275
276   private:
277    // Parameters needed to reconstruct the underlying stream.
278    // webrtc::VideoSendStream doesn't support setting a lot of options on the
279    // fly, so when those need to be changed we tear down and reconstruct with
280    // similar parameters depending on which options changed etc.
281    struct VideoSendStreamParameters {
282      VideoSendStreamParameters(
283          const webrtc::VideoSendStream::Config& config,
284          const VideoOptions& options,
285          int max_bitrate_bps,
286          const rtc::Optional<VideoCodecSettings>& codec_settings);
287      webrtc::VideoSendStream::Config config;
288      VideoOptions options;
289      int max_bitrate_bps;
290      rtc::Optional<VideoCodecSettings> codec_settings;
291      // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
292      // typically changes when setting a new resolution or reconfiguring
293      // bitrates.
294      webrtc::VideoEncoderConfig encoder_config;
295    };
296
297    struct AllocatedEncoder {
298      AllocatedEncoder(webrtc::VideoEncoder* encoder,
299                       webrtc::VideoCodecType type,
300                       bool external);
301      webrtc::VideoEncoder* encoder;
302      webrtc::VideoEncoder* external_encoder;
303      webrtc::VideoCodecType type;
304      bool external;
305    };
306
307    struct Dimensions {
308      // Initial encoder configuration (QCIF, 176x144) frame (to ensure that
309      // hardware encoders can be initialized). This gives us low memory usage
310      // but also makes it so configuration errors are discovered at the time we
311      // apply the settings rather than when we get the first frame (waiting for
312      // the first frame to know that you gave a bad codec parameter could make
313      // debugging hard).
314      // TODO(pbos): Consider setting up encoders lazily.
315      Dimensions() : width(176), height(144), is_screencast(false) {}
316      int width;
317      int height;
318      bool is_screencast;
319    };
320
321    union VideoEncoderSettings {
322      webrtc::VideoCodecVP8 vp8;
323      webrtc::VideoCodecVP9 vp9;
324    };
325
326    static std::vector<webrtc::VideoStream> CreateVideoStreams(
327        const VideoCodec& codec,
328        const VideoOptions& options,
329        int max_bitrate_bps,
330        size_t num_streams);
331    static std::vector<webrtc::VideoStream> CreateSimulcastVideoStreams(
332        const VideoCodec& codec,
333        const VideoOptions& options,
334        int max_bitrate_bps,
335        size_t num_streams);
336
337    void* ConfigureVideoEncoderSettings(const VideoCodec& codec,
338                                        const VideoOptions& options,
339                                        bool is_screencast)
340        EXCLUSIVE_LOCKS_REQUIRED(lock_);
341
342    AllocatedEncoder CreateVideoEncoder(const VideoCodec& codec)
343        EXCLUSIVE_LOCKS_REQUIRED(lock_);
344    void DestroyVideoEncoder(AllocatedEncoder* encoder)
345        EXCLUSIVE_LOCKS_REQUIRED(lock_);
346    void SetCodecAndOptions(const VideoCodecSettings& codec,
347                            const VideoOptions& options)
348        EXCLUSIVE_LOCKS_REQUIRED(lock_);
349    void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_);
350    webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
351        const Dimensions& dimensions,
352        const VideoCodec& codec) const EXCLUSIVE_LOCKS_REQUIRED(lock_);
353    void SetDimensions(int width, int height, bool is_screencast)
354        EXCLUSIVE_LOCKS_REQUIRED(lock_);
355
356    const std::vector<uint32_t> ssrcs_;
357    const std::vector<SsrcGroup> ssrc_groups_;
358    webrtc::Call* const call_;
359    WebRtcVideoEncoderFactory* const external_encoder_factory_
360        GUARDED_BY(lock_);
361
362    rtc::CriticalSection lock_;
363    webrtc::VideoSendStream* stream_ GUARDED_BY(lock_);
364    VideoSendStreamParameters parameters_ GUARDED_BY(lock_);
365    VideoEncoderSettings encoder_settings_ GUARDED_BY(lock_);
366    AllocatedEncoder allocated_encoder_ GUARDED_BY(lock_);
367    Dimensions last_dimensions_ GUARDED_BY(lock_);
368
369    VideoCapturer* capturer_ GUARDED_BY(lock_);
370    bool sending_ GUARDED_BY(lock_);
371    bool muted_ GUARDED_BY(lock_);
372    VideoFormat format_ GUARDED_BY(lock_);
373    int old_adapt_changes_ GUARDED_BY(lock_);
374
375    // The timestamp of the first frame received
376    // Used to generate the timestamps of subsequent frames
377    int64_t first_frame_timestamp_ms_ GUARDED_BY(lock_);
378
379    // The timestamp of the last frame received
380    // Used to generate timestamp for the black frame when capturer is removed
381    int64_t last_frame_timestamp_ms_ GUARDED_BY(lock_);
382  };
383
384  // Wrapper for the receiver part, contains configs etc. that are needed to
385  // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper
386  // between webrtc::VideoRenderer and cricket::VideoRenderer.
387  class WebRtcVideoReceiveStream : public webrtc::VideoRenderer {
388   public:
389    WebRtcVideoReceiveStream(
390        webrtc::Call* call,
391        const StreamParams& sp,
392        const webrtc::VideoReceiveStream::Config& config,
393        WebRtcVideoDecoderFactory* external_decoder_factory,
394        bool default_stream,
395        const std::vector<VideoCodecSettings>& recv_codecs,
396        bool disable_prerenderer_smoothing);
397    ~WebRtcVideoReceiveStream();
398
399    const std::vector<uint32_t>& GetSsrcs() const;
400
401    void SetLocalSsrc(uint32_t local_ssrc);
402    void SetFeedbackParameters(bool nack_enabled,
403                               bool remb_enabled,
404                               bool transport_cc_enabled);
405    void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
406    void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions);
407    // TODO(deadbeef): Move logic from SetRecvCodecs/SetRtpExtensions/etc.
408    // into this method. Currently this method only sets the RTCP mode.
409    void SetRecvParameters(const VideoRecvParameters& recv_params);
410
411    void RenderFrame(const webrtc::VideoFrame& frame,
412                     int time_to_render_ms) override;
413    bool IsTextureSupported() const override;
414    bool SmoothsRenderedFrames() const override;
415    bool IsDefaultStream() const;
416
417    void SetRenderer(cricket::VideoRenderer* renderer);
418    cricket::VideoRenderer* GetRenderer();
419
420    VideoReceiverInfo GetVideoReceiverInfo();
421
422   private:
423    struct AllocatedDecoder {
424      AllocatedDecoder(webrtc::VideoDecoder* decoder,
425                       webrtc::VideoCodecType type,
426                       bool external);
427      webrtc::VideoDecoder* decoder;
428      // Decoder wrapped into a fallback decoder to permit software fallback.
429      webrtc::VideoDecoder* external_decoder;
430      webrtc::VideoCodecType type;
431      bool external;
432    };
433
434    void SetSize(int width, int height);
435    void RecreateWebRtcStream();
436
437    AllocatedDecoder CreateOrReuseVideoDecoder(
438        std::vector<AllocatedDecoder>* old_decoder,
439        const VideoCodec& codec);
440    void ClearDecoders(std::vector<AllocatedDecoder>* allocated_decoders);
441
442    std::string GetCodecNameFromPayloadType(int payload_type);
443
444    webrtc::Call* const call_;
445    const std::vector<uint32_t> ssrcs_;
446    const std::vector<SsrcGroup> ssrc_groups_;
447
448    webrtc::VideoReceiveStream* stream_;
449    const bool default_stream_;
450    webrtc::VideoReceiveStream::Config config_;
451
452    WebRtcVideoDecoderFactory* const external_decoder_factory_;
453    std::vector<AllocatedDecoder> allocated_decoders_;
454
455    const bool disable_prerenderer_smoothing_;
456
457    rtc::CriticalSection renderer_lock_;
458    cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_);
459    int last_width_ GUARDED_BY(renderer_lock_);
460    int last_height_ GUARDED_BY(renderer_lock_);
461    // Expands remote RTP timestamps to int64_t to be able to estimate how long
462    // the stream has been running.
463    rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
464        GUARDED_BY(renderer_lock_);
465    int64_t first_frame_timestamp_ GUARDED_BY(renderer_lock_);
466    // Start NTP time is estimated as current remote NTP time (estimated from
467    // RTCP) minus the elapsed time, as soon as remote NTP time is available.
468    int64_t estimated_remote_start_ntp_time_ms_ GUARDED_BY(renderer_lock_);
469  };
470
471  void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine);
472  void SetDefaultOptions();
473
474  bool SendRtp(const uint8_t* data,
475               size_t len,
476               const webrtc::PacketOptions& options) override;
477  bool SendRtcp(const uint8_t* data, size_t len) override;
478
479  void StartAllSendStreams();
480  void StopAllSendStreams();
481
482  static std::vector<VideoCodecSettings> MapCodecs(
483      const std::vector<VideoCodec>& codecs);
484  std::vector<VideoCodecSettings> FilterSupportedCodecs(
485      const std::vector<VideoCodecSettings>& mapped_codecs) const;
486  static bool ReceiveCodecsHaveChanged(std::vector<VideoCodecSettings> before,
487                                       std::vector<VideoCodecSettings> after);
488
489  void FillSenderStats(VideoMediaInfo* info);
490  void FillReceiverStats(VideoMediaInfo* info);
491  void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
492                                    VideoMediaInfo* info);
493
494  rtc::ThreadChecker thread_checker_;
495
496  uint32_t rtcp_receiver_report_ssrc_;
497  bool sending_;
498  webrtc::Call* const call_;
499
500  uint32_t default_send_ssrc_;
501
502  DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
503  UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
504
505  // Separate list of set capturers used to signal CPU adaptation. These should
506  // not be locked while calling methods that take other locks to prevent
507  // lock-order inversions.
508  rtc::CriticalSection capturer_crit_;
509  bool signal_cpu_adaptation_ GUARDED_BY(capturer_crit_);
510  std::map<uint32_t, VideoCapturer*> capturers_ GUARDED_BY(capturer_crit_);
511
512  rtc::CriticalSection stream_crit_;
513  // Using primary-ssrc (first ssrc) as key.
514  std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
515      GUARDED_BY(stream_crit_);
516  std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
517      GUARDED_BY(stream_crit_);
518  std::set<uint32_t> send_ssrcs_ GUARDED_BY(stream_crit_);
519  std::set<uint32_t> receive_ssrcs_ GUARDED_BY(stream_crit_);
520
521  rtc::Optional<VideoCodecSettings> send_codec_;
522  std::vector<webrtc::RtpExtension> send_rtp_extensions_;
523
524  WebRtcVideoEncoderFactory* const external_encoder_factory_;
525  WebRtcVideoDecoderFactory* const external_decoder_factory_;
526  std::vector<VideoCodecSettings> recv_codecs_;
527  std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
528  webrtc::Call::Config::BitrateConfig bitrate_config_;
529  VideoOptions options_;
530  // TODO(deadbeef): Don't duplicate information between
531  // send_params/recv_params, rtp_extensions, options, etc.
532  VideoSendParameters send_params_;
533  VideoRecvParameters recv_params_;
534};
535
536}  // namespace cricket
537
538#endif  // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
539