1/*
2 * libjingle
3 * Copyright 2004 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_
29#define TALK_MEDIA_WEBRTCVOICEENGINE_H_
30
31#include <map>
32#include <set>
33#include <string>
34#include <vector>
35
36#include "talk/media/base/rtputils.h"
37#include "talk/media/webrtc/webrtccommon.h"
38#include "talk/media/webrtc/webrtcexport.h"
39#include "talk/media/webrtc/webrtcvoe.h"
40#include "talk/session/media/channel.h"
41#include "webrtc/base/buffer.h"
42#include "webrtc/base/byteorder.h"
43#include "webrtc/base/logging.h"
44#include "webrtc/base/scoped_ptr.h"
45#include "webrtc/base/stream.h"
46#include "webrtc/common.h"
47
48#if !defined(LIBPEERCONNECTION_LIB) && \
49    !defined(LIBPEERCONNECTION_IMPLEMENTATION)
50// If you hit this, then you've tried to include this header from outside
51// the shared library.  An instance of this class must only be created from
52// within the library that actually implements it.  Otherwise use the
53// WebRtcMediaEngine to construct an instance.
54#error "Bogus include."
55#endif
56
57namespace webrtc {
58class VideoEngine;
59}
60
61namespace cricket {
62
63// WebRtcSoundclipStream is an adapter object that allows a memory stream to be
64// passed into WebRtc, and support looping.
65class WebRtcSoundclipStream : public webrtc::InStream {
66 public:
67  WebRtcSoundclipStream(const char* buf, size_t len)
68      : mem_(buf, len), loop_(true) {
69  }
70  void set_loop(bool loop) { loop_ = loop; }
71  virtual int Read(void* buf, int len);
72  virtual int Rewind();
73
74 private:
75  rtc::MemoryStream mem_;
76  bool loop_;
77};
78
79// WebRtcMonitorStream is used to monitor a stream coming from WebRtc.
80// For now we just dump the data.
81class WebRtcMonitorStream : public webrtc::OutStream {
82  virtual bool Write(const void *buf, int len) {
83    return true;
84  }
85};
86
87class AudioDeviceModule;
88class AudioRenderer;
89class VoETraceWrapper;
90class VoEWrapper;
91class VoiceProcessor;
92class WebRtcSoundclipMedia;
93class WebRtcVoiceMediaChannel;
94
95// WebRtcVoiceEngine is a class to be used with CompositeMediaEngine.
96// It uses the WebRtc VoiceEngine library for audio handling.
97class WebRtcVoiceEngine
98    : public webrtc::VoiceEngineObserver,
99      public webrtc::TraceCallback,
100      public webrtc::VoEMediaProcess  {
101 public:
102  WebRtcVoiceEngine();
103  // Dependency injection for testing.
104  WebRtcVoiceEngine(VoEWrapper* voe_wrapper,
105                    VoEWrapper* voe_wrapper_sc,
106                    VoETraceWrapper* tracing);
107  ~WebRtcVoiceEngine();
108  bool Init(rtc::Thread* worker_thread);
109  void Terminate();
110
111  int GetCapabilities();
112  VoiceMediaChannel* CreateChannel();
113
114  SoundclipMedia* CreateSoundclip();
115
116  AudioOptions GetOptions() const { return options_; }
117  bool SetOptions(const AudioOptions& options);
118  // Overrides, when set, take precedence over the options on a
119  // per-option basis.  For example, if AGC is set in options and AEC
120  // is set in overrides, AGC and AEC will be both be set.  Overrides
121  // can also turn off options.  For example, if AGC is set to "on" in
122  // options and AGC is set to "off" in overrides, the result is that
123  // AGC will be off until different overrides are applied or until
124  // the overrides are cleared.  Only one set of overrides is present
125  // at a time (they do not "stack").  And when the overrides are
126  // cleared, the media engine's state reverts back to the options set
127  // via SetOptions.  This allows us to have both "persistent options"
128  // (the normal options) and "temporary options" (overrides).
129  bool SetOptionOverrides(const AudioOptions& options);
130  bool ClearOptionOverrides();
131  bool SetDelayOffset(int offset);
132  bool SetDevices(const Device* in_device, const Device* out_device);
133  bool GetOutputVolume(int* level);
134  bool SetOutputVolume(int level);
135  int GetInputLevel();
136  bool SetLocalMonitor(bool enable);
137
138  const std::vector<AudioCodec>& codecs();
139  bool FindCodec(const AudioCodec& codec);
140  bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec);
141
142  const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
143
144  void SetLogging(int min_sev, const char* filter);
145
146  bool RegisterProcessor(uint32 ssrc,
147                         VoiceProcessor* voice_processor,
148                         MediaProcessorDirection direction);
149  bool UnregisterProcessor(uint32 ssrc,
150                           VoiceProcessor* voice_processor,
151                           MediaProcessorDirection direction);
152
153  // Method from webrtc::VoEMediaProcess
154  virtual void Process(int channel,
155                       webrtc::ProcessingTypes type,
156                       int16_t audio10ms[],
157                       int length,
158                       int sampling_freq,
159                       bool is_stereo);
160
161  // For tracking WebRtc channels. Needed because we have to pause them
162  // all when switching devices.
163  // May only be called by WebRtcVoiceMediaChannel.
164  void RegisterChannel(WebRtcVoiceMediaChannel *channel);
165  void UnregisterChannel(WebRtcVoiceMediaChannel *channel);
166
167  // May only be called by WebRtcSoundclipMedia.
168  void RegisterSoundclip(WebRtcSoundclipMedia *channel);
169  void UnregisterSoundclip(WebRtcSoundclipMedia *channel);
170
171  // Called by WebRtcVoiceMediaChannel to set a gain offset from
172  // the default AGC target level.
173  bool AdjustAgcLevel(int delta);
174
175  VoEWrapper* voe() { return voe_wrapper_.get(); }
176  VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); }
177  int GetLastEngineError();
178
179  // Set the external ADMs. This can only be called before Init.
180  bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm,
181                            webrtc::AudioDeviceModule* adm_sc);
182
183  // Starts AEC dump using existing file.
184  bool StartAecDump(rtc::PlatformFile file);
185
186  // Check whether the supplied trace should be ignored.
187  bool ShouldIgnoreTrace(const std::string& trace);
188
189  // Create a VoiceEngine Channel.
190  int CreateMediaVoiceChannel();
191  int CreateSoundclipVoiceChannel();
192
193 private:
194  typedef std::vector<WebRtcSoundclipMedia *> SoundclipList;
195  typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList;
196  typedef sigslot::
197      signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal;
198
199  void Construct();
200  void ConstructCodecs();
201  bool InitInternal();
202  bool EnsureSoundclipEngineInit();
203  void SetTraceFilter(int filter);
204  void SetTraceOptions(const std::string& options);
205  // Applies either options or overrides.  Every option that is "set"
206  // will be applied.  Every option not "set" will be ignored.  This
207  // allows us to selectively turn on and off different options easily
208  // at any time.
209  bool ApplyOptions(const AudioOptions& options);
210  virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
211  virtual void CallbackOnError(int channel, int errCode);
212  // Given the device type, name, and id, find device id. Return true and
213  // set the output parameter rtc_id if successful.
214  bool FindWebRtcAudioDeviceId(
215      bool is_input, const std::string& dev_name, int dev_id, int* rtc_id);
216  bool FindChannelAndSsrc(int channel_num,
217                          WebRtcVoiceMediaChannel** channel,
218                          uint32* ssrc) const;
219  bool FindChannelNumFromSsrc(uint32 ssrc,
220                              MediaProcessorDirection direction,
221                              int* channel_num);
222  bool ChangeLocalMonitor(bool enable);
223  bool PauseLocalMonitor();
224  bool ResumeLocalMonitor();
225
226  bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction,
227                                  uint32 ssrc,
228                                  VoiceProcessor* voice_processor,
229                                  MediaProcessorDirection processor_direction);
230
231  void StartAecDump(const std::string& filename);
232  void StopAecDump();
233  int CreateVoiceChannel(VoEWrapper* voe);
234
235  // When a voice processor registers with the engine, it is connected
236  // to either the Rx or Tx signals, based on the direction parameter.
237  // SignalXXMediaFrame will be invoked for every audio packet.
238  FrameSignal SignalRxMediaFrame;
239  FrameSignal SignalTxMediaFrame;
240
241  static const int kDefaultLogSeverity = rtc::LS_WARNING;
242
243  // The primary instance of WebRtc VoiceEngine.
244  rtc::scoped_ptr<VoEWrapper> voe_wrapper_;
245  // A secondary instance, for playing out soundclips (on the 'ring' device).
246  rtc::scoped_ptr<VoEWrapper> voe_wrapper_sc_;
247  bool voe_wrapper_sc_initialized_;
248  rtc::scoped_ptr<VoETraceWrapper> tracing_;
249  // The external audio device manager
250  webrtc::AudioDeviceModule* adm_;
251  webrtc::AudioDeviceModule* adm_sc_;
252  int log_filter_;
253  std::string log_options_;
254  bool is_dumping_aec_;
255  std::vector<AudioCodec> codecs_;
256  std::vector<RtpHeaderExtension> rtp_header_extensions_;
257  bool desired_local_monitor_enable_;
258  rtc::scoped_ptr<WebRtcMonitorStream> monitor_;
259  SoundclipList soundclips_;
260  ChannelList channels_;
261  // channels_ can be read from WebRtc callback thread. We need a lock on that
262  // callback as well as the RegisterChannel/UnregisterChannel.
263  rtc::CriticalSection channels_cs_;
264  webrtc::AgcConfig default_agc_config_;
265
266  webrtc::Config voe_config_;
267
268  bool initialized_;
269  // See SetOptions and SetOptionOverrides for a description of the
270  // difference between options and overrides.
271  // options_ are the base options, which combined with the
272  // option_overrides_, create the current options being used.
273  // options_ is stored so that when option_overrides_ is cleared, we
274  // can restore the options_ without the option_overrides.
275  AudioOptions options_;
276  AudioOptions option_overrides_;
277
278  // When the media processor registers with the engine, the ssrc is cached
279  // here so that a look up need not be made when the callback is invoked.
280  // This is necessary because the lookup results in mux_channels_cs lock being
281  // held and if a remote participant leaves the hangout at the same time
282  // we hit a deadlock.
283  uint32 tx_processor_ssrc_;
284  uint32 rx_processor_ssrc_;
285
286  rtc::CriticalSection signal_media_critical_;
287
288  // Cache received experimental_aec and experimental_ns values, and apply them
289  // in case they are missing in the audio options. We need to do this because
290  // SetExtraOptions() will revert to defaults for options which are not
291  // provided.
292  Settable<bool> experimental_aec_;
293  Settable<bool> experimental_ns_;
294};
295
296// WebRtcMediaChannel is a class that implements the common WebRtc channel
297// functionality.
298template <class T, class E>
299class WebRtcMediaChannel : public T, public webrtc::Transport {
300 public:
301  WebRtcMediaChannel(E *engine, int channel)
302      : engine_(engine), voe_channel_(channel) {}
303  E *engine() { return engine_; }
304  int voe_channel() const { return voe_channel_; }
305  bool valid() const { return voe_channel_ != -1; }
306
307 protected:
308  // implements Transport interface
309  virtual int SendPacket(int channel, const void *data, int len) {
310    rtc::Buffer packet(data, len, kMaxRtpPacketLen);
311    if (!T::SendPacket(&packet)) {
312      return -1;
313    }
314    return len;
315  }
316
317  virtual int SendRTCPPacket(int channel, const void *data, int len) {
318    rtc::Buffer packet(data, len, kMaxRtpPacketLen);
319    return T::SendRtcp(&packet) ? len : -1;
320  }
321
322 private:
323  E *engine_;
324  int voe_channel_;
325};
326
327// WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses
328// WebRtc Voice Engine.
329class WebRtcVoiceMediaChannel
330    : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> {
331 public:
332  explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine);
333  virtual ~WebRtcVoiceMediaChannel();
334  virtual bool SetOptions(const AudioOptions& options);
335  virtual bool GetOptions(AudioOptions* options) const {
336    *options = options_;
337    return true;
338  }
339  virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs);
340  virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs);
341  virtual bool SetRecvRtpHeaderExtensions(
342      const std::vector<RtpHeaderExtension>& extensions);
343  virtual bool SetSendRtpHeaderExtensions(
344      const std::vector<RtpHeaderExtension>& extensions);
345  virtual bool SetPlayout(bool playout);
346  bool PausePlayout();
347  bool ResumePlayout();
348  virtual bool SetSend(SendFlags send);
349  bool PauseSend();
350  bool ResumeSend();
351  virtual bool AddSendStream(const StreamParams& sp);
352  virtual bool RemoveSendStream(uint32 ssrc);
353  virtual bool AddRecvStream(const StreamParams& sp);
354  virtual bool RemoveRecvStream(uint32 ssrc);
355  virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer);
356  virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer);
357  virtual bool GetActiveStreams(AudioInfo::StreamList* actives);
358  virtual int GetOutputLevel();
359  virtual int GetTimeSinceLastTyping();
360  virtual void SetTypingDetectionParameters(int time_window,
361      int cost_per_typing, int reporting_threshold, int penalty_decay,
362      int type_event_delay);
363  virtual bool SetOutputScaling(uint32 ssrc, double left, double right);
364  virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right);
365
366  virtual bool SetRingbackTone(const char *buf, int len);
367  virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop);
368  virtual bool CanInsertDtmf();
369  virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags);
370
371  virtual void OnPacketReceived(rtc::Buffer* packet,
372                                const rtc::PacketTime& packet_time);
373  virtual void OnRtcpReceived(rtc::Buffer* packet,
374                              const rtc::PacketTime& packet_time);
375  virtual void OnReadyToSend(bool ready) {}
376  virtual bool MuteStream(uint32 ssrc, bool on);
377  virtual bool SetStartSendBandwidth(int bps);
378  virtual bool SetMaxSendBandwidth(int bps);
379  virtual bool GetStats(VoiceMediaInfo* info);
380  // Gets last reported error from WebRtc voice engine.  This should be only
381  // called in response a failure.
382  virtual void GetLastMediaError(uint32* ssrc,
383                                 VoiceMediaChannel::Error* error);
384  bool FindSsrc(int channel_num, uint32* ssrc);
385  void OnError(uint32 ssrc, int error);
386
387  bool sending() const { return send_ != SEND_NOTHING; }
388  int GetReceiveChannelNum(uint32 ssrc);
389  int GetSendChannelNum(uint32 ssrc);
390
391  bool SetupSharedBandwidthEstimation(webrtc::VideoEngine* vie,
392                                      int vie_channel);
393 protected:
394  int GetLastEngineError() { return engine()->GetLastEngineError(); }
395  int GetOutputLevel(int channel);
396  bool GetRedSendCodec(const AudioCodec& red_codec,
397                       const std::vector<AudioCodec>& all_codecs,
398                       webrtc::CodecInst* send_codec);
399  bool EnableRtcp(int channel);
400  bool ResetRecvCodecs(int channel);
401  bool SetPlayout(int channel, bool playout);
402  static uint32 ParseSsrc(const void* data, size_t len, bool rtcp);
403  static Error WebRtcErrorToChannelError(int err_code);
404
405 private:
406  class WebRtcVoiceChannelRenderer;
407  // Map of ssrc to WebRtcVoiceChannelRenderer object.  A new object of
408  // WebRtcVoiceChannelRenderer will be created for every new stream and
409  // will be destroyed when the stream goes away.
410  typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap;
411  typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool,
412      unsigned char);
413
414  void SetNack(int channel, bool nack_enabled);
415  void SetNack(const ChannelMap& channels, bool nack_enabled);
416  bool SetSendCodec(const webrtc::CodecInst& send_codec);
417  bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec);
418  bool ChangePlayout(bool playout);
419  bool ChangeSend(SendFlags send);
420  bool ChangeSend(int channel, SendFlags send);
421  void ConfigureSendChannel(int channel);
422  bool ConfigureRecvChannel(int channel);
423  bool DeleteChannel(int channel);
424  bool InConferenceMode() const {
425    return options_.conference_mode.GetWithDefaultIfUnset(false);
426  }
427  bool IsDefaultChannel(int channel_id) const {
428    return channel_id == voe_channel();
429  }
430  bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs);
431  bool SetSendBandwidthInternal(int bps);
432
433  bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
434                          const RtpHeaderExtension* extension);
435  bool SetupSharedBweOnChannel(int voe_channel);
436
437  bool SetChannelRecvRtpHeaderExtensions(
438    int channel_id,
439    const std::vector<RtpHeaderExtension>& extensions);
440  bool SetChannelSendRtpHeaderExtensions(
441    int channel_id,
442    const std::vector<RtpHeaderExtension>& extensions);
443
444  rtc::scoped_ptr<WebRtcSoundclipStream> ringback_tone_;
445  std::set<int> ringback_channels_;  // channels playing ringback
446  std::vector<AudioCodec> recv_codecs_;
447  std::vector<AudioCodec> send_codecs_;
448  rtc::scoped_ptr<webrtc::CodecInst> send_codec_;
449  bool send_bw_setting_;
450  int send_bw_bps_;
451  AudioOptions options_;
452  bool dtmf_allowed_;
453  bool desired_playout_;
454  bool nack_enabled_;
455  bool playout_;
456  bool typing_noise_detected_;
457  SendFlags desired_send_;
458  SendFlags send_;
459  // shared_bwe_vie_ and shared_bwe_vie_channel_ together identifies a WebRTC
460  // VideoEngine channel that this voice channel should forward incoming packets
461  // to for Bandwidth Estimation purposes.
462  webrtc::VideoEngine* shared_bwe_vie_;
463  int shared_bwe_vie_channel_;
464
465  // send_channels_ contains the channels which are being used for sending.
466  // When the default channel (voe_channel) is used for sending, it is
467  // contained in send_channels_, otherwise not.
468  ChannelMap send_channels_;
469  std::vector<RtpHeaderExtension> send_extensions_;
470  uint32 default_receive_ssrc_;
471  // Note the default channel (voe_channel()) can reside in both
472  // receive_channels_ and send_channels_ in non-conference mode and in that
473  // case it will only be there if a non-zero default_receive_ssrc_ is set.
474  ChannelMap receive_channels_;  // for multiple sources
475  // receive_channels_ can be read from WebRtc callback thread.  Access from
476  // the WebRtc thread must be synchronized with edits on the worker thread.
477  // Reads on the worker thread are ok.
478  //
479  std::vector<RtpHeaderExtension> receive_extensions_;
480  // Do not lock this on the VoE media processor thread; potential for deadlock
481  // exists.
482  mutable rtc::CriticalSection receive_channels_cs_;
483};
484
485}  // namespace cricket
486
487#endif  // TALK_MEDIA_WEBRTCVOICEENGINE_H_
488