webrtcvoiceengine.h revision 3cefbc99f4cc2db744cb130ca629768401a59eb4 
1/*
2 * libjingle
3 * Copyright 2004 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_
29#define TALK_MEDIA_WEBRTCVOICEENGINE_H_
30
31#include <map>
32#include <set>
33#include <string>
34#include <vector>
35
36#include "talk/media/base/rtputils.h"
37#include "talk/media/webrtc/webrtccommon.h"
38#include "talk/media/webrtc/webrtcexport.h"
39#include "talk/media/webrtc/webrtcvoe.h"
40#include "talk/session/media/channel.h"
41#include "webrtc/base/buffer.h"
42#include "webrtc/base/byteorder.h"
43#include "webrtc/base/logging.h"
44#include "webrtc/base/scoped_ptr.h"
45#include "webrtc/base/stream.h"
46#include "webrtc/common.h"
47
48#if !defined(LIBPEERCONNECTION_LIB) && \
49    !defined(LIBPEERCONNECTION_IMPLEMENTATION)
50// If you hit this, then you've tried to include this header from outside
51// the shared library.  An instance of this class must only be created from
52// within the library that actually implements it.  Otherwise use the
53// WebRtcMediaEngine to construct an instance.
54#error "Bogus include."
55#endif
56
57namespace webrtc {
58class VideoEngine;
59}
60
61namespace cricket {
62
63// WebRtcSoundclipStream is an adapter object that allows a memory stream to be
64// passed into WebRtc, and support looping.
65class WebRtcSoundclipStream : public webrtc::InStream {
66 public:
67  WebRtcSoundclipStream(const char* buf, size_t len)
68      : mem_(buf, len), loop_(true) {
69  }
70  void set_loop(bool loop) { loop_ = loop; }
71
72  virtual int Read(void* buf, int len) OVERRIDE;
73  virtual int Rewind() OVERRIDE;
74
75 private:
76  rtc::MemoryStream mem_;
77  bool loop_;
78};
79
80// WebRtcMonitorStream is used to monitor a stream coming from WebRtc.
81// For now we just dump the data.
82class WebRtcMonitorStream : public webrtc::OutStream {
83  virtual bool Write(const void *buf, int len) OVERRIDE {
84    return true;
85  }
86};
87
88class AudioDeviceModule;
89class AudioRenderer;
90class VoETraceWrapper;
91class VoEWrapper;
92class VoiceProcessor;
93class WebRtcSoundclipMedia;
94class WebRtcVoiceMediaChannel;
95
96// WebRtcVoiceEngine is a class to be used with CompositeMediaEngine.
97// It uses the WebRtc VoiceEngine library for audio handling.
98class WebRtcVoiceEngine
99    : public webrtc::VoiceEngineObserver,
100      public webrtc::TraceCallback,
101      public webrtc::VoEMediaProcess  {
102 public:
103  WebRtcVoiceEngine();
104  // Dependency injection for testing.
105  WebRtcVoiceEngine(VoEWrapper* voe_wrapper,
106                    VoEWrapper* voe_wrapper_sc,
107                    VoETraceWrapper* tracing);
108  ~WebRtcVoiceEngine();
109  bool Init(rtc::Thread* worker_thread);
110  void Terminate();
111
112  int GetCapabilities();
113  VoiceMediaChannel* CreateChannel();
114
115  SoundclipMedia* CreateSoundclip();
116
117  AudioOptions GetOptions() const { return options_; }
118  bool SetOptions(const AudioOptions& options);
119  // Overrides, when set, take precedence over the options on a
120  // per-option basis.  For example, if AGC is set in options and AEC
121  // is set in overrides, AGC and AEC will be both be set.  Overrides
122  // can also turn off options.  For example, if AGC is set to "on" in
123  // options and AGC is set to "off" in overrides, the result is that
124  // AGC will be off until different overrides are applied or until
125  // the overrides are cleared.  Only one set of overrides is present
126  // at a time (they do not "stack").  And when the overrides are
127  // cleared, the media engine's state reverts back to the options set
128  // via SetOptions.  This allows us to have both "persistent options"
129  // (the normal options) and "temporary options" (overrides).
130  bool SetOptionOverrides(const AudioOptions& options);
131  bool ClearOptionOverrides();
132  bool SetDelayOffset(int offset);
133  bool SetDevices(const Device* in_device, const Device* out_device);
134  bool GetOutputVolume(int* level);
135  bool SetOutputVolume(int level);
136  int GetInputLevel();
137  bool SetLocalMonitor(bool enable);
138
139  const std::vector<AudioCodec>& codecs();
140  bool FindCodec(const AudioCodec& codec);
141  bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec);
142
143  const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
144
145  void SetLogging(int min_sev, const char* filter);
146
147  bool RegisterProcessor(uint32 ssrc,
148                         VoiceProcessor* voice_processor,
149                         MediaProcessorDirection direction);
150  bool UnregisterProcessor(uint32 ssrc,
151                           VoiceProcessor* voice_processor,
152                           MediaProcessorDirection direction);
153
154  // Method from webrtc::VoEMediaProcess
155  virtual void Process(int channel,
156                       webrtc::ProcessingTypes type,
157                       int16_t audio10ms[],
158                       int length,
159                       int sampling_freq,
160                       bool is_stereo) OVERRIDE;
161
162  // For tracking WebRtc channels. Needed because we have to pause them
163  // all when switching devices.
164  // May only be called by WebRtcVoiceMediaChannel.
165  void RegisterChannel(WebRtcVoiceMediaChannel *channel);
166  void UnregisterChannel(WebRtcVoiceMediaChannel *channel);
167
168  // May only be called by WebRtcSoundclipMedia.
169  void RegisterSoundclip(WebRtcSoundclipMedia *channel);
170  void UnregisterSoundclip(WebRtcSoundclipMedia *channel);
171
172  // Called by WebRtcVoiceMediaChannel to set a gain offset from
173  // the default AGC target level.
174  bool AdjustAgcLevel(int delta);
175
176  VoEWrapper* voe() { return voe_wrapper_.get(); }
177  VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); }
178  int GetLastEngineError();
179
180  // Set the external ADMs. This can only be called before Init.
181  bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm,
182                            webrtc::AudioDeviceModule* adm_sc);
183
184  // Starts AEC dump using existing file.
185  bool StartAecDump(rtc::PlatformFile file);
186
187  // Check whether the supplied trace should be ignored.
188  bool ShouldIgnoreTrace(const std::string& trace);
189
190  // Create a VoiceEngine Channel.
191  int CreateMediaVoiceChannel();
192  int CreateSoundclipVoiceChannel();
193
194 private:
195  typedef std::vector<WebRtcSoundclipMedia *> SoundclipList;
196  typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList;
197  typedef sigslot::
198      signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal;
199
200  void Construct();
201  void ConstructCodecs();
202  bool InitInternal();
203  bool EnsureSoundclipEngineInit();
204  void SetTraceFilter(int filter);
205  void SetTraceOptions(const std::string& options);
206  // Applies either options or overrides.  Every option that is "set"
207  // will be applied.  Every option not "set" will be ignored.  This
208  // allows us to selectively turn on and off different options easily
209  // at any time.
210  bool ApplyOptions(const AudioOptions& options);
211
212  // webrtc::TraceCallback:
213  virtual void Print(webrtc::TraceLevel level,
214                     const char* trace,
215                     int length) OVERRIDE;
216
217  // webrtc::VoiceEngineObserver:
218  virtual void CallbackOnError(int channel, int errCode) OVERRIDE;
219
220  // Given the device type, name, and id, find device id. Return true and
221  // set the output parameter rtc_id if successful.
222  bool FindWebRtcAudioDeviceId(
223      bool is_input, const std::string& dev_name, int dev_id, int* rtc_id);
224  bool FindChannelAndSsrc(int channel_num,
225                          WebRtcVoiceMediaChannel** channel,
226                          uint32* ssrc) const;
227  bool FindChannelNumFromSsrc(uint32 ssrc,
228                              MediaProcessorDirection direction,
229                              int* channel_num);
230  bool ChangeLocalMonitor(bool enable);
231  bool PauseLocalMonitor();
232  bool ResumeLocalMonitor();
233
234  bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction,
235                                  uint32 ssrc,
236                                  VoiceProcessor* voice_processor,
237                                  MediaProcessorDirection processor_direction);
238
239  void StartAecDump(const std::string& filename);
240  void StopAecDump();
241  int CreateVoiceChannel(VoEWrapper* voe);
242
243  // When a voice processor registers with the engine, it is connected
244  // to either the Rx or Tx signals, based on the direction parameter.
245  // SignalXXMediaFrame will be invoked for every audio packet.
246  FrameSignal SignalRxMediaFrame;
247  FrameSignal SignalTxMediaFrame;
248
249  static const int kDefaultLogSeverity = rtc::LS_WARNING;
250
251  // The primary instance of WebRtc VoiceEngine.
252  rtc::scoped_ptr<VoEWrapper> voe_wrapper_;
253  // A secondary instance, for playing out soundclips (on the 'ring' device).
254  rtc::scoped_ptr<VoEWrapper> voe_wrapper_sc_;
255  bool voe_wrapper_sc_initialized_;
256  rtc::scoped_ptr<VoETraceWrapper> tracing_;
257  // The external audio device manager
258  webrtc::AudioDeviceModule* adm_;
259  webrtc::AudioDeviceModule* adm_sc_;
260  int log_filter_;
261  std::string log_options_;
262  bool is_dumping_aec_;
263  std::vector<AudioCodec> codecs_;
264  std::vector<RtpHeaderExtension> rtp_header_extensions_;
265  bool desired_local_monitor_enable_;
266  rtc::scoped_ptr<WebRtcMonitorStream> monitor_;
267  SoundclipList soundclips_;
268  ChannelList channels_;
269  // channels_ can be read from WebRtc callback thread. We need a lock on that
270  // callback as well as the RegisterChannel/UnregisterChannel.
271  rtc::CriticalSection channels_cs_;
272  webrtc::AgcConfig default_agc_config_;
273
274  webrtc::Config voe_config_;
275
276  bool initialized_;
277  // See SetOptions and SetOptionOverrides for a description of the
278  // difference between options and overrides.
279  // options_ are the base options, which combined with the
280  // option_overrides_, create the current options being used.
281  // options_ is stored so that when option_overrides_ is cleared, we
282  // can restore the options_ without the option_overrides.
283  AudioOptions options_;
284  AudioOptions option_overrides_;
285
286  // When the media processor registers with the engine, the ssrc is cached
287  // here so that a look up need not be made when the callback is invoked.
288  // This is necessary because the lookup results in mux_channels_cs lock being
289  // held and if a remote participant leaves the hangout at the same time
290  // we hit a deadlock.
291  uint32 tx_processor_ssrc_;
292  uint32 rx_processor_ssrc_;
293
294  rtc::CriticalSection signal_media_critical_;
295
296  // Cache received experimental_aec and experimental_ns values, and apply them
297  // in case they are missing in the audio options. We need to do this because
298  // SetExtraOptions() will revert to defaults for options which are not
299  // provided.
300  Settable<bool> experimental_aec_;
301  Settable<bool> experimental_ns_;
302};
303
304// WebRtcMediaChannel is a class that implements the common WebRtc channel
305// functionality.
306template <class T, class E>
307class WebRtcMediaChannel : public T, public webrtc::Transport {
308 public:
309  WebRtcMediaChannel(E *engine, int channel)
310      : engine_(engine), voe_channel_(channel) {}
311  E *engine() { return engine_; }
312  int voe_channel() const { return voe_channel_; }
313  bool valid() const { return voe_channel_ != -1; }
314
315 protected:
316  // implements Transport interface
317  virtual int SendPacket(int channel, const void *data, int len) OVERRIDE {
318    rtc::Buffer packet(data, len, kMaxRtpPacketLen);
319    if (!T::SendPacket(&packet)) {
320      return -1;
321    }
322    return len;
323  }
324
325  virtual int SendRTCPPacket(int channel, const void *data, int len) OVERRIDE {
326    rtc::Buffer packet(data, len, kMaxRtpPacketLen);
327    return T::SendRtcp(&packet) ? len : -1;
328  }
329
330 private:
331  E *engine_;
332  int voe_channel_;
333};
334
335// WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses
336// WebRtc Voice Engine.
337class WebRtcVoiceMediaChannel
338    : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> {
339 public:
340  explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine);
341  virtual ~WebRtcVoiceMediaChannel();
342  virtual bool SetOptions(const AudioOptions& options);
343  virtual bool GetOptions(AudioOptions* options) const {
344    *options = options_;
345    return true;
346  }
347  virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs);
348  virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs);
349  virtual bool SetRecvRtpHeaderExtensions(
350      const std::vector<RtpHeaderExtension>& extensions);
351  virtual bool SetSendRtpHeaderExtensions(
352      const std::vector<RtpHeaderExtension>& extensions);
353  virtual bool SetPlayout(bool playout);
354  bool PausePlayout();
355  bool ResumePlayout();
356  virtual bool SetSend(SendFlags send);
357  bool PauseSend();
358  bool ResumeSend();
359  virtual bool AddSendStream(const StreamParams& sp);
360  virtual bool RemoveSendStream(uint32 ssrc);
361  virtual bool AddRecvStream(const StreamParams& sp);
362  virtual bool RemoveRecvStream(uint32 ssrc);
363  virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer);
364  virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer);
365  virtual bool GetActiveStreams(AudioInfo::StreamList* actives);
366  virtual int GetOutputLevel();
367  virtual int GetTimeSinceLastTyping();
368  virtual void SetTypingDetectionParameters(int time_window,
369      int cost_per_typing, int reporting_threshold, int penalty_decay,
370      int type_event_delay);
371  virtual bool SetOutputScaling(uint32 ssrc, double left, double right);
372  virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right);
373
374  virtual bool SetRingbackTone(const char *buf, int len);
375  virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop);
376  virtual bool CanInsertDtmf();
377  virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags);
378
379  virtual void OnPacketReceived(rtc::Buffer* packet,
380                                const rtc::PacketTime& packet_time);
381  virtual void OnRtcpReceived(rtc::Buffer* packet,
382                              const rtc::PacketTime& packet_time);
383  virtual void OnReadyToSend(bool ready) {}
384  virtual bool MuteStream(uint32 ssrc, bool on);
385  virtual bool SetMaxSendBandwidth(int bps);
386  virtual bool GetStats(VoiceMediaInfo* info);
387  // Gets last reported error from WebRtc voice engine.  This should be only
388  // called in response a failure.
389  virtual void GetLastMediaError(uint32* ssrc,
390                                 VoiceMediaChannel::Error* error);
391  bool FindSsrc(int channel_num, uint32* ssrc);
392  void OnError(uint32 ssrc, int error);
393
394  bool sending() const { return send_ != SEND_NOTHING; }
395  int GetReceiveChannelNum(uint32 ssrc);
396  int GetSendChannelNum(uint32 ssrc);
397
398  bool SetupSharedBandwidthEstimation(webrtc::VideoEngine* vie,
399                                      int vie_channel);
400 protected:
401  int GetLastEngineError() { return engine()->GetLastEngineError(); }
402  int GetOutputLevel(int channel);
403  bool GetRedSendCodec(const AudioCodec& red_codec,
404                       const std::vector<AudioCodec>& all_codecs,
405                       webrtc::CodecInst* send_codec);
406  bool EnableRtcp(int channel);
407  bool ResetRecvCodecs(int channel);
408  bool SetPlayout(int channel, bool playout);
409  static uint32 ParseSsrc(const void* data, size_t len, bool rtcp);
410  static Error WebRtcErrorToChannelError(int err_code);
411
412 private:
413  class WebRtcVoiceChannelRenderer;
414  // Map of ssrc to WebRtcVoiceChannelRenderer object.  A new object of
415  // WebRtcVoiceChannelRenderer will be created for every new stream and
416  // will be destroyed when the stream goes away.
417  typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap;
418  typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool,
419      unsigned char);
420
421  void SetNack(int channel, bool nack_enabled);
422  void SetNack(const ChannelMap& channels, bool nack_enabled);
423  bool SetSendCodec(const webrtc::CodecInst& send_codec);
424  bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec);
425  bool ChangePlayout(bool playout);
426  bool ChangeSend(SendFlags send);
427  bool ChangeSend(int channel, SendFlags send);
428  void ConfigureSendChannel(int channel);
429  bool ConfigureRecvChannel(int channel);
430  bool DeleteChannel(int channel);
431  bool InConferenceMode() const {
432    return options_.conference_mode.GetWithDefaultIfUnset(false);
433  }
434  bool IsDefaultChannel(int channel_id) const {
435    return channel_id == voe_channel();
436  }
437  bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs);
438  bool SetSendBandwidthInternal(int bps);
439
440  bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
441                          const RtpHeaderExtension* extension);
442  bool SetupSharedBweOnChannel(int voe_channel);
443
444  bool SetChannelRecvRtpHeaderExtensions(
445    int channel_id,
446    const std::vector<RtpHeaderExtension>& extensions);
447  bool SetChannelSendRtpHeaderExtensions(
448    int channel_id,
449    const std::vector<RtpHeaderExtension>& extensions);
450
451  rtc::scoped_ptr<WebRtcSoundclipStream> ringback_tone_;
452  std::set<int> ringback_channels_;  // channels playing ringback
453  std::vector<AudioCodec> recv_codecs_;
454  std::vector<AudioCodec> send_codecs_;
455  rtc::scoped_ptr<webrtc::CodecInst> send_codec_;
456  bool send_bw_setting_;
457  int send_bw_bps_;
458  AudioOptions options_;
459  bool dtmf_allowed_;
460  bool desired_playout_;
461  bool nack_enabled_;
462  bool playout_;
463  bool typing_noise_detected_;
464  SendFlags desired_send_;
465  SendFlags send_;
466  // shared_bwe_vie_ and shared_bwe_vie_channel_ together identifies a WebRTC
467  // VideoEngine channel that this voice channel should forward incoming packets
468  // to for Bandwidth Estimation purposes.
469  webrtc::VideoEngine* shared_bwe_vie_;
470  int shared_bwe_vie_channel_;
471
472  // send_channels_ contains the channels which are being used for sending.
473  // When the default channel (voe_channel) is used for sending, it is
474  // contained in send_channels_, otherwise not.
475  ChannelMap send_channels_;
476  std::vector<RtpHeaderExtension> send_extensions_;
477  uint32 default_receive_ssrc_;
478  // Note the default channel (voe_channel()) can reside in both
479  // receive_channels_ and send_channels_ in non-conference mode and in that
480  // case it will only be there if a non-zero default_receive_ssrc_ is set.
481  ChannelMap receive_channels_;  // for multiple sources
482  // receive_channels_ can be read from WebRtc callback thread.  Access from
483  // the WebRtc thread must be synchronized with edits on the worker thread.
484  // Reads on the worker thread are ok.
485  //
486  std::vector<RtpHeaderExtension> receive_extensions_;
487  // Do not lock this on the VoE media processor thread; potential for deadlock
488  // exists.
489  mutable rtc::CriticalSection receive_channels_cs_;
490};
491
492}  // namespace cricket
493
494#endif  // TALK_MEDIA_WEBRTCVOICEENGINE_H_
495