webrtcvoiceengine.h revision c28a896a7bbd8a1ffef44a1f66ac67c43b4eeada 
1/*
2 * libjingle
3 * Copyright 2004 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_
29#define TALK_MEDIA_WEBRTCVOICEENGINE_H_
30
31#include <map>
32#include <set>
33#include <string>
34#include <vector>
35
36#include "talk/media/base/rtputils.h"
37#include "talk/media/webrtc/webrtccommon.h"
38#include "talk/media/webrtc/webrtcvoe.h"
39#include "talk/session/media/channel.h"
40#include "webrtc/base/buffer.h"
41#include "webrtc/base/byteorder.h"
42#include "webrtc/base/logging.h"
43#include "webrtc/base/scoped_ptr.h"
44#include "webrtc/base/stream.h"
45#include "webrtc/base/thread_checker.h"
46#include "webrtc/call.h"
47#include "webrtc/common.h"
48#include "webrtc/config.h"
49
50namespace webrtc {
51class VideoEngine;
52}
53
54namespace cricket {
55
56// WebRtcSoundclipStream is an adapter object that allows a memory stream to be
57// passed into WebRtc, and support looping.
58class WebRtcSoundclipStream : public webrtc::InStream {
59 public:
60  WebRtcSoundclipStream(const char* buf, size_t len)
61      : mem_(buf, len), loop_(true) {
62  }
63  void set_loop(bool loop) { loop_ = loop; }
64
65  int Read(void* buf, size_t len) override;
66  int Rewind() override;
67
68 private:
69  rtc::MemoryStream mem_;
70  bool loop_;
71};
72
73// WebRtcMonitorStream is used to monitor a stream coming from WebRtc.
74// For now we just dump the data.
75class WebRtcMonitorStream : public webrtc::OutStream {
76  bool Write(const void* buf, size_t len) override { return true; }
77};
78
79class AudioDeviceModule;
80class AudioRenderer;
81class VoETraceWrapper;
82class VoEWrapper;
83class VoiceProcessor;
84class WebRtcVoiceMediaChannel;
85
86// WebRtcVoiceEngine is a class to be used with CompositeMediaEngine.
87// It uses the WebRtc VoiceEngine library for audio handling.
88class WebRtcVoiceEngine
89    : public webrtc::VoiceEngineObserver,
90      public webrtc::TraceCallback,
91      public webrtc::VoEMediaProcess  {
92  friend class WebRtcVoiceMediaChannel;
93
94 public:
95  WebRtcVoiceEngine();
96  // Dependency injection for testing.
97  WebRtcVoiceEngine(VoEWrapper* voe_wrapper, VoETraceWrapper* tracing);
98  ~WebRtcVoiceEngine();
99  bool Init(rtc::Thread* worker_thread);
100  void Terminate();
101
102  int GetCapabilities();
103  VoiceMediaChannel* CreateChannel(const AudioOptions& options);
104
105  AudioOptions GetOptions() const { return options_; }
106  bool SetOptions(const AudioOptions& options);
107  bool SetDelayOffset(int offset);
108  bool SetDevices(const Device* in_device, const Device* out_device);
109  bool GetOutputVolume(int* level);
110  bool SetOutputVolume(int level);
111  int GetInputLevel();
112  bool SetLocalMonitor(bool enable);
113
114  const std::vector<AudioCodec>& codecs();
115  bool FindCodec(const AudioCodec& codec);
116  bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec);
117
118  const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
119
120  void SetLogging(int min_sev, const char* filter);
121
122  bool RegisterProcessor(uint32 ssrc,
123                         VoiceProcessor* voice_processor,
124                         MediaProcessorDirection direction);
125  bool UnregisterProcessor(uint32 ssrc,
126                           VoiceProcessor* voice_processor,
127                           MediaProcessorDirection direction);
128
129  // Method from webrtc::VoEMediaProcess
130  void Process(int channel,
131               webrtc::ProcessingTypes type,
132               int16_t audio10ms[],
133               int length,
134               int sampling_freq,
135               bool is_stereo) override;
136
137  // For tracking WebRtc channels. Needed because we have to pause them
138  // all when switching devices.
139  // May only be called by WebRtcVoiceMediaChannel.
140  void RegisterChannel(WebRtcVoiceMediaChannel *channel);
141  void UnregisterChannel(WebRtcVoiceMediaChannel *channel);
142
143  // Called by WebRtcVoiceMediaChannel to set a gain offset from
144  // the default AGC target level.
145  bool AdjustAgcLevel(int delta);
146
147  VoEWrapper* voe() { return voe_wrapper_.get(); }
148  int GetLastEngineError();
149
150  // Set the external ADM. This can only be called before Init.
151  bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm);
152
153  // Starts AEC dump using existing file.
154  bool StartAecDump(rtc::PlatformFile file);
155
156  // Check whether the supplied trace should be ignored.
157  bool ShouldIgnoreTrace(const std::string& trace);
158
159  // Create a VoiceEngine Channel.
160  int CreateMediaVoiceChannel();
161
162 private:
163  typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList;
164  typedef sigslot::
165      signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal;
166
167  void Construct();
168  void ConstructCodecs();
169  bool GetVoeCodec(int index, webrtc::CodecInst* codec);
170  bool InitInternal();
171  void SetTraceFilter(int filter);
172  void SetTraceOptions(const std::string& options);
173  // Applies either options or overrides.  Every option that is "set"
174  // will be applied.  Every option not "set" will be ignored.  This
175  // allows us to selectively turn on and off different options easily
176  // at any time.
177  bool ApplyOptions(const AudioOptions& options);
178  // Overrides, when set, take precedence over the options on a
179  // per-option basis.  For example, if AGC is set in options and AEC
180  // is set in overrides, AGC and AEC will be both be set.  Overrides
181  // can also turn off options.  For example, if AGC is set to "on" in
182  // options and AGC is set to "off" in overrides, the result is that
183  // AGC will be off until different overrides are applied or until
184  // the overrides are cleared.  Only one set of overrides is present
185  // at a time (they do not "stack").  And when the overrides are
186  // cleared, the media engine's state reverts back to the options set
187  // via SetOptions.  This allows us to have both "persistent options"
188  // (the normal options) and "temporary options" (overrides).
189  bool SetOptionOverrides(const AudioOptions& options);
190  bool ClearOptionOverrides();
191
192  // webrtc::TraceCallback:
193  void Print(webrtc::TraceLevel level, const char* trace, int length) override;
194
195  // webrtc::VoiceEngineObserver:
196  void CallbackOnError(int channel, int errCode) override;
197
198  // Given the device type, name, and id, find device id. Return true and
199  // set the output parameter rtc_id if successful.
200  bool FindWebRtcAudioDeviceId(
201      bool is_input, const std::string& dev_name, int dev_id, int* rtc_id);
202  bool FindChannelAndSsrc(int channel_num,
203                          WebRtcVoiceMediaChannel** channel,
204                          uint32* ssrc) const;
205  bool FindChannelNumFromSsrc(uint32 ssrc,
206                              MediaProcessorDirection direction,
207                              int* channel_num);
208  bool ChangeLocalMonitor(bool enable);
209  bool PauseLocalMonitor();
210  bool ResumeLocalMonitor();
211
212  bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction,
213                                  uint32 ssrc,
214                                  VoiceProcessor* voice_processor,
215                                  MediaProcessorDirection processor_direction);
216
217  void StartAecDump(const std::string& filename);
218  void StopAecDump();
219  int CreateVoiceChannel(VoEWrapper* voe);
220
221  // When a voice processor registers with the engine, it is connected
222  // to either the Rx or Tx signals, based on the direction parameter.
223  // SignalXXMediaFrame will be invoked for every audio packet.
224  FrameSignal SignalRxMediaFrame;
225  FrameSignal SignalTxMediaFrame;
226
227  static const int kDefaultLogSeverity = rtc::LS_WARNING;
228
229  // The primary instance of WebRtc VoiceEngine.
230  rtc::scoped_ptr<VoEWrapper> voe_wrapper_;
231  rtc::scoped_ptr<VoETraceWrapper> tracing_;
232  // The external audio device manager
233  webrtc::AudioDeviceModule* adm_;
234  int log_filter_;
235  std::string log_options_;
236  bool is_dumping_aec_;
237  std::vector<AudioCodec> codecs_;
238  std::vector<RtpHeaderExtension> rtp_header_extensions_;
239  bool desired_local_monitor_enable_;
240  rtc::scoped_ptr<WebRtcMonitorStream> monitor_;
241  ChannelList channels_;
242  // channels_ can be read from WebRtc callback thread. We need a lock on that
243  // callback as well as the RegisterChannel/UnregisterChannel.
244  rtc::CriticalSection channels_cs_;
245  webrtc::AgcConfig default_agc_config_;
246
247  webrtc::Config voe_config_;
248
249  bool initialized_;
250  // See SetOptions and SetOptionOverrides for a description of the
251  // difference between options and overrides.
252  // options_ are the base options, which combined with the
253  // option_overrides_, create the current options being used.
254  // options_ is stored so that when option_overrides_ is cleared, we
255  // can restore the options_ without the option_overrides.
256  AudioOptions options_;
257  AudioOptions option_overrides_;
258
259  // When the media processor registers with the engine, the ssrc is cached
260  // here so that a look up need not be made when the callback is invoked.
261  // This is necessary because the lookup results in mux_channels_cs lock being
262  // held and if a remote participant leaves the hangout at the same time
263  // we hit a deadlock.
264  uint32 tx_processor_ssrc_;
265  uint32 rx_processor_ssrc_;
266
267  rtc::CriticalSection signal_media_critical_;
268
269  // Cache received experimental_aec, delay_agnostic_aec and experimental_ns
270  // values, and apply them in case they are missing in the audio options. We
271  // need to do this because SetExtraOptions() will revert to defaults for
272  // options which are not provided.
273  Settable<bool> experimental_aec_;
274  Settable<bool> delay_agnostic_aec_;
275  Settable<bool> experimental_ns_;
276};
277
278// WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses
279// WebRtc Voice Engine.
280class WebRtcVoiceMediaChannel : public VoiceMediaChannel,
281                                public webrtc::Transport {
282 public:
283  explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine);
284  ~WebRtcVoiceMediaChannel() override;
285
286  int voe_channel() const { return voe_channel_; }
287  bool valid() const { return voe_channel_ != -1; }
288
289  bool SetOptions(const AudioOptions& options) override;
290  bool GetOptions(AudioOptions* options) const override {
291    *options = options_;
292    return true;
293  }
294  bool SetRecvCodecs(const std::vector<AudioCodec>& codecs) override;
295  bool SetSendCodecs(const std::vector<AudioCodec>& codecs) override;
296  bool SetRecvRtpHeaderExtensions(
297      const std::vector<RtpHeaderExtension>& extensions) override;
298  bool SetSendRtpHeaderExtensions(
299      const std::vector<RtpHeaderExtension>& extensions) override;
300  bool SetPlayout(bool playout) override;
301  bool PausePlayout();
302  bool ResumePlayout();
303  bool SetSend(SendFlags send) override;
304  bool PauseSend();
305  bool ResumeSend();
306  bool AddSendStream(const StreamParams& sp) override;
307  bool RemoveSendStream(uint32 ssrc) override;
308  bool AddRecvStream(const StreamParams& sp) override;
309  bool RemoveRecvStream(uint32 ssrc) override;
310  bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer) override;
311  bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer) override;
312  bool GetActiveStreams(AudioInfo::StreamList* actives) override;
313  int GetOutputLevel() override;
314  int GetTimeSinceLastTyping() override;
315  void SetTypingDetectionParameters(int time_window,
316                                    int cost_per_typing,
317                                    int reporting_threshold,
318                                    int penalty_decay,
319                                    int type_event_delay) override;
320  bool SetOutputScaling(uint32 ssrc, double left, double right) override;
321  bool GetOutputScaling(uint32 ssrc, double* left, double* right) override;
322
323  bool SetRingbackTone(const char* buf, int len) override;
324  bool PlayRingbackTone(uint32 ssrc, bool play, bool loop) override;
325  bool CanInsertDtmf() override;
326  bool InsertDtmf(uint32 ssrc, int event, int duration, int flags) override;
327
328  void OnPacketReceived(rtc::Buffer* packet,
329                        const rtc::PacketTime& packet_time) override;
330  void OnRtcpReceived(rtc::Buffer* packet,
331                      const rtc::PacketTime& packet_time) override;
332  void OnReadyToSend(bool ready) override {}
333  bool MuteStream(uint32 ssrc, bool on) override;
334  bool SetMaxSendBandwidth(int bps) override;
335  bool GetStats(VoiceMediaInfo* info) override;
336  // Gets last reported error from WebRtc voice engine.  This should be only
337  // called in response a failure.
338  void GetLastMediaError(uint32* ssrc,
339                         VoiceMediaChannel::Error* error) override;
340
341  // implements Transport interface
342  int SendPacket(int channel, const void* data, size_t len) override {
343    rtc::Buffer packet(reinterpret_cast<const uint8_t*>(data), len,
344                       kMaxRtpPacketLen);
345    return VoiceMediaChannel::SendPacket(&packet) ? static_cast<int>(len) : -1;
346  }
347
348  int SendRTCPPacket(int channel, const void* data, size_t len) override {
349    rtc::Buffer packet(reinterpret_cast<const uint8_t*>(data), len,
350                       kMaxRtpPacketLen);
351    return VoiceMediaChannel::SendRtcp(&packet) ? static_cast<int>(len) : -1;
352  }
353
354  bool FindSsrc(int channel_num, uint32* ssrc);
355  void OnError(uint32 ssrc, int error);
356
357  bool sending() const { return send_ != SEND_NOTHING; }
358  int GetReceiveChannelNum(uint32 ssrc);
359  int GetSendChannelNum(uint32 ssrc);
360
361  void SetCall(webrtc::Call* call);
362
363 private:
364  WebRtcVoiceEngine* engine() { return engine_; }
365  int GetLastEngineError() { return engine()->GetLastEngineError(); }
366  int GetOutputLevel(int channel);
367  bool GetRedSendCodec(const AudioCodec& red_codec,
368                       const std::vector<AudioCodec>& all_codecs,
369                       webrtc::CodecInst* send_codec);
370  bool EnableRtcp(int channel);
371  bool ResetRecvCodecs(int channel);
372  bool SetPlayout(int channel, bool playout);
373  static uint32 ParseSsrc(const void* data, size_t len, bool rtcp);
374  static Error WebRtcErrorToChannelError(int err_code);
375
376  class WebRtcVoiceChannelRenderer;
377  // Map of ssrc to WebRtcVoiceChannelRenderer object.  A new object of
378  // WebRtcVoiceChannelRenderer will be created for every new stream and
379  // will be destroyed when the stream goes away.
380  typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap;
381  typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool,
382      unsigned char);
383
384  void SetNack(int channel, bool nack_enabled);
385  void SetNack(const ChannelMap& channels, bool nack_enabled);
386  bool SetSendCodec(const webrtc::CodecInst& send_codec);
387  bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec);
388  bool ChangePlayout(bool playout);
389  bool ChangeSend(SendFlags send);
390  bool ChangeSend(int channel, SendFlags send);
391  void ConfigureSendChannel(int channel);
392  bool ConfigureRecvChannel(int channel);
393  bool DeleteChannel(int channel);
394  bool InConferenceMode() const {
395    return options_.conference_mode.GetWithDefaultIfUnset(false);
396  }
397  bool IsDefaultChannel(int channel_id) const {
398    return channel_id == voe_channel();
399  }
400  bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs);
401  bool SetSendBitrateInternal(int bps);
402
403  bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
404                          const RtpHeaderExtension* extension);
405  void TryAddAudioRecvStream(uint32 ssrc);
406  void TryRemoveAudioRecvStream(uint32 ssrc);
407
408  bool SetChannelRecvRtpHeaderExtensions(
409    int channel_id,
410    const std::vector<RtpHeaderExtension>& extensions);
411  bool SetChannelSendRtpHeaderExtensions(
412    int channel_id,
413    const std::vector<RtpHeaderExtension>& extensions);
414
415  rtc::ThreadChecker thread_checker_;
416
417  WebRtcVoiceEngine* engine_;
418  const int voe_channel_;
419  rtc::scoped_ptr<WebRtcSoundclipStream> ringback_tone_;
420  std::set<int> ringback_channels_;  // channels playing ringback
421  std::vector<AudioCodec> recv_codecs_;
422  std::vector<AudioCodec> send_codecs_;
423  rtc::scoped_ptr<webrtc::CodecInst> send_codec_;
424  bool send_bitrate_setting_;
425  int send_bitrate_bps_;
426  AudioOptions options_;
427  bool dtmf_allowed_;
428  bool desired_playout_;
429  bool nack_enabled_;
430  bool playout_;
431  bool typing_noise_detected_;
432  SendFlags desired_send_;
433  SendFlags send_;
434  webrtc::Call* call_;
435
436  // send_channels_ contains the channels which are being used for sending.
437  // When the default channel (voe_channel) is used for sending, it is
438  // contained in send_channels_, otherwise not.
439  ChannelMap send_channels_;
440  std::vector<RtpHeaderExtension> send_extensions_;
441  uint32 default_receive_ssrc_;
442  // Note the default channel (voe_channel()) can reside in both
443  // receive_channels_ and send_channels_ in non-conference mode and in that
444  // case it will only be there if a non-zero default_receive_ssrc_ is set.
445  ChannelMap receive_channels_;  // for multiple sources
446  std::map<uint32, webrtc::AudioReceiveStream*> receive_streams_;
447  // receive_channels_ can be read from WebRtc callback thread.  Access from
448  // the WebRtc thread must be synchronized with edits on the worker thread.
449  // Reads on the worker thread are ok.
450  //
451  std::vector<RtpHeaderExtension> receive_extensions_;
452  std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
453
454  // Do not lock this on the VoE media processor thread; potential for deadlock
455  // exists.
456  mutable rtc::CriticalSection receive_channels_cs_;
457};
458
459}  // namespace cricket
460
461#endif  // TALK_MEDIA_WEBRTCVOICEENGINE_H_
462