1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12#define WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
13
14#include <list>
15#include <string>
16#include <vector>
17
18#include "webrtc/base/criticalsection.h"
19#include "webrtc/base/scoped_ptr.h"
20#include "webrtc/base/thread_annotations.h"
21#include "webrtc/modules/audio_processing/audio_buffer.h"
22#include "webrtc/modules/audio_processing/include/audio_processing.h"
23#include "webrtc/system_wrappers/include/file_wrapper.h"
24
25#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
26// Files generated at build-time by the protobuf compiler.
27#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
28#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
29#else
30#include "webrtc/audio_processing/debug.pb.h"
31#endif
32#endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
33
34namespace webrtc {
35
36class AgcManagerDirect;
37class AudioConverter;
38
39template<typename T>
40class Beamformer;
41
42class AudioProcessingImpl : public AudioProcessing {
43 public:
44  // Methods forcing APM to run in a single-threaded manner.
45  // Acquires both the render and capture locks.
46  explicit AudioProcessingImpl(const Config& config);
47  // AudioProcessingImpl takes ownership of beamformer.
48  AudioProcessingImpl(const Config& config, Beamformer<float>* beamformer);
49  virtual ~AudioProcessingImpl();
50  int Initialize() override;
51  int Initialize(int input_sample_rate_hz,
52                 int output_sample_rate_hz,
53                 int reverse_sample_rate_hz,
54                 ChannelLayout input_layout,
55                 ChannelLayout output_layout,
56                 ChannelLayout reverse_layout) override;
57  int Initialize(const ProcessingConfig& processing_config) override;
58  void SetExtraOptions(const Config& config) override;
59  void UpdateHistogramsOnCallEnd() override;
60  int StartDebugRecording(const char filename[kMaxFilenameSize]) override;
61  int StartDebugRecording(FILE* handle) override;
62  int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) override;
63  int StopDebugRecording() override;
64
65  // Capture-side exclusive methods possibly running APM in a
66  // multi-threaded manner. Acquire the capture lock.
67  int ProcessStream(AudioFrame* frame) override;
68  int ProcessStream(const float* const* src,
69                    size_t samples_per_channel,
70                    int input_sample_rate_hz,
71                    ChannelLayout input_layout,
72                    int output_sample_rate_hz,
73                    ChannelLayout output_layout,
74                    float* const* dest) override;
75  int ProcessStream(const float* const* src,
76                    const StreamConfig& input_config,
77                    const StreamConfig& output_config,
78                    float* const* dest) override;
79  void set_output_will_be_muted(bool muted) override;
80  int set_stream_delay_ms(int delay) override;
81  void set_delay_offset_ms(int offset) override;
82  int delay_offset_ms() const override;
83  void set_stream_key_pressed(bool key_pressed) override;
84  int input_sample_rate_hz() const override;
85
86  // Render-side exclusive methods possibly running APM in a
87  // multi-threaded manner. Acquire the render lock.
88  int AnalyzeReverseStream(AudioFrame* frame) override;
89  int ProcessReverseStream(AudioFrame* frame) override;
90  int AnalyzeReverseStream(const float* const* data,
91                           size_t samples_per_channel,
92                           int sample_rate_hz,
93                           ChannelLayout layout) override;
94  int ProcessReverseStream(const float* const* src,
95                           const StreamConfig& reverse_input_config,
96                           const StreamConfig& reverse_output_config,
97                           float* const* dest) override;
98
99  // Methods only accessed from APM submodules or
100  // from AudioProcessing tests in a single-threaded manner.
101  // Hence there is no need for locks in these.
102  int proc_sample_rate_hz() const override;
103  int proc_split_sample_rate_hz() const override;
104  size_t num_input_channels() const override;
105  size_t num_proc_channels() const override;
106  size_t num_output_channels() const override;
107  size_t num_reverse_channels() const override;
108  int stream_delay_ms() const override;
109  bool was_stream_delay_set() const override
110      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
111
112  // Methods returning pointers to APM submodules.
113  // No locks are aquired in those, as those locks
114  // would offer no protection (the submodules are
115  // created only once in a single-treaded manner
116  // during APM creation).
117  EchoCancellation* echo_cancellation() const override;
118  EchoControlMobile* echo_control_mobile() const override;
119  GainControl* gain_control() const override;
120  HighPassFilter* high_pass_filter() const override;
121  LevelEstimator* level_estimator() const override;
122  NoiseSuppression* noise_suppression() const override;
123  VoiceDetection* voice_detection() const override;
124
125 protected:
126  // Overridden in a mock.
127  virtual int InitializeLocked()
128      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
129
130 private:
131  struct ApmPublicSubmodules;
132  struct ApmPrivateSubmodules;
133
134#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
135  // State for the debug dump.
136  struct ApmDebugDumpThreadState {
137    ApmDebugDumpThreadState() : event_msg(new audioproc::Event()) {}
138    rtc::scoped_ptr<audioproc::Event> event_msg;  // Protobuf message.
139    std::string event_str;  // Memory for protobuf serialization.
140
141    // Serialized string of last saved APM configuration.
142    std::string last_serialized_config;
143  };
144
145  struct ApmDebugDumpState {
146    ApmDebugDumpState() : debug_file(FileWrapper::Create()) {}
147    rtc::scoped_ptr<FileWrapper> debug_file;
148    ApmDebugDumpThreadState render;
149    ApmDebugDumpThreadState capture;
150  };
151#endif
152
153  // Method for modifying the formats struct that are called from both
154  // the render and capture threads. The check for whether modifications
155  // are needed is done while holding the render lock only, thereby avoiding
156  // that the capture thread blocks the render thread.
157  // The struct is modified in a single-threaded manner by holding both the
158  // render and capture locks.
159  int MaybeInitialize(const ProcessingConfig& config)
160      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
161
162  int MaybeInitializeRender(const ProcessingConfig& processing_config)
163      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
164
165  int MaybeInitializeCapture(const ProcessingConfig& processing_config)
166      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
167
168  // Method for checking for the need of conversion. Accesses the formats
169  // structs in a read manner but the requirement for the render lock to be held
170  // was added as it currently anyway is always called in that manner.
171  bool rev_conversion_needed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
172  bool render_check_rev_conversion_needed() const
173      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
174
175  // Methods requiring APM running in a single-threaded manner.
176  // Are called with both the render and capture locks already
177  // acquired.
178  void InitializeExperimentalAgc()
179      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
180  void InitializeTransient()
181      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
182  void InitializeBeamformer()
183      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
184  void InitializeIntelligibility()
185      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
186  void InitializeHighPassFilter()
187      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
188  void InitializeNoiseSuppression()
189      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
190  void InitializeLevelEstimator()
191      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
192  void InitializeVoiceDetection()
193      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
194  int InitializeLocked(const ProcessingConfig& config)
195      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
196
197  // Capture-side exclusive methods possibly running APM in a multi-threaded
198  // manner that are called with the render lock already acquired.
199  int ProcessStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
200  bool output_copy_needed(bool is_data_processed) const
201      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
202  bool is_data_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
203  bool synthesis_needed(bool is_data_processed) const
204      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
205  bool analysis_needed(bool is_data_processed) const
206      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
207  void MaybeUpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
208
209  // Render-side exclusive methods possibly running APM in a multi-threaded
210  // manner that are called with the render lock already acquired.
211  // TODO(ekm): Remove once all clients updated to new interface.
212  int AnalyzeReverseStreamLocked(const float* const* src,
213                                 const StreamConfig& input_config,
214                                 const StreamConfig& output_config)
215      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
216  bool is_rev_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
217  int ProcessReverseStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
218
219// Debug dump methods that are internal and called without locks.
220// TODO(peah): Make thread safe.
221#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
222  // TODO(andrew): make this more graceful. Ideally we would split this stuff
223  // out into a separate class with an "enabled" and "disabled" implementation.
224  static int WriteMessageToDebugFile(FileWrapper* debug_file,
225                                     rtc::CriticalSection* crit_debug,
226                                     ApmDebugDumpThreadState* debug_state);
227  int WriteInitMessage() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
228
229  // Writes Config message. If not |forced|, only writes the current config if
230  // it is different from the last saved one; if |forced|, writes the config
231  // regardless of the last saved.
232  int WriteConfigMessage(bool forced) EXCLUSIVE_LOCKS_REQUIRED(crit_capture_)
233      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
234
235  // Critical section.
236  mutable rtc::CriticalSection crit_debug_;
237
238  // Debug dump state.
239  ApmDebugDumpState debug_dump_;
240#endif
241
242  // Critical sections.
243  mutable rtc::CriticalSection crit_render_ ACQUIRED_BEFORE(crit_capture_);
244  mutable rtc::CriticalSection crit_capture_;
245
246  // Structs containing the pointers to the submodules.
247  rtc::scoped_ptr<ApmPublicSubmodules> public_submodules_;
248  rtc::scoped_ptr<ApmPrivateSubmodules> private_submodules_
249      GUARDED_BY(crit_capture_);
250
251  // State that is written to while holding both the render and capture locks
252  // but can be read without any lock being held.
253  // As this is only accessed internally of APM, and all internal methods in APM
254  // either are holding the render or capture locks, this construct is safe as
255  // it is not possible to read the variables while writing them.
256  struct ApmFormatState {
257    ApmFormatState()
258        :  // Format of processing streams at input/output call sites.
259          api_format({{{kSampleRate16kHz, 1, false},
260                       {kSampleRate16kHz, 1, false},
261                       {kSampleRate16kHz, 1, false},
262                       {kSampleRate16kHz, 1, false}}}),
263          rev_proc_format(kSampleRate16kHz, 1) {}
264    ProcessingConfig api_format;
265    StreamConfig rev_proc_format;
266  } formats_;
267
268  // APM constants.
269  const struct ApmConstants {
270    ApmConstants(int agc_startup_min_volume,
271                 bool use_new_agc,
272                 bool intelligibility_enabled)
273        :  // Format of processing streams at input/output call sites.
274          agc_startup_min_volume(agc_startup_min_volume),
275          use_new_agc(use_new_agc),
276          intelligibility_enabled(intelligibility_enabled) {}
277    int agc_startup_min_volume;
278    bool use_new_agc;
279    bool intelligibility_enabled;
280  } constants_;
281
282  struct ApmCaptureState {
283    ApmCaptureState(bool transient_suppressor_enabled,
284                    const std::vector<Point>& array_geometry,
285                    SphericalPointf target_direction)
286        : aec_system_delay_jumps(-1),
287          delay_offset_ms(0),
288          was_stream_delay_set(false),
289          last_stream_delay_ms(0),
290          last_aec_system_delay_ms(0),
291          stream_delay_jumps(-1),
292          output_will_be_muted(false),
293          key_pressed(false),
294          transient_suppressor_enabled(transient_suppressor_enabled),
295          array_geometry(array_geometry),
296          target_direction(target_direction),
297          fwd_proc_format(kSampleRate16kHz),
298          split_rate(kSampleRate16kHz) {}
299    int aec_system_delay_jumps;
300    int delay_offset_ms;
301    bool was_stream_delay_set;
302    int last_stream_delay_ms;
303    int last_aec_system_delay_ms;
304    int stream_delay_jumps;
305    bool output_will_be_muted;
306    bool key_pressed;
307    bool transient_suppressor_enabled;
308    std::vector<Point> array_geometry;
309    SphericalPointf target_direction;
310    rtc::scoped_ptr<AudioBuffer> capture_audio;
311    // Only the rate and samples fields of fwd_proc_format_ are used because the
312    // forward processing number of channels is mutable and is tracked by the
313    // capture_audio_.
314    StreamConfig fwd_proc_format;
315    int split_rate;
316  } capture_ GUARDED_BY(crit_capture_);
317
318  struct ApmCaptureNonLockedState {
319    ApmCaptureNonLockedState(bool beamformer_enabled)
320        : fwd_proc_format(kSampleRate16kHz),
321          split_rate(kSampleRate16kHz),
322          stream_delay_ms(0),
323          beamformer_enabled(beamformer_enabled) {}
324    // Only the rate and samples fields of fwd_proc_format_ are used because the
325    // forward processing number of channels is mutable and is tracked by the
326    // capture_audio_.
327    StreamConfig fwd_proc_format;
328    int split_rate;
329    int stream_delay_ms;
330    bool beamformer_enabled;
331  } capture_nonlocked_;
332
333  struct ApmRenderState {
334    rtc::scoped_ptr<AudioConverter> render_converter;
335    rtc::scoped_ptr<AudioBuffer> render_audio;
336  } render_ GUARDED_BY(crit_render_);
337};
338
339}  // namespace webrtc
340
341#endif  // WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
342