1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
6#define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
7
8#include "base/atomicops.h"
9#include "base/files/file.h"
10#include "base/synchronization/lock.h"
11#include "base/threading/thread_checker.h"
12#include "base/time/time.h"
13#include "content/common/content_export.h"
14#include "content/renderer/media/aec_dump_message_filter.h"
15#include "content/renderer/media/webrtc_audio_device_impl.h"
16#include "media/base/audio_converter.h"
17#include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
18#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
19#include "third_party/webrtc/modules/interface/module_common_types.h"
20
21namespace blink {
22class WebMediaConstraints;
23}
24
25namespace media {
26class AudioBus;
27class AudioFifo;
28class AudioParameters;
29}  // namespace media
30
31namespace webrtc {
32class AudioFrame;
33class TypingDetection;
34}
35
36namespace content {
37
38class MediaStreamAudioBus;
39class MediaStreamAudioFifo;
40class RTCMediaConstraints;
41
42using webrtc::AudioProcessorInterface;
43
44// This class owns an object of webrtc::AudioProcessing which contains signal
45// processing components like AGC, AEC and NS. It enables the components based
46// on the getUserMedia constraints, processes the data and outputs it in a unit
47// of 10 ms data chunk.
48class CONTENT_EXPORT MediaStreamAudioProcessor :
49    NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink),
50    NON_EXPORTED_BASE(public AudioProcessorInterface),
51    NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) {
52 public:
53  // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise
54  // returns true.
55  static bool IsAudioTrackProcessingEnabled();
56
57  // |playout_data_source| is used to register this class as a sink to the
58  // WebRtc playout data for processing AEC. If clients do not enable AEC,
59  // |playout_data_source| won't be used.
60  MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints,
61                            int effects,
62                            WebRtcPlayoutDataSource* playout_data_source);
63
64  // Called when the format of the capture data has changed.
65  // Called on the main render thread. The caller is responsible for stopping
66  // the capture thread before calling this method.
67  // After this method, the capture thread will be changed to a new capture
68  // thread.
69  void OnCaptureFormatChanged(const media::AudioParameters& source_params);
70
71  // Pushes capture data in |audio_source| to the internal FIFO. Each call to
72  // this method should be followed by calls to ProcessAndConsumeData() while
73  // it returns false, to pull out all available data.
74  // Called on the capture audio thread.
75  void PushCaptureData(const media::AudioBus* audio_source);
76
77  // Processes a block of 10 ms data from the internal FIFO and outputs it via
78  // |out|. |out| is the address of the pointer that will be pointed to
79  // the post-processed data if the method is returning a true. The lifetime
80  // of the data represeted by |out| is guaranteed until this method is called
81  // again.
82  // |new_volume| receives the new microphone volume from the AGC.
83  // The new microphone volume range is [0, 255], and the value will be 0 if
84  // the microphone volume should not be adjusted.
85  // Returns true if the internal FIFO has at least 10 ms data for processing,
86  // otherwise false.
87  // Called on the capture audio thread.
88  //
89  // TODO(ajm): Don't we want this to output float?
90  bool ProcessAndConsumeData(base::TimeDelta capture_delay,
91                             int volume,
92                             bool key_pressed,
93                             int* new_volume,
94                             int16** out);
95
96  // Stops the audio processor, no more AEC dump or render data after calling
97  // this method.
98  void Stop();
99
100  // The audio formats of the capture input to and output from the processor.
101  // Must only be called on the main render or audio capture threads.
102  const media::AudioParameters& InputFormat() const;
103  const media::AudioParameters& OutputFormat() const;
104
105  // Accessor to check if the audio processing is enabled or not.
106  bool has_audio_processing() const { return audio_processing_ != NULL; }
107
108  // AecDumpMessageFilter::AecDumpDelegate implementation.
109  // Called on the main render thread.
110  virtual void OnAecDumpFile(
111      const IPC::PlatformFileForTransit& file_handle) OVERRIDE;
112  virtual void OnDisableAecDump() OVERRIDE;
113  virtual void OnIpcClosing() OVERRIDE;
114
115 protected:
116  friend class base::RefCountedThreadSafe<MediaStreamAudioProcessor>;
117  virtual ~MediaStreamAudioProcessor();
118
119 private:
120  friend class MediaStreamAudioProcessorTest;
121  FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
122                           GetAecDumpMessageFilter);
123
124  // WebRtcPlayoutDataSource::Sink implementation.
125  virtual void OnPlayoutData(media::AudioBus* audio_bus,
126                             int sample_rate,
127                             int audio_delay_milliseconds) OVERRIDE;
128  virtual void OnPlayoutDataSourceChanged() OVERRIDE;
129
130  // webrtc::AudioProcessorInterface implementation.
131  // This method is called on the libjingle thread.
132  virtual void GetStats(AudioProcessorStats* stats) OVERRIDE;
133
134  // Helper to initialize the WebRtc AudioProcessing.
135  void InitializeAudioProcessingModule(
136      const blink::WebMediaConstraints& constraints, int effects);
137
138  // Helper to initialize the capture converter.
139  void InitializeCaptureFifo(const media::AudioParameters& input_format);
140
141  // Helper to initialize the render converter.
142  void InitializeRenderFifoIfNeeded(int sample_rate,
143                                    int number_of_channels,
144                                    int frames_per_buffer);
145
146  // Called by ProcessAndConsumeData().
147  // Returns the new microphone volume in the range of |0, 255].
148  // When the volume does not need to be updated, it returns 0.
149  int ProcessData(const float* const* process_ptrs,
150                  int process_frames,
151                  base::TimeDelta capture_delay,
152                  int volume,
153                  bool key_pressed,
154                  float* const* output_ptrs);
155
156  // Cached value for the render delay latency. This member is accessed by
157  // both the capture audio thread and the render audio thread.
158  base::subtle::Atomic32 render_delay_ms_;
159
160  // Module to handle processing and format conversion.
161  scoped_ptr<webrtc::AudioProcessing> audio_processing_;
162
163  // FIFO to provide 10 ms capture chunks.
164  scoped_ptr<MediaStreamAudioFifo> capture_fifo_;
165  // Receives processing output.
166  scoped_ptr<MediaStreamAudioBus> output_bus_;
167  // Receives interleaved int16 data for output.
168  scoped_ptr<int16[]> output_data_;
169
170  // FIFO to provide 10 ms render chunks when the AEC is enabled.
171  scoped_ptr<MediaStreamAudioFifo> render_fifo_;
172
173  // These are mutated on the main render thread in OnCaptureFormatChanged().
174  // The caller guarantees this does not run concurrently with accesses on the
175  // capture audio thread.
176  media::AudioParameters input_format_;
177  media::AudioParameters output_format_;
178  // Only used on the render audio thread.
179  media::AudioParameters render_format_;
180
181  // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
182  // lifetime of RenderThread.
183  WebRtcPlayoutDataSource* playout_data_source_;
184
185  // Used to DCHECK that some methods are called on the main render thread.
186  base::ThreadChecker main_thread_checker_;
187  // Used to DCHECK that some methods are called on the capture audio thread.
188  base::ThreadChecker capture_thread_checker_;
189  // Used to DCHECK that some methods are called on the render audio thread.
190  base::ThreadChecker render_thread_checker_;
191
192  // Flag to enable stereo channel mirroring.
193  bool audio_mirroring_;
194
195  scoped_ptr<webrtc::TypingDetection> typing_detector_;
196  // This flag is used to show the result of typing detection.
197  // It can be accessed by the capture audio thread and by the libjingle thread
198  // which calls GetStats().
199  base::subtle::Atomic32 typing_detected_;
200
201  // Communication with browser for AEC dump.
202  scoped_refptr<AecDumpMessageFilter> aec_dump_message_filter_;
203
204  // Flag to avoid executing Stop() more than once.
205  bool stopped_;
206};
207
208}  // namespace content
209
210#endif  // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
211