media_stream_audio_processor.h revision cedac228d2dd51db4b79ea1e72c7f249408ee061 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
6#define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
7
8#include "base/atomicops.h"
9#include "base/files/file.h"
10#include "base/synchronization/lock.h"
11#include "base/threading/thread_checker.h"
12#include "base/time/time.h"
13#include "content/common/content_export.h"
14#include "content/renderer/media/webrtc_audio_device_impl.h"
15#include "media/base/audio_converter.h"
16#include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
17#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
18#include "third_party/webrtc/modules/interface/module_common_types.h"
19
20namespace blink {
21class WebMediaConstraints;
22}
23
24namespace media {
25class AudioBus;
26class AudioFifo;
27class AudioParameters;
28}  // namespace media
29
30namespace webrtc {
31class AudioFrame;
32class TypingDetection;
33}
34
35namespace content {
36
37class RTCMediaConstraints;
38
39using webrtc::AudioProcessorInterface;
40
41// This class owns an object of webrtc::AudioProcessing which contains signal
42// processing components like AGC, AEC and NS. It enables the components based
43// on the getUserMedia constraints, processes the data and outputs it in a unit
44// of 10 ms data chunk.
45class CONTENT_EXPORT MediaStreamAudioProcessor :
46    NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink),
47    NON_EXPORTED_BASE(public AudioProcessorInterface) {
48 public:
49  // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise
50  // returns true.
51  static bool IsAudioTrackProcessingEnabled();
52
53  // |playout_data_source| is used to register this class as a sink to the
54  // WebRtc playout data for processing AEC. If clients do not enable AEC,
55  // |playout_data_source| won't be used.
56  MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints,
57                            int effects,
58                            WebRtcPlayoutDataSource* playout_data_source);
59
60  // Called when format of the capture data has changed.
61  // Called on the main render thread.  The caller is responsible for stopping
62  // the capture thread before calling this method.
63  // After this method, the capture thread will be changed to a new capture
64  // thread.
65  void OnCaptureFormatChanged(const media::AudioParameters& source_params);
66
67  // Pushes capture data in |audio_source| to the internal FIFO.
68  // Called on the capture audio thread.
69  void PushCaptureData(media::AudioBus* audio_source);
70
71  // Processes a block of 10 ms data from the internal FIFO and outputs it via
72  // |out|. |out| is the address of the pointer that will be pointed to
73  // the post-processed data if the method is returning a true. The lifetime
74  // of the data represeted by |out| is guaranteed to outlive the method call.
75  // That also says *|out| won't change until this method is called again.
76  // |new_volume| receives the new microphone volume from the AGC.
77  // The new microphoen volume range is [0, 255], and the value will be 0 if
78  // the microphone volume should not be adjusted.
79  // Returns true if the internal FIFO has at least 10 ms data for processing,
80  // otherwise false.
81  // |capture_delay|, |volume| and |key_pressed| will be passed to
82  // webrtc::AudioProcessing to help processing the data.
83  // Called on the capture audio thread.
84  bool ProcessAndConsumeData(base::TimeDelta capture_delay,
85                             int volume,
86                             bool key_pressed,
87                             int* new_volume,
88                             int16** out);
89
90  // The audio format of the input to the processor.
91  const media::AudioParameters& InputFormat() const;
92
93  // The audio format of the output from the processor.
94  const media::AudioParameters& OutputFormat() const;
95
96  // Accessor to check if the audio processing is enabled or not.
97  bool has_audio_processing() const { return audio_processing_ != NULL; }
98
99  // Starts/Stops the Aec dump on the |audio_processing_|.
100  // Called on the main render thread.
101  // This method takes the ownership of |aec_dump_file|.
102  void StartAecDump(base::File aec_dump_file);
103  void StopAecDump();
104
105 protected:
106  friend class base::RefCountedThreadSafe<MediaStreamAudioProcessor>;
107  virtual ~MediaStreamAudioProcessor();
108
109 private:
110  friend class MediaStreamAudioProcessorTest;
111
112  class MediaStreamAudioConverter;
113
114  // WebRtcPlayoutDataSource::Sink implementation.
115  virtual void OnPlayoutData(media::AudioBus* audio_bus,
116                             int sample_rate,
117                             int audio_delay_milliseconds) OVERRIDE;
118  virtual void OnPlayoutDataSourceChanged() OVERRIDE;
119
120  // webrtc::AudioProcessorInterface implementation.
121  // This method is called on the libjingle thread.
122  virtual void GetStats(AudioProcessorStats* stats) OVERRIDE;
123
124  // Helper to initialize the WebRtc AudioProcessing.
125  void InitializeAudioProcessingModule(
126      const blink::WebMediaConstraints& constraints, int effects);
127
128  // Helper to initialize the capture converter.
129  void InitializeCaptureConverter(const media::AudioParameters& source_params);
130
131  // Helper to initialize the render converter.
132  void InitializeRenderConverterIfNeeded(int sample_rate,
133                                         int number_of_channels,
134                                         int frames_per_buffer);
135
136  // Called by ProcessAndConsumeData().
137  // Returns the new microphone volume in the range of |0, 255].
138  // When the volume does not need to be updated, it returns 0.
139  int ProcessData(webrtc::AudioFrame* audio_frame,
140                  base::TimeDelta capture_delay,
141                  int volume,
142                  bool key_pressed);
143
144  // Called when the processor is going away.
145  void StopAudioProcessing();
146
147  // Cached value for the render delay latency. This member is accessed by
148  // both the capture audio thread and the render audio thread.
149  base::subtle::Atomic32 render_delay_ms_;
150
151  // webrtc::AudioProcessing module which does AEC, AGC, NS, HighPass filter,
152  // ..etc.
153  scoped_ptr<webrtc::AudioProcessing> audio_processing_;
154
155  // Converter used for the down-mixing and resampling of the capture data.
156  scoped_ptr<MediaStreamAudioConverter> capture_converter_;
157
158  // AudioFrame used to hold the output of |capture_converter_|.
159  webrtc::AudioFrame capture_frame_;
160
161  // Converter used for the down-mixing and resampling of the render data when
162  // the AEC is enabled.
163  scoped_ptr<MediaStreamAudioConverter> render_converter_;
164
165  // AudioFrame used to hold the output of |render_converter_|.
166  webrtc::AudioFrame render_frame_;
167
168  // Data bus to help converting interleaved data to an AudioBus.
169  scoped_ptr<media::AudioBus> render_data_bus_;
170
171  // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
172  // lifetime of RenderThread.
173  WebRtcPlayoutDataSource* const playout_data_source_;
174
175  // Used to DCHECK that the destructor is called on the main render thread.
176  base::ThreadChecker main_thread_checker_;
177
178  // Used to DCHECK that some methods are called on the capture audio thread.
179  base::ThreadChecker capture_thread_checker_;
180
181  // Used to DCHECK that PushRenderData() is called on the render audio thread.
182  base::ThreadChecker render_thread_checker_;
183
184  // Flag to enable the stereo channels mirroring.
185  bool audio_mirroring_;
186
187  // Used by the typing detection.
188  scoped_ptr<webrtc::TypingDetection> typing_detector_;
189
190  // This flag is used to show the result of typing detection.
191  // It can be accessed by the capture audio thread and by the libjingle thread
192  // which calls GetStats().
193  base::subtle::Atomic32 typing_detected_;
194};
195
196}  // namespace content
197
198#endif  // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
199