audio_buffer.cc revision 103657b48442dedd1742fca4a73d5131bf4ae624 
1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/audio_processing/audio_buffer.h"
12
13#include "webrtc/common_audio/include/audio_util.h"
14#include "webrtc/common_audio/resampler/push_sinc_resampler.h"
15#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
16
17namespace webrtc {
18namespace {
19
20enum {
21  kSamplesPer8kHzChannel = 80,
22  kSamplesPer16kHzChannel = 160,
23  kSamplesPer32kHzChannel = 320
24};
25
26bool HasKeyboardChannel(AudioProcessing::ChannelLayout layout) {
27  switch (layout) {
28    case AudioProcessing::kMono:
29    case AudioProcessing::kStereo:
30      return false;
31    case AudioProcessing::kMonoAndKeyboard:
32    case AudioProcessing::kStereoAndKeyboard:
33      return true;
34  }
35  assert(false);
36  return false;
37}
38
39int KeyboardChannelIndex(AudioProcessing::ChannelLayout layout) {
40  switch (layout) {
41    case AudioProcessing::kMono:
42    case AudioProcessing::kStereo:
43      assert(false);
44      return -1;
45    case AudioProcessing::kMonoAndKeyboard:
46      return 1;
47    case AudioProcessing::kStereoAndKeyboard:
48      return 2;
49  }
50  assert(false);
51  return -1;
52}
53
54
55void StereoToMono(const float* left, const float* right, float* out,
56                  int samples_per_channel) {
57  for (int i = 0; i < samples_per_channel; ++i) {
58    out[i] = (left[i] + right[i]) / 2;
59  }
60}
61
62void StereoToMono(const int16_t* left, const int16_t* right, int16_t* out,
63                  int samples_per_channel) {
64  for (int i = 0; i < samples_per_channel; ++i) {
65    out[i] = (left[i] + right[i]) >> 1;
66  }
67}
68
69}  // namespace
70
71class SplitChannelBuffer {
72 public:
73  SplitChannelBuffer(int samples_per_split_channel, int num_channels)
74      : low_(samples_per_split_channel, num_channels),
75        high_(samples_per_split_channel, num_channels) {
76  }
77  ~SplitChannelBuffer() {}
78
79  int16_t* low_channel(int i) { return low_.channel(i); }
80  int16_t* high_channel(int i) { return high_.channel(i); }
81
82 private:
83  ChannelBuffer<int16_t> low_;
84  ChannelBuffer<int16_t> high_;
85};
86
87AudioBuffer::AudioBuffer(int input_samples_per_channel,
88                         int num_input_channels,
89                         int process_samples_per_channel,
90                         int num_process_channels,
91                         int output_samples_per_channel)
92  : input_samples_per_channel_(input_samples_per_channel),
93    num_input_channels_(num_input_channels),
94    proc_samples_per_channel_(process_samples_per_channel),
95    num_proc_channels_(num_process_channels),
96    output_samples_per_channel_(output_samples_per_channel),
97    samples_per_split_channel_(proc_samples_per_channel_),
98    num_mixed_channels_(0),
99    num_mixed_low_pass_channels_(0),
100    data_was_mixed_(false),
101    reference_copied_(false),
102    activity_(AudioFrame::kVadUnknown),
103    is_muted_(false),
104    data_(NULL),
105    keyboard_data_(NULL),
106    channels_(new ChannelBuffer<int16_t>(proc_samples_per_channel_,
107                                         num_proc_channels_)) {
108  assert(input_samples_per_channel_ > 0);
109  assert(proc_samples_per_channel_ > 0);
110  assert(output_samples_per_channel_ > 0);
111  assert(num_input_channels_ > 0 && num_input_channels_ <= 2);
112  assert(num_proc_channels_ <= num_input_channels);
113
114  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
115    input_buffer_.reset(new ChannelBuffer<float>(input_samples_per_channel_,
116                                                 num_proc_channels_));
117  }
118
119  if (input_samples_per_channel_ != proc_samples_per_channel_ ||
120      output_samples_per_channel_ != proc_samples_per_channel_) {
121    // Create an intermediate buffer for resampling.
122    process_buffer_.reset(new ChannelBuffer<float>(proc_samples_per_channel_,
123                                                   num_proc_channels_));
124  }
125
126  if (input_samples_per_channel_ != proc_samples_per_channel_) {
127    input_resamplers_.reserve(num_proc_channels_);
128    for (int i = 0; i < num_proc_channels_; ++i) {
129      input_resamplers_.push_back(
130          new PushSincResampler(input_samples_per_channel_,
131                                proc_samples_per_channel_));
132    }
133  }
134
135  if (output_samples_per_channel_ != proc_samples_per_channel_) {
136    output_resamplers_.reserve(num_proc_channels_);
137    for (int i = 0; i < num_proc_channels_; ++i) {
138      output_resamplers_.push_back(
139          new PushSincResampler(proc_samples_per_channel_,
140                                output_samples_per_channel_));
141    }
142  }
143
144  if (proc_samples_per_channel_ == kSamplesPer32kHzChannel) {
145    samples_per_split_channel_ = kSamplesPer16kHzChannel;
146    split_channels_.reset(new SplitChannelBuffer(samples_per_split_channel_,
147                                                 num_proc_channels_));
148    filter_states_.reset(new SplitFilterStates[num_proc_channels_]);
149  }
150}
151
152AudioBuffer::~AudioBuffer() {}
153
154void AudioBuffer::CopyFrom(const float* const* data,
155                           int samples_per_channel,
156                           AudioProcessing::ChannelLayout layout) {
157  assert(samples_per_channel == input_samples_per_channel_);
158  assert(ChannelsFromLayout(layout) == num_input_channels_);
159  InitForNewData();
160
161  if (HasKeyboardChannel(layout)) {
162    keyboard_data_ = data[KeyboardChannelIndex(layout)];
163  }
164
165  // Downmix.
166  const float* const* data_ptr = data;
167  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
168    StereoToMono(data[0],
169                 data[1],
170                 input_buffer_->channel(0),
171                 input_samples_per_channel_);
172    data_ptr = input_buffer_->channels();
173  }
174
175  // Resample.
176  if (input_samples_per_channel_ != proc_samples_per_channel_) {
177    for (int i = 0; i < num_proc_channels_; ++i) {
178      input_resamplers_[i]->Resample(data_ptr[i],
179                                     input_samples_per_channel_,
180                                     process_buffer_->channel(i),
181                                     proc_samples_per_channel_);
182    }
183    data_ptr = process_buffer_->channels();
184  }
185
186  // Convert to int16.
187  for (int i = 0; i < num_proc_channels_; ++i) {
188    ScaleAndRoundToInt16(data_ptr[i], proc_samples_per_channel_,
189                         channels_->channel(i));
190  }
191}
192
193void AudioBuffer::CopyTo(int samples_per_channel,
194                         AudioProcessing::ChannelLayout layout,
195                         float* const* data) {
196  assert(samples_per_channel == output_samples_per_channel_);
197  assert(ChannelsFromLayout(layout) == num_proc_channels_);
198
199  // Convert to float.
200  float* const* data_ptr = data;
201  if (output_samples_per_channel_ != proc_samples_per_channel_) {
202    // Convert to an intermediate buffer for subsequent resampling.
203    data_ptr = process_buffer_->channels();
204  }
205  for (int i = 0; i < num_proc_channels_; ++i) {
206    ScaleToFloat(channels_->channel(i), proc_samples_per_channel_, data_ptr[i]);
207  }
208
209  // Resample.
210  if (output_samples_per_channel_ != proc_samples_per_channel_) {
211    for (int i = 0; i < num_proc_channels_; ++i) {
212      output_resamplers_[i]->Resample(data_ptr[i],
213                                      proc_samples_per_channel_,
214                                      data[i],
215                                      output_samples_per_channel_);
216    }
217  }
218}
219
220void AudioBuffer::InitForNewData() {
221  data_ = NULL;
222  keyboard_data_ = NULL;
223  data_was_mixed_ = false;
224  num_mixed_channels_ = 0;
225  num_mixed_low_pass_channels_ = 0;
226  reference_copied_ = false;
227  activity_ = AudioFrame::kVadUnknown;
228  is_muted_ = false;
229}
230
231int16_t* AudioBuffer::data(int channel) const {
232  assert(channel >= 0 && channel < num_proc_channels_);
233  if (data_ != NULL) {
234    return data_;
235  }
236
237  return channels_->channel(channel);
238}
239
240int16_t* AudioBuffer::low_pass_split_data(int channel) const {
241  assert(channel >= 0 && channel < num_proc_channels_);
242  if (split_channels_.get() == NULL) {
243    return data(channel);
244  }
245
246  return split_channels_->low_channel(channel);
247}
248
249int16_t* AudioBuffer::high_pass_split_data(int channel) const {
250  assert(channel >= 0 && channel < num_proc_channels_);
251  if (split_channels_.get() == NULL) {
252    return NULL;
253  }
254
255  return split_channels_->high_channel(channel);
256}
257
258int16_t* AudioBuffer::mixed_data(int channel) const {
259  assert(channel >= 0 && channel < num_mixed_channels_);
260
261  return mixed_channels_->channel(channel);
262}
263
264int16_t* AudioBuffer::mixed_low_pass_data(int channel) const {
265  assert(channel >= 0 && channel < num_mixed_low_pass_channels_);
266
267  return mixed_low_pass_channels_->channel(channel);
268}
269
270int16_t* AudioBuffer::low_pass_reference(int channel) const {
271  assert(channel >= 0 && channel < num_proc_channels_);
272  if (!reference_copied_) {
273    return NULL;
274  }
275
276  return low_pass_reference_channels_->channel(channel);
277}
278
279const float* AudioBuffer::keyboard_data() const {
280  return keyboard_data_;
281}
282
283SplitFilterStates* AudioBuffer::filter_states(int channel) const {
284  assert(channel >= 0 && channel < num_proc_channels_);
285  return &filter_states_[channel];
286}
287
288void AudioBuffer::set_activity(AudioFrame::VADActivity activity) {
289  activity_ = activity;
290}
291
292AudioFrame::VADActivity AudioBuffer::activity() const {
293  return activity_;
294}
295
296bool AudioBuffer::is_muted() const {
297  return is_muted_;
298}
299
300int AudioBuffer::num_channels() const {
301  return num_proc_channels_;
302}
303
304int AudioBuffer::samples_per_channel() const {
305  return proc_samples_per_channel_;
306}
307
308int AudioBuffer::samples_per_split_channel() const {
309  return samples_per_split_channel_;
310}
311
312int AudioBuffer::samples_per_keyboard_channel() const {
313  // We don't resample the keyboard channel.
314  return input_samples_per_channel_;
315}
316
317// TODO(andrew): Do deinterleaving and mixing in one step?
318void AudioBuffer::DeinterleaveFrom(AudioFrame* frame) {
319  assert(proc_samples_per_channel_ == input_samples_per_channel_);
320  assert(num_proc_channels_ == num_input_channels_);
321  assert(frame->num_channels_ == num_proc_channels_);
322  assert(frame->samples_per_channel_ ==  proc_samples_per_channel_);
323  InitForNewData();
324  activity_ = frame->vad_activity_;
325  if (frame->energy_ == 0) {
326    is_muted_ = true;
327  }
328
329  if (num_proc_channels_ == 1) {
330    // We can get away with a pointer assignment in this case.
331    data_ = frame->data_;
332    return;
333  }
334
335  int16_t* interleaved = frame->data_;
336  for (int i = 0; i < num_proc_channels_; i++) {
337    int16_t* deinterleaved = channels_->channel(i);
338    int interleaved_idx = i;
339    for (int j = 0; j < proc_samples_per_channel_; j++) {
340      deinterleaved[j] = interleaved[interleaved_idx];
341      interleaved_idx += num_proc_channels_;
342    }
343  }
344}
345
346void AudioBuffer::InterleaveTo(AudioFrame* frame, bool data_changed) const {
347  assert(proc_samples_per_channel_ == output_samples_per_channel_);
348  assert(num_proc_channels_ == num_input_channels_);
349  assert(frame->num_channels_ == num_proc_channels_);
350  assert(frame->samples_per_channel_ == proc_samples_per_channel_);
351  frame->vad_activity_ = activity_;
352
353  if (!data_changed) {
354    return;
355  }
356
357  if (num_proc_channels_ == 1) {
358    if (data_was_mixed_) {
359      memcpy(frame->data_,
360             channels_->channel(0),
361             sizeof(int16_t) * proc_samples_per_channel_);
362    } else {
363      // These should point to the same buffer in this case.
364      assert(data_ == frame->data_);
365    }
366
367    return;
368  }
369
370  int16_t* interleaved = frame->data_;
371  for (int i = 0; i < num_proc_channels_; i++) {
372    int16_t* deinterleaved = channels_->channel(i);
373    int interleaved_idx = i;
374    for (int j = 0; j < proc_samples_per_channel_; j++) {
375      interleaved[interleaved_idx] = deinterleaved[j];
376      interleaved_idx += num_proc_channels_;
377    }
378  }
379}
380
381void AudioBuffer::CopyAndMix(int num_mixed_channels) {
382  // We currently only support the stereo to mono case.
383  assert(num_proc_channels_ == 2);
384  assert(num_mixed_channels == 1);
385  if (!mixed_channels_.get()) {
386    mixed_channels_.reset(
387        new ChannelBuffer<int16_t>(proc_samples_per_channel_,
388                                   num_mixed_channels));
389  }
390
391  StereoToMono(channels_->channel(0),
392               channels_->channel(1),
393               mixed_channels_->channel(0),
394               proc_samples_per_channel_);
395
396  num_mixed_channels_ = num_mixed_channels;
397}
398
399void AudioBuffer::CopyAndMixLowPass(int num_mixed_channels) {
400  // We currently only support the stereo to mono case.
401  assert(num_proc_channels_ == 2);
402  assert(num_mixed_channels == 1);
403  if (!mixed_low_pass_channels_.get()) {
404    mixed_low_pass_channels_.reset(
405        new ChannelBuffer<int16_t>(samples_per_split_channel_,
406                                   num_mixed_channels));
407  }
408
409  StereoToMono(low_pass_split_data(0),
410               low_pass_split_data(1),
411               mixed_low_pass_channels_->channel(0),
412               samples_per_split_channel_);
413
414  num_mixed_low_pass_channels_ = num_mixed_channels;
415}
416
417void AudioBuffer::CopyLowPassToReference() {
418  reference_copied_ = true;
419  if (!low_pass_reference_channels_.get()) {
420    low_pass_reference_channels_.reset(
421        new ChannelBuffer<int16_t>(samples_per_split_channel_,
422                                   num_proc_channels_));
423  }
424  for (int i = 0; i < num_proc_channels_; i++) {
425    low_pass_reference_channels_->CopyFrom(low_pass_split_data(i), i);
426  }
427}
428
429}  // namespace webrtc
430