media/audio/audio_output_resampler.cc

// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/audio/audio_output_resampler.h"

#include "base/bind.h"
#include "base/bind_helpers.h"
#include "base/command_line.h"
#include "base/compiler_specific.h"
#include "base/message_loop.h"
#include "base/metrics/histogram.h"
#include "base/time.h"
#include "media/audio/audio_io.h"
#include "media/audio/audio_output_dispatcher_impl.h"
#include "media/audio/audio_output_proxy.h"
#include "media/audio/audio_util.h"
#include "media/audio/sample_rates.h"
#include "media/base/audio_pull_fifo.h"
#include "media/base/channel_mixer.h"
#include "media/base/limits.h"
#include "media/base/media_switches.h"
#include "media/base/multi_channel_resampler.h"

namespace media {

class OnMoreDataResampler : public AudioOutputStream::AudioSourceCallback {
 public:
  OnMoreDataResampler(double io_ratio,
                      const AudioParameters& input_params,
                      const AudioParameters& output_params);
  virtual ~OnMoreDataResampler();

  // AudioSourceCallback interface.
  virtual int OnMoreData(AudioBus* dest,
                         AudioBuffersState buffers_state) OVERRIDE;
  virtual int OnMoreIOData(AudioBus* source,
                           AudioBus* dest,
                           AudioBuffersState buffers_state) OVERRIDE;
  virtual void OnError(AudioOutputStream* stream, int code) OVERRIDE;
  virtual void WaitTillDataReady() OVERRIDE;

  // Sets |source_callback_|.  If this is not a new object, then Stop() must be
  // called before Start().
  void Start(AudioOutputStream::AudioSourceCallback* callback);

  // Clears |source_callback_| and flushes the resampler.
  void Stop();

 private:
  // Called by MultiChannelResampler when more data is necessary.
  void ProvideInput(AudioBus* audio_bus);

  // Called by AudioPullFifo when more data is necessary.  Requires
  // |source_lock_| to have been acquired.
  void SourceCallback_Locked(AudioBus* audio_bus);

  // Passes through |source| to the |source_callback_| OnMoreIOData() call.
  void SourceIOCallback_Locked(AudioBus* source, AudioBus* dest);

  // Ratio of input bytes to output bytes used to correct playback delay with
  // regard to buffering and resampling.
  double io_ratio_;

  // Source callback and associated lock.
  base::Lock source_lock_;
  AudioOutputStream::AudioSourceCallback* source_callback_;

  // Last AudioBuffersState object received via OnMoreData(), used to correct
  // playback delay by ProvideInput() and passed on to |source_callback_|.
  AudioBuffersState current_buffers_state_;

  // Total number of bytes (in terms of output parameters) stored in resampler
  // or FIFO buffers which have not been sent to the audio device.
  int outstanding_audio_bytes_;

  // Used to buffer data between the client and the output device in cases where
  // the client buffer size is not the same as the output device buffer size.
  // Bound to SourceCallback_Locked() so must only be used when |source_lock_|
  // has already been acquired.
  scoped_ptr<AudioPullFifo> audio_fifo_;

  // Handles resampling.
  scoped_ptr<MultiChannelResampler> resampler_;

  // Handles channel transforms.  |unmixed_audio_| is a temporary destination
  // for audio data before it goes into the channel mixer.
  scoped_ptr<ChannelMixer> channel_mixer_;
  scoped_ptr<AudioBus> unmixed_audio_;

  int output_bytes_per_frame_;
  int input_bytes_per_frame_;

  // Since resampling is expensive, figure out if we should downmix channels
  // before resampling.
  bool downmix_early_;

  DISALLOW_COPY_AND_ASSIGN(OnMoreDataResampler);
};

// Record UMA statistics for hardware output configuration.
static void RecordStats(const AudioParameters& output_params) {
  UMA_HISTOGRAM_ENUMERATION(
      "Media.HardwareAudioBitsPerChannel", output_params.bits_per_sample(),
      limits::kMaxBitsPerSample);
  UMA_HISTOGRAM_ENUMERATION(
      "Media.HardwareAudioChannelLayout", output_params.channel_layout(),
      CHANNEL_LAYOUT_MAX);
  UMA_HISTOGRAM_ENUMERATION(
      "Media.HardwareAudioChannelCount", output_params.channels(),
      limits::kMaxChannels);

  AudioSampleRate asr = media::AsAudioSampleRate(output_params.sample_rate());
  if (asr != kUnexpectedAudioSampleRate) {
    UMA_HISTOGRAM_ENUMERATION(
        "Media.HardwareAudioSamplesPerSecond", asr, kUnexpectedAudioSampleRate);
  } else {
    UMA_HISTOGRAM_COUNTS(
        "Media.HardwareAudioSamplesPerSecondUnexpected",
        output_params.sample_rate());
  }
}

// Record UMA statistics for hardware output configuration after fallback.
static void RecordFallbackStats(const AudioParameters& output_params) {
  UMA_HISTOGRAM_BOOLEAN("Media.FallbackToHighLatencyAudioPath", true);
  UMA_HISTOGRAM_ENUMERATION(
      "Media.FallbackHardwareAudioBitsPerChannel",
      output_params.bits_per_sample(), limits::kMaxBitsPerSample);
  UMA_HISTOGRAM_ENUMERATION(
      "Media.FallbackHardwareAudioChannelLayout",
      output_params.channel_layout(), CHANNEL_LAYOUT_MAX);
  UMA_HISTOGRAM_ENUMERATION(
      "Media.FallbackHardwareAudioChannelCount",
      output_params.channels(), limits::kMaxChannels);

  AudioSampleRate asr = media::AsAudioSampleRate(output_params.sample_rate());
  if (asr != kUnexpectedAudioSampleRate) {
    UMA_HISTOGRAM_ENUMERATION(
        "Media.FallbackHardwareAudioSamplesPerSecond",
        asr, kUnexpectedAudioSampleRate);
  } else {
    UMA_HISTOGRAM_COUNTS(
        "Media.FallbackHardwareAudioSamplesPerSecondUnexpected",
        output_params.sample_rate());
  }
}

// Converts low latency based |output_params| into high latency appropriate
// output parameters in error situations.
static AudioParameters SetupFallbackParams(
    const AudioParameters& input_params, const AudioParameters& output_params) {
  // Choose AudioParameters appropriate for opening the device in high latency
  // mode.  |kMinLowLatencyFrameSize| is arbitrarily based on Pepper Flash's
  // MAXIMUM frame size for low latency.
  static const int kMinLowLatencyFrameSize = 2048;
  int frames_per_buffer = std::min(
      std::max(input_params.frames_per_buffer(), kMinLowLatencyFrameSize),
      static_cast<int>(
          GetHighLatencyOutputBufferSize(input_params.sample_rate())));

  return AudioParameters(
      AudioParameters::AUDIO_PCM_LINEAR, input_params.channel_layout(),
      input_params.sample_rate(), input_params.bits_per_sample(),
      frames_per_buffer);
}

AudioOutputResampler::AudioOutputResampler(AudioManager* audio_manager,
                                           const AudioParameters& input_params,
                                           const AudioParameters& output_params,
                                           const base::TimeDelta& close_delay)
    : AudioOutputDispatcher(audio_manager, input_params),
      io_ratio_(1),
      close_delay_(close_delay),
      output_params_(output_params),
      streams_opened_(false) {
  DCHECK(input_params.IsValid());
  DCHECK(output_params.IsValid());
  DCHECK_EQ(output_params_.format(), AudioParameters::AUDIO_PCM_LOW_LATENCY);

  // Record UMA statistics for the hardware configuration.
  RecordStats(output_params);

  Initialize();
}

AudioOutputResampler::~AudioOutputResampler() {
  DCHECK(callbacks_.empty());
}

void AudioOutputResampler::Initialize() {
  DCHECK(!streams_opened_);
  DCHECK(callbacks_.empty());

  io_ratio_ = 1;

  // Only resample or rebuffer if the input parameters don't match the output
  // parameters to avoid any unnecessary work.
  if (params_.channels() != output_params_.channels() ||
      params_.sample_rate() != output_params_.sample_rate() ||
      params_.bits_per_sample() != output_params_.bits_per_sample() ||
      params_.frames_per_buffer() != output_params_.frames_per_buffer()) {
    if (params_.sample_rate() != output_params_.sample_rate()) {
      double io_sample_rate_ratio = params_.sample_rate() /
          static_cast<double>(output_params_.sample_rate());
      // Include the I/O resampling ratio in our global I/O ratio.
      io_ratio_ *= io_sample_rate_ratio;
    }

    // Include bits per channel differences.
    io_ratio_ *= static_cast<double>(params_.bits_per_sample()) /
        output_params_.bits_per_sample();

    // Include channel count differences.
    io_ratio_ *= static_cast<double>(params_.channels()) /
        output_params_.channels();

    DVLOG(1) << "I/O ratio is " << io_ratio_;
  } else {
    DVLOG(1) << "Input and output params are the same; in pass-through mode.";
  }

  // TODO(dalecurtis): All this code should be merged into AudioOutputMixer once
  // we've stabilized the issues there.
  dispatcher_ = new AudioOutputDispatcherImpl(
      audio_manager_, output_params_, close_delay_);
}

bool AudioOutputResampler::OpenStream() {
  DCHECK_EQ(MessageLoop::current(), message_loop_);

  if (dispatcher_->OpenStream()) {
    // Only record the UMA statistic if we didn't fallback during construction
    // and only for the first stream we open.
    if (!streams_opened_ &&
        output_params_.format() == AudioParameters::AUDIO_PCM_LOW_LATENCY) {
      UMA_HISTOGRAM_BOOLEAN("Media.FallbackToHighLatencyAudioPath", false);
    }
    streams_opened_ = true;
    return true;
  }

  // If we've already tried to open the stream in high latency mode or we've
  // successfully opened a stream previously, there's nothing more to be done.
  if (output_params_.format() == AudioParameters::AUDIO_PCM_LINEAR ||
      streams_opened_ || !callbacks_.empty()) {
    return false;
  }

  DCHECK_EQ(output_params_.format(), AudioParameters::AUDIO_PCM_LOW_LATENCY);

  if (CommandLine::ForCurrentProcess()->HasSwitch(
          switches::kDisableAudioFallback)) {
    LOG(ERROR) << "Open failed and automatic fallback to high latency audio "
               << "path is disabled, aborting.";
    return false;
  }

  DLOG(ERROR) << "Unable to open audio device in low latency mode.  Falling "
              << "back to high latency audio output.";

  // Record UMA statistics about the hardware which triggered the failure so
  // we can debug and triage later.
  RecordFallbackStats(output_params_);
  output_params_ = SetupFallbackParams(params_, output_params_);
  Initialize();

  // Retry, if this fails, there's nothing left to do but report the error back.
  return dispatcher_->OpenStream();
}

bool AudioOutputResampler::StartStream(
    AudioOutputStream::AudioSourceCallback* callback,
    AudioOutputProxy* stream_proxy) {
  DCHECK_EQ(MessageLoop::current(), message_loop_);

  OnMoreDataResampler* resampler_callback = NULL;
  CallbackMap::iterator it = callbacks_.find(stream_proxy);
  if (it == callbacks_.end()) {
    resampler_callback = new OnMoreDataResampler(
        io_ratio_, params_, output_params_);
    callbacks_[stream_proxy] = resampler_callback;
  } else {
    resampler_callback = it->second;
  }
  resampler_callback->Start(callback);
  return dispatcher_->StartStream(resampler_callback, stream_proxy);
}

void AudioOutputResampler::StreamVolumeSet(AudioOutputProxy* stream_proxy,
                                           double volume) {
  DCHECK_EQ(MessageLoop::current(), message_loop_);
  dispatcher_->StreamVolumeSet(stream_proxy, volume);
}

void AudioOutputResampler::StopStream(AudioOutputProxy* stream_proxy) {
  DCHECK_EQ(MessageLoop::current(), message_loop_);
  dispatcher_->StopStream(stream_proxy);

  // Now that StopStream() has completed the underlying physical stream should
  // be stopped and no longer calling OnMoreData(), making it safe to Stop() the
  // OnMoreDataResampler.
  CallbackMap::iterator it = callbacks_.find(stream_proxy);
  if (it != callbacks_.end())
    it->second->Stop();
}

void AudioOutputResampler::CloseStream(AudioOutputProxy* stream_proxy) {
  DCHECK_EQ(MessageLoop::current(), message_loop_);
  dispatcher_->CloseStream(stream_proxy);

  // We assume that StopStream() is always called prior to CloseStream(), so
  // that it is safe to delete the OnMoreDataResampler here.
  CallbackMap::iterator it = callbacks_.find(stream_proxy);
  if (it != callbacks_.end()) {
    delete it->second;
    callbacks_.erase(it);
  }
}

void AudioOutputResampler::Shutdown() {
  DCHECK_EQ(MessageLoop::current(), message_loop_);

  // No AudioOutputProxy objects should hold a reference to us when we get
  // to this stage.
  DCHECK(HasOneRef()) << "Only the AudioManager should hold a reference";

  dispatcher_->Shutdown();
  DCHECK(callbacks_.empty());
}

OnMoreDataResampler::OnMoreDataResampler(
    double io_ratio, const AudioParameters& input_params,
    const AudioParameters& output_params)
    : io_ratio_(io_ratio),
      source_callback_(NULL),
      outstanding_audio_bytes_(0),
      output_bytes_per_frame_(output_params.GetBytesPerFrame()),
      input_bytes_per_frame_(input_params.GetBytesPerFrame()),
      downmix_early_(false) {
  // Handle different input and output channel layouts.
  if (input_params.channel_layout() != output_params.channel_layout()) {
    DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout()
             << " to " << output_params.channel_layout() << "; from "
             << input_params.channels() << " channels to "
             << output_params.channels() << " channels.";
    channel_mixer_.reset(new ChannelMixer(
        input_params.channel_layout(), output_params.channel_layout()));

    // Pare off data as early as we can for efficiency.
    downmix_early_ = input_params.channels() > output_params.channels();
    if (downmix_early_) {
      DVLOG(1) << "Remixing channel layout prior to resampling.";
      // If we're downmixing early we need a temporary AudioBus which matches
      // the the input channel count and input frame size since we're passing
      // |unmixed_audio_| directly to the |source_callback_|.
      unmixed_audio_ = AudioBus::Create(input_params);
    } else {
      // Instead, if we're not downmixing early we need a temporary AudioBus
      // which matches the input channel count but uses the output frame size
      // since we'll mix into the AudioBus from the output stream.
      unmixed_audio_ = AudioBus::Create(
          input_params.channels(), output_params.frames_per_buffer());
    }
  }

  // Only resample if necessary since it's expensive.
  if (input_params.sample_rate() != output_params.sample_rate()) {
    DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to "
             << output_params.sample_rate();
    double io_sample_rate_ratio = input_params.sample_rate() /
        static_cast<double>(output_params.sample_rate());
    resampler_.reset(new MultiChannelResampler(
        downmix_early_ ? output_params.channels() :
            input_params.channels(),
        io_sample_rate_ratio, base::Bind(
            &OnMoreDataResampler::ProvideInput, base::Unretained(this))));
  }

  // Since the resampler / output device may want a different buffer size than
  // the caller asked for, we need to use a FIFO to ensure that both sides
  // read in chunk sizes they're configured for.
  if (input_params.sample_rate() != output_params.sample_rate() ||
      input_params.frames_per_buffer() != output_params.frames_per_buffer()) {
    DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer()
             << " to " << output_params.frames_per_buffer();
    audio_fifo_.reset(new AudioPullFifo(
        downmix_early_ ? output_params.channels() :
            input_params.channels(),
        input_params.frames_per_buffer(), base::Bind(
            &OnMoreDataResampler::SourceCallback_Locked,
            base::Unretained(this))));
  }
}

OnMoreDataResampler::~OnMoreDataResampler() {}

void OnMoreDataResampler::Start(
    AudioOutputStream::AudioSourceCallback* callback) {
  base::AutoLock auto_lock(source_lock_);
  DCHECK(!source_callback_);
  source_callback_ = callback;
}

void OnMoreDataResampler::Stop() {
  base::AutoLock auto_lock(source_lock_);
  source_callback_ = NULL;
  outstanding_audio_bytes_ = 0;
  if (audio_fifo_)
    audio_fifo_->Clear();
  if (resampler_)
    resampler_->Flush();
}

int OnMoreDataResampler::OnMoreData(AudioBus* dest,
                                    AudioBuffersState buffers_state) {
  return OnMoreIOData(NULL, dest, buffers_state);
}

int OnMoreDataResampler::OnMoreIOData(AudioBus* source,
                                      AudioBus* dest,
                                      AudioBuffersState buffers_state) {
  base::AutoLock auto_lock(source_lock_);
  // While we waited for |source_lock_| the callback might have been cleared.
  if (!source_callback_) {
    dest->Zero();
    return dest->frames();
  }

  current_buffers_state_ = buffers_state;

  bool needs_mixing = channel_mixer_ && !downmix_early_;
  AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest;

  if (!resampler_ && !audio_fifo_) {
    // We have no internal buffers, so clear any outstanding audio data.
    outstanding_audio_bytes_ = 0;
    SourceIOCallback_Locked(source, temp_dest);
  } else {
    if (resampler_)
      resampler_->Resample(temp_dest, temp_dest->frames());
    else
      ProvideInput(temp_dest);

    // Calculate how much data is left in the internal FIFO and resampler.
    outstanding_audio_bytes_ -= temp_dest->frames() * output_bytes_per_frame_;
  }

  if (needs_mixing) {
    DCHECK_EQ(temp_dest->frames(), dest->frames());
    channel_mixer_->Transform(temp_dest, dest);
  }

  // Due to rounding errors while multiplying against |io_ratio_|,
  // |outstanding_audio_bytes_| might (rarely) slip below zero.
  if (outstanding_audio_bytes_ < 0) {
    DLOG(ERROR) << "Outstanding audio bytes went negative! Value: "
                << outstanding_audio_bytes_;
    outstanding_audio_bytes_ = 0;
  }

  // Always return the full number of frames requested, ProvideInput() will pad
  // with silence if it wasn't able to acquire enough data.
  return dest->frames();
}

void OnMoreDataResampler::SourceCallback_Locked(AudioBus* dest) {
  SourceIOCallback_Locked(NULL, dest);
}

void OnMoreDataResampler::SourceIOCallback_Locked(AudioBus* source,
                                                  AudioBus* dest) {
  source_lock_.AssertAcquired();

  // Adjust playback delay to include the state of the internal buffers used by
  // the resampler and/or the FIFO.  Since the sample rate and bits per channel
  // may be different, we need to scale this value appropriately.
  AudioBuffersState new_buffers_state;
  new_buffers_state.pending_bytes = io_ratio_ *
      (current_buffers_state_.total_bytes() + outstanding_audio_bytes_);

  bool needs_downmix = channel_mixer_ && downmix_early_;
  AudioBus* temp_dest = needs_downmix ? unmixed_audio_.get() : dest;

  // Retrieve data from the original callback.  Zero any unfilled frames.
  int frames = source_callback_->OnMoreIOData(
      source, temp_dest, new_buffers_state);
  if (frames < temp_dest->frames())
    temp_dest->ZeroFramesPartial(frames, temp_dest->frames() - frames);

  // Scale the number of frames we got back in terms of input bytes to output
  // bytes accordingly.
  outstanding_audio_bytes_ +=
      (temp_dest->frames() * input_bytes_per_frame_) / io_ratio_;

  if (needs_downmix) {
    DCHECK_EQ(temp_dest->frames(), dest->frames());
    channel_mixer_->Transform(temp_dest, dest);
  }
}

void OnMoreDataResampler::ProvideInput(AudioBus* audio_bus) {
  audio_fifo_->Consume(audio_bus, audio_bus->frames());
}

void OnMoreDataResampler::OnError(AudioOutputStream* stream, int code) {
  base::AutoLock auto_lock(source_lock_);
  if (source_callback_)
    source_callback_->OnError(stream, code);
}

void OnMoreDataResampler::WaitTillDataReady() {
  base::AutoLock auto_lock(source_lock_);
  if (source_callback_ && !outstanding_audio_bytes_)
    source_callback_->WaitTillDataReady();
}

}  // namespace media