1b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson/* 2b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * Copyright (C) 2011 The Android Open Source Project 3b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * 4b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * Licensed under the Apache License, Version 2.0 (the "License"); 5b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * you may not use this file except in compliance with the License. 6b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * You may obtain a copy of the License at 7b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * 8b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * http://www.apache.org/licenses/LICENSE-2.0 9b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * 10b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * Unless required by applicable law or agreed to in writing, software 11b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * distributed under the License is distributed on an "AS IS" BASIS, 12b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * See the License for the specific language governing permissions and 14b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson * limitations under the License. 15b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson */ 16b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 17b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#include "sola_time_scaler.h" 18b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 19b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#include <math.h> 20b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#include <hlogging.h> 21b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#include <algorithm> 22b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 23b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#include "ring_buffer.h" 24b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 25b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#define FLAGS_sola_ring_buffer 2.0 26b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson#define FLAGS_sola_enable_correlation true 27b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 28b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 29b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonnamespace video_editing { 30b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 31b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Returns a cross-correlation score for the specified buffers. 32b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonint SolaAnalyzer::Correlate(const float* buffer1, const float* buffer2, 33b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int num_frames) { 34b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 35b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 36b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int score = 0; 37b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_frames *= num_channels_; 38b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson while (num_frames-- > 0) { 39b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Increment the score if the sign bits match. 40b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson score += ((bit_cast<int32>(*buffer1++) ^ bit_cast<int32>(*buffer2++)) >= 0) 41b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson ? 1 : 0; 42b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 43b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return score; 44b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 45b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 46b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Trivial SolaAnalyzer class to bypass correlation. 47b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonclass SolaBypassAnalyzer : public SolaAnalyzer { 48b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson public: 49b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson SolaBypassAnalyzer() { } 50b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson virtual int Correlate(const float*, const float*, int num_frames) { 51b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return num_frames * num_channels_; 52b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 53b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson}; 54b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 55b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 56b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Default constructor. 57b83ad73794088498d6d38cd3b4fc9311f505d051Hugo HudsonSolaTimeScaler::SolaTimeScaler() 58b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson : input_buffer_(NULL), output_buffer_(NULL), analyzer_(NULL) { 59b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson sample_rate_ = 0; 60b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_channels_ = 0; 61b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 62b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson draining_ = false; 63b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson initialized_ = false; 64b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 65b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 66b83ad73794088498d6d38cd3b4fc9311f505d051Hugo HudsonSolaTimeScaler::~SolaTimeScaler() { 67b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete input_buffer_; 68b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete output_buffer_; 69b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete analyzer_; 70b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 71b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 72b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Injects a SolaAnalyzer instance for analyzing signal frames. 73b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::set_analyzer(SolaAnalyzer* analyzer) { 74b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson MutexLock lock(&mutex_); // lock out processing while updating 75b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete analyzer_; 76b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson analyzer_ = analyzer; 77b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 78b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 79b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Initializes a SOLA timescaler. 80b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::Init(double sample_rate, 81b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int num_channels, 82b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson double initial_speed, 83b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson double window_duration, 84b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson double overlap_duration) { 85b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson MutexLock lock(&mutex_); // lock out processing while updating 86b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 87b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson sample_rate_ = sample_rate; 88b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_channels_ = num_channels; 89b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson speed_ = initial_speed; 90b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson window_duration_ = window_duration; 91b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson overlap_duration_ = overlap_duration; 92b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 93b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson initialized_ = true; 94b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson GenerateParameters(); 95b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson Reset(); 96b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 97b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 98b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Adjusts the rate scaling factor. 99b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::set_speed(double speed) { 100b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson MutexLock lock(&mutex_); // lock out processing while updating 101b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 102b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson speed_ = speed; 103b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson GenerateParameters(); 104b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 105b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 106b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Generates processing parameters from the current settings. 107b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::GenerateParameters() { 108b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (speed_ < 0.1) { 109b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson LOGE("Requested speed %fx limited to 0.1x", speed_); 110b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson speed_ = 0.1; 111b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else if (speed_ > 8.0) { 112b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson LOGE("Requested speed %fx limited to 8.0x", speed_); 113b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson speed_ = 8.0; 114b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 115b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 116b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson ratio_ = 1.0 / speed_; 117b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 118b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_window_frames_ = nearbyint(sample_rate_ * window_duration_); 119b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 120b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Limit the overlap to half the window size, and round up to an odd number. 121b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Half of overlap window (rounded down) is also a useful number. 122b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson overlap_duration_ = min(overlap_duration_, window_duration_ / 2.0); 123b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_overlap_frames_ = nearbyint(sample_rate_ * overlap_duration_); 124b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_overlap_frames_ |= 1; 125b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson half_overlap_frames_ = num_overlap_frames_ >> 1; 126b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 127b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (speed_ >= 1.) { 128b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // For compression (speed up), adjacent input windows overlap in the output. 129b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_window_offset_ = num_window_frames_; 130b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson target_merge_offset_ = nearbyint(num_window_frames_ * ratio_); 131b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else { 132b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // For expansion (slow down), each input window start point overlaps the 133b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // previous, and they are placed adjacently in the output 134b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // (+/- half the overlap size). 135b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_window_offset_ = nearbyint(num_window_frames_ * speed_); 136b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson target_merge_offset_ = num_window_frames_; 137b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 138b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 139b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Make sure we copy enough extra data to be able to perform a 140b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // frame correlation over the range of target merge point +/- half overlap, 141b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // even when the previous merge point was adjusted backwards a half overlap. 142b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson max_frames_to_merge_ = max(num_window_frames_, 143b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson target_merge_offset_ + (2 * num_overlap_frames_)); 144b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson min_output_to_hold_= 145b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson max_frames_to_merge_ + num_overlap_frames_ - target_merge_offset_; 146b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 147b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 148b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// The input buffer has one writer and reader. 149b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// The output buffer has one reader/updater, and one reader/consumer. 150b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonstatic const int kInputReader = 0; 151b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonstatic const int kOutputAnalysis = 0; 152b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonstatic const int kOutputConsumer = 1; 153b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 154b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::Reset() { 155b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 156b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson double duration = max(FLAGS_sola_ring_buffer, 20. * window_duration_); 157b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson draining_ = false; 158b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 159b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete input_buffer_; 160b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_buffer_ = new RingBuffer(); 161b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_buffer_->Init(static_cast<int> 162b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson (sample_rate_ * duration), num_channels_, 1); 163b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 164b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson delete output_buffer_; 165b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_ = new RingBuffer(); 166b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Init(static_cast<int> 167b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson (sample_rate_ * ratio_ * duration), num_channels_, 2); 168b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 169b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (analyzer_ == NULL) { 170b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (FLAGS_sola_enable_correlation) { 171b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson analyzer_ = new SolaAnalyzer(); 172b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else { 173b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson analyzer_ = new SolaBypassAnalyzer(); 174b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 175b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 176b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson analyzer_->Init(sample_rate_, num_channels_); 177b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 178b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 179b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Returns the number of frames that the input buffer can accept. 180b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonint SolaTimeScaler::input_limit() const { 181b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 182b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return input_buffer_->overhead(); 183b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 184b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 185b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Returns the number of available output frames. 186b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonint SolaTimeScaler::available() { 187b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 188b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 189b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int available = output_buffer_->available(kOutputConsumer); 190b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (available > min_output_to_hold_) { 191b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson available -= min_output_to_hold_; 192b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else if (draining_) { 193b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson Process(); 194b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson available = output_buffer_->available(kOutputConsumer); 195b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (available > min_output_to_hold_) { 196b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson available -= min_output_to_hold_; 197b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 198b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else { 199b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson available = 0; 200b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 201b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return available; 202b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 203b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 204b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonvoid SolaTimeScaler::Drain() { 205b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 206b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 207b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson draining_ = true; 208b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 209b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 210b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 211b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Feeds audio to the timescaler, and processes as much data as possible. 212b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonint SolaTimeScaler::InjectSamples(float* buffer, int num_frames) { 213b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 214b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 215b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Do not write more frames than the buffer can accept. 216b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_frames = min(input_limit(), num_frames); 217b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (!num_frames) { 218b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return 0; 219b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 220b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 221b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Copy samples to the input buffer and then process whatever can be consumed. 222b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_buffer_->Write(buffer, num_frames); 223b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson Process(); 224b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return num_frames; 225b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 226b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 227b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Retrieves audio data from the timescaler. 228b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonint SolaTimeScaler::RetrieveSamples(float* buffer, int num_frames) { 229b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 230b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 231b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Do not read more frames than available. 232b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_frames = min(available(), num_frames); 233b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (!num_frames) { 234b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return 0; 235b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 236b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 237b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Copy(kOutputConsumer, buffer, num_frames); 238b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Seek(kOutputConsumer, 239b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Tell(kOutputConsumer) + num_frames); 240b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 241b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return num_frames; 242b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 243b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 244b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson// Munges input samples to produce output. 245b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudsonbool SolaTimeScaler::Process() { 246b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson CHECK(initialized_); 247b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson bool generated_data = false; 248b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 249b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // We can only process data if there is sufficient input available 250b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // (or we are draining the latency), and there is sufficient room 251b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // for output to be merged. 252b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson while (((input_buffer_->available(kInputReader) > max_frames_to_merge_) || 253b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson draining_) && (output_buffer_->overhead() >= max_frames_to_merge_)) { 254b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson MutexLock lock(&mutex_); // lock out updates while processing each window 255b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 256b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Determine the number of samples to merge into the output. 257b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int input_count = 258b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson min(input_buffer_->available(kInputReader), max_frames_to_merge_); 259b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (input_count == 0) { 260b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson break; 261b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 262b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // The input reader always points to the next window to process. 263b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float* input_pointer = input_buffer_->GetPointer(kInputReader, input_count); 264b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 265b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // The analysis reader always points to the ideal target merge point, 266b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // minus half an overlap window (ie, the starting point for correlation). 267b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // That means the available data from that point equals the number 268b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // of samples that must be cross-faded. 269b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int output_merge_cnt = output_buffer_->available(kOutputAnalysis); 270b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float* output_pointer = 271b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->GetPointer(kOutputAnalysis, output_merge_cnt); 272b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 273b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // If there is not enough data to do a proper correlation, 274b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // just merge at the ideal target point. Otherwise, 275b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // find the best correlation score, working from the center out. 276b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int merge_offset = min(output_merge_cnt, half_overlap_frames_); 277b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 278b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if ((output_merge_cnt >= (2 * num_overlap_frames_)) && 279b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson (input_count >= num_overlap_frames_)) { 280b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int best_offset = merge_offset; 281b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int best_score = 0; 282b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int score; 283b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson for (int i = 0; i <= half_overlap_frames_; ++i) { 284b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson score = analyzer_->Correlate(input_pointer, 285b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_pointer + ((merge_offset + i) * num_channels_), 286b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_overlap_frames_); 287b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (score > best_score) { 288b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson best_score = score; 289b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson best_offset = merge_offset + i; 290b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (score == (num_overlap_frames_ * num_channels_)) { 291b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson break; // It doesn't get better than perfect. 292b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 293b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 294b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (i > 0) { 295b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson score = analyzer_->Correlate(input_pointer, 296b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_pointer + ((merge_offset - i) * num_channels_), 297b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson num_overlap_frames_); 298b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (score > best_score) { 299b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson best_score = score; 300b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson best_offset = merge_offset - i; 301b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (score == (num_overlap_frames_ * num_channels_)) { 302b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson break; // It doesn't get better than perfect. 303b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 304b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 305b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 306b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 307b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson merge_offset = best_offset; 308b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } else if ((output_merge_cnt > 0) && !draining_) { 309b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson LOGE("no correlation performed"); 310b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 311b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 312b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Crossfade the overlap between input and output, and then 313b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // copy in the remaining input. 314b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int crossfade_count = max(0, (output_merge_cnt - merge_offset)); 315b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson crossfade_count = min(crossfade_count, input_count); 316b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int remaining_count = input_count - crossfade_count; 317b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 318b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float* merge_pointer = output_pointer + (merge_offset * num_channels_); 319b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float flt_count = static_cast<float>(crossfade_count); 320b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson for (int i = 0; i < crossfade_count; ++i) { 321b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Linear cross-fade, for now. 322b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float input_scale = static_cast<float>(i) / flt_count; 323b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson float output_scale = 1. - input_scale; 324b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson for (int j = 0; j < num_channels_; ++j) { 325b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson *merge_pointer = (*merge_pointer * output_scale) + 326b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson (*input_pointer++ * input_scale); 327b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson ++merge_pointer; 328b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 329b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 330b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Copy the merged buffer back into the output, if necessary, and 331b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // append the rest of the window. 332b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->MergeBack(kOutputAnalysis, 333b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_pointer, output_merge_cnt); 334b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Write(input_pointer, remaining_count); 335b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 336b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Advance the output analysis pointer to the next target merge point, 337b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // minus half an overlap window. The target merge point is always 338b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // calculated as a delta from the previous ideal target, not the actual 339b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // target, to avoid drift. 340b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson int output_advance = target_merge_offset_; 341b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (output_merge_cnt < half_overlap_frames_) { 342b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // On the first window, back up the pointer for the next correlation. 343b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Thereafter, that compensation is preserved. 344b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_advance -= half_overlap_frames_; 345b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 346b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 347b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Don't advance beyond the available data, when finishing up. 348b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if (draining_) { 349b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_advance = 350b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson min(output_advance, output_buffer_->available(kOutputAnalysis)); 351b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 352b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Seek(kOutputAnalysis, 353b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson output_buffer_->Tell(kOutputAnalysis) + output_advance); 354b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 355b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson // Advance the input pointer beyond the frames that are no longer needed. 356b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson input_buffer_->Seek(kInputReader, input_buffer_->Tell(kInputReader) + 357b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson min(input_count, input_window_offset_)); 358b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 359b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson if ((crossfade_count + remaining_count) > 0) { 360b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson generated_data = true; 361b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } 362b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson } // while (more to process) 363b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson return generated_data; 364b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} 365b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson 366b83ad73794088498d6d38cd3b4fc9311f505d051Hugo Hudson} // namespace video_editing 367