AudioPlaybackHandler.java revision c3da8818f0598b3ab2cd6f4168349da6d0f72cb1
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16package android.speech.tts; 17 18import android.media.AudioFormat; 19import android.media.AudioTrack; 20import android.util.Log; 21 22import java.util.Iterator; 23import java.util.concurrent.PriorityBlockingQueue; 24import java.util.concurrent.atomic.AtomicLong; 25 26class AudioPlaybackHandler { 27 private static final String TAG = "TTS.AudioPlaybackHandler"; 28 private static final boolean DBG = false; 29 30 private static final int MIN_AUDIO_BUFFER_SIZE = 8192; 31 32 private static final int SYNTHESIS_START = 1; 33 private static final int SYNTHESIS_DATA_AVAILABLE = 2; 34 private static final int SYNTHESIS_DONE = 3; 35 36 private static final int PLAY_AUDIO = 5; 37 private static final int PLAY_SILENCE = 6; 38 39 private static final int SHUTDOWN = -1; 40 41 private static final int DEFAULT_PRIORITY = 1; 42 private static final int HIGH_PRIORITY = 0; 43 44 private final PriorityBlockingQueue<ListEntry> mQueue = 45 new PriorityBlockingQueue<ListEntry>(); 46 private final Thread mHandlerThread; 47 48 private volatile MessageParams mCurrentParams = null; 49 // Used only for book keeping and error detection. 50 private volatile SynthesisMessageParams mLastSynthesisRequest = null; 51 // Used to order incoming messages in our priority queue. 52 private final AtomicLong mSequenceIdCtr = new AtomicLong(0); 53 54 55 AudioPlaybackHandler() { 56 mHandlerThread = new Thread(new MessageLoop(), "TTS.AudioPlaybackThread"); 57 } 58 59 public void start() { 60 mHandlerThread.start(); 61 } 62 63 /** 64 * Stops all synthesis for a given {@code token}. If the current token 65 * is currently being processed, an effort will be made to stop it but 66 * that is not guaranteed. 67 */ 68 synchronized public void stop(MessageParams token) { 69 if (token == null) { 70 return; 71 } 72 73 removeMessages(token); 74 75 if (token.getType() == MessageParams.TYPE_SYNTHESIS) { 76 AudioTrack current = ((SynthesisMessageParams) token).getAudioTrack(); 77 if (current != null) { 78 // Stop the current audio track if it's still playing. 79 // The audio track is thread safe in this regard. 80 current.stop(); 81 } 82 mQueue.add(new ListEntry(SYNTHESIS_DONE, token, HIGH_PRIORITY)); 83 } else { 84 final MessageParams current = getCurrentParams(); 85 86 if (current != null) { 87 if (token.getType() == MessageParams.TYPE_AUDIO) { 88 ((AudioMessageParams) current).getPlayer().stop(); 89 } else if (token.getType() == MessageParams.TYPE_SILENCE) { 90 ((SilenceMessageParams) current).getConditionVariable().open(); 91 } 92 } 93 } 94 } 95 96 synchronized public void removePlaybackItems(String callingApp) { 97 removeMessages(callingApp); 98 stop(getCurrentParams()); 99 } 100 101 synchronized public void removeAllItems() { 102 removeAllMessages(); 103 stop(getCurrentParams()); 104 } 105 106 /** 107 * Shut down the audio playback thread. 108 */ 109 synchronized public void quit() { 110 stop(getCurrentParams()); 111 mQueue.add(new ListEntry(SHUTDOWN, null, HIGH_PRIORITY)); 112 } 113 114 void enqueueSynthesisStart(SynthesisMessageParams token) { 115 mQueue.add(new ListEntry(SYNTHESIS_START, token)); 116 } 117 118 void enqueueSynthesisDataAvailable(SynthesisMessageParams token) { 119 mQueue.add(new ListEntry(SYNTHESIS_DATA_AVAILABLE, token)); 120 } 121 122 void enqueueSynthesisDone(SynthesisMessageParams token) { 123 mQueue.add(new ListEntry(SYNTHESIS_DONE, token)); 124 } 125 126 void enqueueAudio(AudioMessageParams token) { 127 mQueue.add(new ListEntry(PLAY_AUDIO, token)); 128 } 129 130 void enqueueSilence(SilenceMessageParams token) { 131 mQueue.add(new ListEntry(PLAY_SILENCE, token)); 132 } 133 134 // ----------------------------------------- 135 // End of public API methods. 136 // ----------------------------------------- 137 138 // ----------------------------------------- 139 // Methods for managing the message queue. 140 // ----------------------------------------- 141 142 /* 143 * The MessageLoop is a handler like implementation that 144 * processes messages from a priority queue. 145 */ 146 private final class MessageLoop implements Runnable { 147 @Override 148 public void run() { 149 while (true) { 150 ListEntry entry = null; 151 try { 152 entry = mQueue.take(); 153 } catch (InterruptedException ie) { 154 return; 155 } 156 157 if (entry.mWhat == SHUTDOWN) { 158 if (DBG) Log.d(TAG, "MessageLoop : Shutting down"); 159 return; 160 } 161 162 if (DBG) { 163 Log.d(TAG, "MessageLoop : Handling message :" + entry.mWhat 164 + " ,seqId : " + entry.mSequenceId); 165 } 166 167 setCurrentParams(entry.mMessage); 168 handleMessage(entry); 169 setCurrentParams(null); 170 } 171 } 172 } 173 174 /* 175 * Remove all messages from the queue that contain the supplied token. 176 * Note that the Iterator is thread safe, and other methods can safely 177 * continue adding to the queue at this point. 178 */ 179 synchronized private void removeMessages(MessageParams token) { 180 if (token == null) { 181 return; 182 } 183 184 Iterator<ListEntry> it = mQueue.iterator(); 185 186 while (it.hasNext()) { 187 final ListEntry current = it.next(); 188 if (current.mMessage == token) { 189 it.remove(); 190 } 191 } 192 } 193 194 /* 195 * Atomically clear the queue of all messages. 196 */ 197 synchronized private void removeAllMessages() { 198 mQueue.clear(); 199 } 200 201 /* 202 * Remove all messages that originate from a given calling app. 203 */ 204 synchronized private void removeMessages(String callingApp) { 205 Iterator<ListEntry> it = mQueue.iterator(); 206 207 while (it.hasNext()) { 208 final ListEntry current = it.next(); 209 // The null check is to prevent us from removing control messages, 210 // such as a shutdown message. 211 if (current.mMessage != null && 212 callingApp.equals(current.mMessage.getCallingApp())) { 213 it.remove(); 214 } 215 } 216 } 217 218 /* 219 * An element of our priority queue of messages. Each message has a priority, 220 * and a sequence id (defined by the order of enqueue calls). Among messages 221 * with the same priority, messages that were received earlier win out. 222 */ 223 private final class ListEntry implements Comparable<ListEntry> { 224 final int mWhat; 225 final MessageParams mMessage; 226 final int mPriority; 227 final long mSequenceId; 228 229 private ListEntry(int what, MessageParams message) { 230 this(what, message, DEFAULT_PRIORITY); 231 } 232 233 private ListEntry(int what, MessageParams message, int priority) { 234 mWhat = what; 235 mMessage = message; 236 mPriority = priority; 237 mSequenceId = mSequenceIdCtr.incrementAndGet(); 238 } 239 240 @Override 241 public int compareTo(ListEntry that) { 242 if (that == this) { 243 return 0; 244 } 245 246 // Note that this is always 0, 1 or -1. 247 int priorityDiff = mPriority - that.mPriority; 248 if (priorityDiff == 0) { 249 // The == case cannot occur. 250 return (mSequenceId < that.mSequenceId) ? -1 : 1; 251 } 252 253 return priorityDiff; 254 } 255 } 256 257 private void setCurrentParams(MessageParams p) { 258 mCurrentParams = p; 259 } 260 261 private MessageParams getCurrentParams() { 262 return mCurrentParams; 263 } 264 265 // ----------------------------------------- 266 // Methods for dealing with individual messages, the methods 267 // below do the actual work. 268 // ----------------------------------------- 269 270 private void handleMessage(ListEntry entry) { 271 final MessageParams msg = entry.mMessage; 272 if (entry.mWhat == SYNTHESIS_START) { 273 handleSynthesisStart(msg); 274 } else if (entry.mWhat == SYNTHESIS_DATA_AVAILABLE) { 275 handleSynthesisDataAvailable(msg); 276 } else if (entry.mWhat == SYNTHESIS_DONE) { 277 handleSynthesisDone(msg); 278 } else if (entry.mWhat == PLAY_AUDIO) { 279 handleAudio(msg); 280 } else if (entry.mWhat == PLAY_SILENCE) { 281 handleSilence(msg); 282 } 283 } 284 285 // Currently implemented as blocking the audio playback thread for the 286 // specified duration. If a call to stop() is made, the thread 287 // unblocks. 288 private void handleSilence(MessageParams msg) { 289 if (DBG) Log.d(TAG, "handleSilence()"); 290 SilenceMessageParams params = (SilenceMessageParams) msg; 291 if (params.getSilenceDurationMs() > 0) { 292 params.getConditionVariable().block(params.getSilenceDurationMs()); 293 } 294 params.getDispatcher().dispatchUtteranceCompleted(); 295 if (DBG) Log.d(TAG, "handleSilence() done."); 296 } 297 298 // Plays back audio from a given URI. No TTS engine involvement here. 299 private void handleAudio(MessageParams msg) { 300 if (DBG) Log.d(TAG, "handleAudio()"); 301 AudioMessageParams params = (AudioMessageParams) msg; 302 // Note that the BlockingMediaPlayer spawns a separate thread. 303 // 304 // TODO: This can be avoided. 305 params.getPlayer().startAndWait(); 306 params.getDispatcher().dispatchUtteranceCompleted(); 307 if (DBG) Log.d(TAG, "handleAudio() done."); 308 } 309 310 // Denotes the start of a new synthesis request. We create a new 311 // audio track, and prepare it for incoming data. 312 // 313 // Note that since all TTS synthesis happens on a single thread, we 314 // should ALWAYS see the following order : 315 // 316 // handleSynthesisStart -> handleSynthesisDataAvailable(*) -> handleSynthesisDone 317 // OR 318 // handleSynthesisCompleteDataAvailable. 319 private void handleSynthesisStart(MessageParams msg) { 320 if (DBG) Log.d(TAG, "handleSynthesisStart()"); 321 final SynthesisMessageParams param = (SynthesisMessageParams) msg; 322 323 // Oops, looks like the engine forgot to call done(). We go through 324 // extra trouble to clean the data to prevent the AudioTrack resources 325 // from being leaked. 326 if (mLastSynthesisRequest != null) { 327 Log.w(TAG, "Error : Missing call to done() for request : " + 328 mLastSynthesisRequest); 329 handleSynthesisDone(mLastSynthesisRequest); 330 } 331 332 mLastSynthesisRequest = param; 333 334 // Create the audio track. 335 final AudioTrack audioTrack = createStreamingAudioTrack( 336 param.mStreamType, param.mSampleRateInHz, param.mAudioFormat, 337 param.mChannelCount, param.mVolume, param.mPan); 338 339 if (DBG) Log.d(TAG, "Created audio track [" + audioTrack.hashCode() + "]"); 340 341 param.setAudioTrack(audioTrack); 342 } 343 344 // More data available to be flushed to the audio track. 345 private void handleSynthesisDataAvailable(MessageParams msg) { 346 final SynthesisMessageParams param = (SynthesisMessageParams) msg; 347 if (param.getAudioTrack() == null) { 348 Log.w(TAG, "Error : null audio track in handleDataAvailable."); 349 return; 350 } 351 352 if (param != mLastSynthesisRequest) { 353 Log.e(TAG, "Call to dataAvailable without done() / start()"); 354 return; 355 } 356 357 final AudioTrack audioTrack = param.getAudioTrack(); 358 final SynthesisMessageParams.ListEntry bufferCopy = param.getNextBuffer(); 359 360 if (bufferCopy == null) { 361 Log.e(TAG, "No buffers available to play."); 362 return; 363 } 364 365 int playState = audioTrack.getPlayState(); 366 if (playState == AudioTrack.PLAYSTATE_STOPPED) { 367 if (DBG) Log.d(TAG, "AudioTrack stopped, restarting : " + audioTrack.hashCode()); 368 audioTrack.play(); 369 } 370 int count = 0; 371 while (count < bufferCopy.mLength) { 372 // Note that we don't take bufferCopy.mOffset into account because 373 // it is guaranteed to be 0. 374 int written = audioTrack.write(bufferCopy.mBytes, count, bufferCopy.mLength); 375 if (written <= 0) { 376 break; 377 } 378 count += written; 379 } 380 param.mBytesWritten += count; 381 param.mLogger.onPlaybackStart(); 382 } 383 384 private void handleSynthesisDone(MessageParams msg) { 385 final SynthesisMessageParams params = (SynthesisMessageParams) msg; 386 handleSynthesisDone(params); 387 // This call is delayed more than it should be, but we are 388 // certain at this point that we have all the data we want. 389 params.mLogger.onWriteData(); 390 } 391 392 // Wait for the audio track to stop playing, and then release it's resources. 393 private void handleSynthesisDone(SynthesisMessageParams params) { 394 if (DBG) Log.d(TAG, "handleSynthesisDone()"); 395 final AudioTrack audioTrack = params.getAudioTrack(); 396 397 try { 398 if (audioTrack != null) { 399 if (DBG) Log.d(TAG, "Waiting for audio track to complete : " + 400 audioTrack.hashCode()); 401 blockUntilDone(params); 402 if (DBG) Log.d(TAG, "Releasing audio track [" + audioTrack.hashCode() + "]"); 403 // The last call to AudioTrack.write( ) will return only after 404 // all data from the audioTrack has been sent to the mixer, so 405 // it's safe to release at this point. 406 audioTrack.release(); 407 } 408 } finally { 409 params.setAudioTrack(null); 410 params.getDispatcher().dispatchUtteranceCompleted(); 411 mLastSynthesisRequest = null; 412 } 413 } 414 415 private static void blockUntilDone(SynthesisMessageParams params) { 416 if (params.mAudioTrack == null || params.mBytesWritten <= 0) { 417 return; 418 } 419 420 final AudioTrack audioTrack = params.mAudioTrack; 421 final int bytesPerFrame = getBytesPerFrame(params.mAudioFormat); 422 final int lengthInBytes = params.mBytesWritten; 423 final int lengthInFrames = lengthInBytes / bytesPerFrame; 424 425 int currentPosition = 0; 426 while ((currentPosition = audioTrack.getPlaybackHeadPosition()) < lengthInFrames) { 427 if (audioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { 428 break; 429 } 430 431 long estimatedTimeMs = ((lengthInFrames - currentPosition) * 1000) / 432 audioTrack.getSampleRate(); 433 434 if (DBG) Log.d(TAG, "About to sleep for : " + estimatedTimeMs + " ms," + 435 " Playback position : " + currentPosition); 436 try { 437 Thread.sleep(estimatedTimeMs); 438 } catch (InterruptedException ie) { 439 break; 440 } 441 } 442 } 443 444 private static AudioTrack createStreamingAudioTrack(int streamType, int sampleRateInHz, 445 int audioFormat, int channelCount, float volume, float pan) { 446 int channelConfig = getChannelConfig(channelCount); 447 448 int minBufferSizeInBytes 449 = AudioTrack.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat); 450 int bufferSizeInBytes = Math.max(MIN_AUDIO_BUFFER_SIZE, minBufferSizeInBytes); 451 452 AudioTrack audioTrack = new AudioTrack(streamType, sampleRateInHz, channelConfig, 453 audioFormat, bufferSizeInBytes, AudioTrack.MODE_STREAM); 454 if (audioTrack.getState() != AudioTrack.STATE_INITIALIZED) { 455 Log.w(TAG, "Unable to create audio track."); 456 audioTrack.release(); 457 return null; 458 } 459 460 setupVolume(audioTrack, volume, pan); 461 return audioTrack; 462 } 463 464 static int getChannelConfig(int channelCount) { 465 if (channelCount == 1) { 466 return AudioFormat.CHANNEL_OUT_MONO; 467 } else if (channelCount == 2){ 468 return AudioFormat.CHANNEL_OUT_STEREO; 469 } 470 471 return 0; 472 } 473 474 static int getBytesPerFrame(int audioFormat) { 475 if (audioFormat == AudioFormat.ENCODING_PCM_8BIT) { 476 return 1; 477 } else if (audioFormat == AudioFormat.ENCODING_PCM_16BIT) { 478 return 2; 479 } 480 481 return -1; 482 } 483 484 private static void setupVolume(AudioTrack audioTrack, float volume, float pan) { 485 float vol = clip(volume, 0.0f, 1.0f); 486 float panning = clip(pan, -1.0f, 1.0f); 487 float volLeft = vol; 488 float volRight = vol; 489 if (panning > 0.0f) { 490 volLeft *= (1.0f - panning); 491 } else if (panning < 0.0f) { 492 volRight *= (1.0f + panning); 493 } 494 if (DBG) Log.d(TAG, "volLeft=" + volLeft + ",volRight=" + volRight); 495 if (audioTrack.setStereoVolume(volLeft, volRight) != AudioTrack.SUCCESS) { 496 Log.e(TAG, "Failed to set volume"); 497 } 498 } 499 500 private static float clip(float value, float min, float max) { 501 return value > max ? max : (value < min ? min : value); 502 } 503 504} 505