1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media.audiofx; 18 19import android.util.Log; 20import java.lang.ref.WeakReference; 21import java.io.IOException; 22import android.os.Handler; 23import android.os.Looper; 24import android.os.Message; 25 26/** 27 * The Visualizer class enables application to retrieve part of the currently playing audio for 28 * visualization purpose. It is not an audio recording interface and only returns partial and low 29 * quality audio content. However, to protect privacy of certain audio data (e.g voice mail) the use 30 * of the visualizer requires the permission android.permission.RECORD_AUDIO. 31 * <p>The audio session ID passed to the constructor indicates which audio content should be 32 * visualized:<br> 33 * <ul> 34 * <li>If the session is 0, the audio output mix is visualized</li> 35 * <li>If the session is not 0, the audio from a particular {@link android.media.MediaPlayer} or 36 * {@link android.media.AudioTrack} 37 * using this audio session is visualized </li> 38 * </ul> 39 * <p>Two types of representation of audio content can be captured: <br> 40 * <ul> 41 * <li>Waveform data: consecutive 8-bit (unsigned) mono samples by using the 42 * {@link #getWaveForm(byte[])} method</li> 43 * <li>Frequency data: 8-bit magnitude FFT by using the {@link #getFft(byte[])} method</li> 44 * </ul> 45 * <p>The length of the capture can be retrieved or specified by calling respectively 46 * {@link #getCaptureSize()} and {@link #setCaptureSize(int)} methods. The capture size must be a 47 * power of 2 in the range returned by {@link #getCaptureSizeRange()}. 48 * <p>In addition to the polling capture mode described above with {@link #getWaveForm(byte[])} and 49 * {@link #getFft(byte[])} methods, a callback mode is also available by installing a listener by 50 * use of the {@link #setDataCaptureListener(OnDataCaptureListener, int, boolean, boolean)} method. 51 * The rate at which the listener capture method is called as well as the type of data returned is 52 * specified. 53 * <p>Before capturing data, the Visualizer must be enabled by calling the 54 * {@link #setEnabled(boolean)} method. 55 * When data capture is not needed any more, the Visualizer should be disabled. 56 * <p>It is good practice to call the {@link #release()} method when the Visualizer is not used 57 * anymore to free up native resources associated to the Visualizer instance. 58 * <p>Creating a Visualizer on the output mix (audio session 0) requires permission 59 * {@link android.Manifest.permission#MODIFY_AUDIO_SETTINGS} 60 */ 61 62public class Visualizer { 63 64 static { 65 System.loadLibrary("audioeffect_jni"); 66 native_init(); 67 } 68 69 private final static String TAG = "Visualizer-JAVA"; 70 71 /** 72 * State of a Visualizer object that was not successfully initialized upon creation 73 */ 74 public static final int STATE_UNINITIALIZED = 0; 75 /** 76 * State of a Visualizer object that is ready to be used. 77 */ 78 public static final int STATE_INITIALIZED = 1; 79 /** 80 * State of a Visualizer object that is active. 81 */ 82 public static final int STATE_ENABLED = 2; 83 84 // to keep in sync with system/media/audio_effects/include/audio_effects/effect_visualizer.h 85 /** 86 * Defines a capture mode where amplification is applied based on the content of the captured 87 * data. This is the default Visualizer mode, and is suitable for music visualization. 88 */ 89 public static final int SCALING_MODE_NORMALIZED = 0; 90 /** 91 * Defines a capture mode where the playback volume will affect (scale) the range of the 92 * captured data. A low playback volume will lead to low sample and fft values, and vice-versa. 93 */ 94 public static final int SCALING_MODE_AS_PLAYED = 1; 95 96 // to keep in sync with frameworks/base/media/jni/audioeffect/android_media_Visualizer.cpp 97 private static final int NATIVE_EVENT_PCM_CAPTURE = 0; 98 private static final int NATIVE_EVENT_FFT_CAPTURE = 1; 99 private static final int NATIVE_EVENT_SERVER_DIED = 2; 100 101 // Error codes: 102 /** 103 * Successful operation. 104 */ 105 public static final int SUCCESS = 0; 106 /** 107 * Unspecified error. 108 */ 109 public static final int ERROR = -1; 110 /** 111 * Internal operation status. Not returned by any method. 112 */ 113 public static final int ALREADY_EXISTS = -2; 114 /** 115 * Operation failed due to bad object initialization. 116 */ 117 public static final int ERROR_NO_INIT = -3; 118 /** 119 * Operation failed due to bad parameter value. 120 */ 121 public static final int ERROR_BAD_VALUE = -4; 122 /** 123 * Operation failed because it was requested in wrong state. 124 */ 125 public static final int ERROR_INVALID_OPERATION = -5; 126 /** 127 * Operation failed due to lack of memory. 128 */ 129 public static final int ERROR_NO_MEMORY = -6; 130 /** 131 * Operation failed due to dead remote object. 132 */ 133 public static final int ERROR_DEAD_OBJECT = -7; 134 135 //-------------------------------------------------------------------------- 136 // Member variables 137 //-------------------- 138 /** 139 * Indicates the state of the Visualizer instance 140 */ 141 private int mState = STATE_UNINITIALIZED; 142 /** 143 * Lock to synchronize access to mState 144 */ 145 private final Object mStateLock = new Object(); 146 /** 147 * System wide unique Identifier of the visualizer engine used by this Visualizer instance 148 */ 149 private int mId; 150 151 /** 152 * Lock to protect listeners updates against event notifications 153 */ 154 private final Object mListenerLock = new Object(); 155 /** 156 * Handler for events coming from the native code 157 */ 158 private NativeEventHandler mNativeEventHandler = null; 159 /** 160 * PCM and FFT capture listener registered by client 161 */ 162 private OnDataCaptureListener mCaptureListener = null; 163 /** 164 * Server Died listener registered by client 165 */ 166 private OnServerDiedListener mServerDiedListener = null; 167 168 // accessed by native methods 169 private int mNativeVisualizer; 170 private int mJniData; 171 172 //-------------------------------------------------------------------------- 173 // Constructor, Finalize 174 //-------------------- 175 /** 176 * Class constructor. 177 * @param audioSession system wide unique audio session identifier. If audioSession 178 * is not 0, the visualizer will be attached to the MediaPlayer or AudioTrack in the 179 * same audio session. Otherwise, the Visualizer will apply to the output mix. 180 * 181 * @throws java.lang.UnsupportedOperationException 182 * @throws java.lang.RuntimeException 183 */ 184 185 public Visualizer(int audioSession) 186 throws UnsupportedOperationException, RuntimeException { 187 int[] id = new int[1]; 188 189 synchronized (mStateLock) { 190 mState = STATE_UNINITIALIZED; 191 // native initialization 192 int result = native_setup(new WeakReference<Visualizer>(this), audioSession, id); 193 if (result != SUCCESS && result != ALREADY_EXISTS) { 194 Log.e(TAG, "Error code "+result+" when initializing Visualizer."); 195 switch (result) { 196 case ERROR_INVALID_OPERATION: 197 throw (new UnsupportedOperationException("Effect library not loaded")); 198 default: 199 throw (new RuntimeException("Cannot initialize Visualizer engine, error: " 200 +result)); 201 } 202 } 203 mId = id[0]; 204 if (native_getEnabled()) { 205 mState = STATE_ENABLED; 206 } else { 207 mState = STATE_INITIALIZED; 208 } 209 } 210 } 211 212 /** 213 * Releases the native Visualizer resources. It is a good practice to release the 214 * visualization engine when not in use. 215 */ 216 public void release() { 217 synchronized (mStateLock) { 218 native_release(); 219 mState = STATE_UNINITIALIZED; 220 } 221 } 222 223 @Override 224 protected void finalize() { 225 native_finalize(); 226 } 227 228 /** 229 * Enable or disable the visualization engine. 230 * @param enabled requested enable state 231 * @return {@link #SUCCESS} in case of success, 232 * {@link #ERROR_INVALID_OPERATION} or {@link #ERROR_DEAD_OBJECT} in case of failure. 233 * @throws IllegalStateException 234 */ 235 public int setEnabled(boolean enabled) 236 throws IllegalStateException { 237 synchronized (mStateLock) { 238 if (mState == STATE_UNINITIALIZED) { 239 throw(new IllegalStateException("setEnabled() called in wrong state: "+mState)); 240 } 241 int status = SUCCESS; 242 if ((enabled && (mState == STATE_INITIALIZED)) || 243 (!enabled && (mState == STATE_ENABLED))) { 244 status = native_setEnabled(enabled); 245 if (status == SUCCESS) { 246 mState = enabled ? STATE_ENABLED : STATE_INITIALIZED; 247 } 248 } 249 return status; 250 } 251 } 252 253 /** 254 * Get current activation state of the visualizer. 255 * @return true if the visualizer is active, false otherwise 256 */ 257 public boolean getEnabled() 258 { 259 synchronized (mStateLock) { 260 if (mState == STATE_UNINITIALIZED) { 261 throw(new IllegalStateException("getEnabled() called in wrong state: "+mState)); 262 } 263 return native_getEnabled(); 264 } 265 } 266 267 /** 268 * Returns the capture size range. 269 * @return the mininum capture size is returned in first array element and the maximum in second 270 * array element. 271 */ 272 public static native int[] getCaptureSizeRange(); 273 274 /** 275 * Returns the maximum capture rate for the callback capture method. This is the maximum value 276 * for the rate parameter of the 277 * {@link #setDataCaptureListener(OnDataCaptureListener, int, boolean, boolean)} method. 278 * @return the maximum capture rate expressed in milliHertz 279 */ 280 public static native int getMaxCaptureRate(); 281 282 /** 283 * Sets the capture size, i.e. the number of bytes returned by {@link #getWaveForm(byte[])} and 284 * {@link #getFft(byte[])} methods. The capture size must be a power of 2 in the range returned 285 * by {@link #getCaptureSizeRange()}. 286 * This method must not be called when the Visualizer is enabled. 287 * @param size requested capture size 288 * @return {@link #SUCCESS} in case of success, 289 * {@link #ERROR_BAD_VALUE} in case of failure. 290 * @throws IllegalStateException 291 */ 292 public int setCaptureSize(int size) 293 throws IllegalStateException { 294 synchronized (mStateLock) { 295 if (mState != STATE_INITIALIZED) { 296 throw(new IllegalStateException("setCaptureSize() called in wrong state: "+mState)); 297 } 298 return native_setCaptureSize(size); 299 } 300 } 301 302 /** 303 * Returns current capture size. 304 * @return the capture size in bytes. 305 */ 306 public int getCaptureSize() 307 throws IllegalStateException { 308 synchronized (mStateLock) { 309 if (mState == STATE_UNINITIALIZED) { 310 throw(new IllegalStateException("getCaptureSize() called in wrong state: "+mState)); 311 } 312 return native_getCaptureSize(); 313 } 314 } 315 316 /** 317 * Set the type of scaling applied on the captured visualization data. 318 * @param mode see {@link #SCALING_MODE_NORMALIZED} 319 * and {@link #SCALING_MODE_AS_PLAYED} 320 * @return {@link #SUCCESS} in case of success, 321 * {@link #ERROR_BAD_VALUE} in case of failure. 322 * @throws IllegalStateException 323 */ 324 public int setScalingMode(int mode) 325 throws IllegalStateException { 326 synchronized (mStateLock) { 327 if (mState == STATE_UNINITIALIZED) { 328 throw(new IllegalStateException("setScalingMode() called in wrong state: " 329 + mState)); 330 } 331 return native_setScalingMode(mode); 332 } 333 } 334 335 /** 336 * Returns the current scaling mode on the captured visualization data. 337 * @return the scaling mode, see {@link #SCALING_MODE_NORMALIZED} 338 * and {@link #SCALING_MODE_AS_PLAYED}. 339 * @throws IllegalStateException 340 */ 341 public int getScalingMode() 342 throws IllegalStateException { 343 synchronized (mStateLock) { 344 if (mState == STATE_UNINITIALIZED) { 345 throw(new IllegalStateException("getScalingMode() called in wrong state: " 346 + mState)); 347 } 348 return native_getScalingMode(); 349 } 350 } 351 352 /** 353 * Returns the sampling rate of the captured audio. 354 * @return the sampling rate in milliHertz. 355 */ 356 public int getSamplingRate() 357 throws IllegalStateException { 358 synchronized (mStateLock) { 359 if (mState == STATE_UNINITIALIZED) { 360 throw(new IllegalStateException("getSamplingRate() called in wrong state: "+mState)); 361 } 362 return native_getSamplingRate(); 363 } 364 } 365 366 /** 367 * Returns a waveform capture of currently playing audio content. The capture consists in 368 * a number of consecutive 8-bit (unsigned) mono PCM samples equal to the capture size returned 369 * by {@link #getCaptureSize()}. 370 * <p>This method must be called when the Visualizer is enabled. 371 * @param waveform array of bytes where the waveform should be returned 372 * @return {@link #SUCCESS} in case of success, 373 * {@link #ERROR_NO_MEMORY}, {@link #ERROR_INVALID_OPERATION} or {@link #ERROR_DEAD_OBJECT} 374 * in case of failure. 375 * @throws IllegalStateException 376 */ 377 public int getWaveForm(byte[] waveform) 378 throws IllegalStateException { 379 synchronized (mStateLock) { 380 if (mState != STATE_ENABLED) { 381 throw(new IllegalStateException("getWaveForm() called in wrong state: "+mState)); 382 } 383 return native_getWaveForm(waveform); 384 } 385 } 386 /** 387 * Returns a frequency capture of currently playing audio content. 388 * <p>This method must be called when the Visualizer is enabled. 389 * <p>The capture is an 8-bit magnitude FFT, the frequency range covered being 0 (DC) to half of 390 * the sampling rate returned by {@link #getSamplingRate()}. The capture returns the real and 391 * imaginary parts of a number of frequency points equal to half of the capture size plus one. 392 * <p>Note: only the real part is returned for the first point (DC) and the last point 393 * (sampling frequency / 2). 394 * <p>The layout in the returned byte array is as follows: 395 * <ul> 396 * <li> n is the capture size returned by getCaptureSize()</li> 397 * <li> Rfk, Ifk are respectively the real and imaginary parts of the kth frequency 398 * component</li> 399 * <li> If Fs is the sampling frequency retuned by getSamplingRate() the kth frequency is: 400 * (k*Fs)/(n/2) </li> 401 * </ul> 402 * <table border="0" cellspacing="0" cellpadding="0"> 403 * <tr><td>Index </p></td> 404 * <td>0 </p></td> 405 * <td>1 </p></td> 406 * <td>2 </p></td> 407 * <td>3 </p></td> 408 * <td>4 </p></td> 409 * <td>5 </p></td> 410 * <td>... </p></td> 411 * <td>n - 2 </p></td> 412 * <td>n - 1 </p></td></tr> 413 * <tr><td>Data </p></td> 414 * <td>Rf0 </p></td> 415 * <td>Rf(n/2) </p></td> 416 * <td>Rf1 </p></td> 417 * <td>If1 </p></td> 418 * <td>Rf2 </p></td> 419 * <td>If2 </p></td> 420 * <td>... </p></td> 421 * <td>Rf(n-1)/2 </p></td> 422 * <td>If(n-1)/2 </p></td></tr> 423 * </table> 424 * @param fft array of bytes where the FFT should be returned 425 * @return {@link #SUCCESS} in case of success, 426 * {@link #ERROR_NO_MEMORY}, {@link #ERROR_INVALID_OPERATION} or {@link #ERROR_DEAD_OBJECT} 427 * in case of failure. 428 * @throws IllegalStateException 429 */ 430 public int getFft(byte[] fft) 431 throws IllegalStateException { 432 synchronized (mStateLock) { 433 if (mState != STATE_ENABLED) { 434 throw(new IllegalStateException("getFft() called in wrong state: "+mState)); 435 } 436 return native_getFft(fft); 437 } 438 } 439 440 //--------------------------------------------------------- 441 // Interface definitions 442 //-------------------- 443 /** 444 * The OnDataCaptureListener interface defines methods called by the Visualizer to periodically 445 * update the audio visualization capture. 446 * The client application can implement this interface and register the listener with the 447 * {@link #setDataCaptureListener(OnDataCaptureListener, int, boolean, boolean)} method. 448 */ 449 public interface OnDataCaptureListener { 450 /** 451 * Method called when a new waveform capture is available. 452 * <p>Data in the waveform buffer is valid only within the scope of the callback. 453 * Applications which needs access to the waveform data after returning from the callback 454 * should make a copy of the data instead of holding a reference. 455 * @param visualizer Visualizer object on which the listener is registered. 456 * @param waveform array of bytes containing the waveform representation. 457 * @param samplingRate sampling rate of the audio visualized. 458 */ 459 void onWaveFormDataCapture(Visualizer visualizer, byte[] waveform, int samplingRate); 460 461 /** 462 * Method called when a new frequency capture is available. 463 * <p>Data in the fft buffer is valid only within the scope of the callback. 464 * Applications which needs access to the fft data after returning from the callback 465 * should make a copy of the data instead of holding a reference. 466 * @param visualizer Visualizer object on which the listener is registered. 467 * @param fft array of bytes containing the frequency representation. 468 * @param samplingRate sampling rate of the audio visualized. 469 */ 470 void onFftDataCapture(Visualizer visualizer, byte[] fft, int samplingRate); 471 } 472 473 /** 474 * Registers an OnDataCaptureListener interface and specifies the rate at which the capture 475 * should be updated as well as the type of capture requested. 476 * <p>Call this method with a null listener to stop receiving the capture updates. 477 * @param listener OnDataCaptureListener registered 478 * @param rate rate in milliHertz at which the capture should be updated 479 * @param waveform true if a waveform capture is requested: the onWaveFormDataCapture() 480 * method will be called on the OnDataCaptureListener interface. 481 * @param fft true if a frequency capture is requested: the onFftDataCapture() method will be 482 * called on the OnDataCaptureListener interface. 483 * @return {@link #SUCCESS} in case of success, 484 * {@link #ERROR_NO_INIT} or {@link #ERROR_BAD_VALUE} in case of failure. 485 */ 486 public int setDataCaptureListener(OnDataCaptureListener listener, 487 int rate, boolean waveform, boolean fft) { 488 synchronized (mListenerLock) { 489 mCaptureListener = listener; 490 } 491 if (listener == null) { 492 // make sure capture callback is stopped in native code 493 waveform = false; 494 fft = false; 495 } 496 int status = native_setPeriodicCapture(rate, waveform, fft); 497 if (status == SUCCESS) { 498 if ((listener != null) && (mNativeEventHandler == null)) { 499 Looper looper; 500 if ((looper = Looper.myLooper()) != null) { 501 mNativeEventHandler = new NativeEventHandler(this, looper); 502 } else if ((looper = Looper.getMainLooper()) != null) { 503 mNativeEventHandler = new NativeEventHandler(this, looper); 504 } else { 505 mNativeEventHandler = null; 506 status = ERROR_NO_INIT; 507 } 508 } 509 } 510 return status; 511 } 512 513 /** 514 * @hide 515 * 516 * The OnServerDiedListener interface defines a method called by the Visualizer to indicate that 517 * the connection to the native media server has been broken and that the Visualizer object will 518 * need to be released and re-created. 519 * The client application can implement this interface and register the listener with the 520 * {@link #setServerDiedListener(OnServerDiedListener)} method. 521 */ 522 public interface OnServerDiedListener { 523 /** 524 * @hide 525 * 526 * Method called when the native media server has died. 527 * <p>If the native media server encounters a fatal error and needs to restart, the binder 528 * connection from the {@link #Visualizer} to the media server will be broken. Data capture 529 * callbacks will stop happening, and client initiated calls to the {@link #Visualizer} 530 * instance will fail with the error code {@link #DEAD_OBJECT}. To restore functionality, 531 * clients should {@link #release()} their old visualizer and create a new instance. 532 */ 533 void onServerDied(); 534 } 535 536 /** 537 * @hide 538 * 539 * Registers an OnServerDiedListener interface. 540 * <p>Call this method with a null listener to stop receiving server death notifications. 541 * @return {@link #SUCCESS} in case of success, 542 */ 543 public int setServerDiedListener(OnServerDiedListener listener) { 544 synchronized (mListenerLock) { 545 mServerDiedListener = listener; 546 } 547 return SUCCESS; 548 } 549 550 /** 551 * Helper class to handle the forwarding of native events to the appropriate listeners 552 */ 553 private class NativeEventHandler extends Handler 554 { 555 private Visualizer mVisualizer; 556 557 public NativeEventHandler(Visualizer v, Looper looper) { 558 super(looper); 559 mVisualizer = v; 560 } 561 562 private void handleCaptureMessage(Message msg) { 563 OnDataCaptureListener l = null; 564 synchronized (mListenerLock) { 565 l = mVisualizer.mCaptureListener; 566 } 567 568 if (l != null) { 569 byte[] data = (byte[])msg.obj; 570 int samplingRate = msg.arg1; 571 572 switch(msg.what) { 573 case NATIVE_EVENT_PCM_CAPTURE: 574 l.onWaveFormDataCapture(mVisualizer, data, samplingRate); 575 break; 576 case NATIVE_EVENT_FFT_CAPTURE: 577 l.onFftDataCapture(mVisualizer, data, samplingRate); 578 break; 579 default: 580 Log.e(TAG,"Unknown native event in handleCaptureMessge: "+msg.what); 581 break; 582 } 583 } 584 } 585 586 private void handleServerDiedMessage(Message msg) { 587 OnServerDiedListener l = null; 588 synchronized (mListenerLock) { 589 l = mVisualizer.mServerDiedListener; 590 } 591 592 if (l != null) 593 l.onServerDied(); 594 } 595 596 @Override 597 public void handleMessage(Message msg) { 598 if (mVisualizer == null) { 599 return; 600 } 601 602 switch(msg.what) { 603 case NATIVE_EVENT_PCM_CAPTURE: 604 case NATIVE_EVENT_FFT_CAPTURE: 605 handleCaptureMessage(msg); 606 break; 607 case NATIVE_EVENT_SERVER_DIED: 608 handleServerDiedMessage(msg); 609 break; 610 default: 611 Log.e(TAG,"Unknown native event: "+msg.what); 612 break; 613 } 614 } 615 } 616 617 //--------------------------------------------------------- 618 // Interface definitions 619 //-------------------- 620 621 private static native final void native_init(); 622 623 private native final int native_setup(Object audioeffect_this, 624 int audioSession, 625 int[] id); 626 627 private native final void native_finalize(); 628 629 private native final void native_release(); 630 631 private native final int native_setEnabled(boolean enabled); 632 633 private native final boolean native_getEnabled(); 634 635 private native final int native_setCaptureSize(int size); 636 637 private native final int native_getCaptureSize(); 638 639 private native final int native_setScalingMode(int mode); 640 641 private native final int native_getScalingMode(); 642 643 private native final int native_getSamplingRate(); 644 645 private native final int native_getWaveForm(byte[] waveform); 646 647 private native final int native_getFft(byte[] fft); 648 649 private native final int native_setPeriodicCapture(int rate, boolean waveForm, boolean fft); 650 651 //--------------------------------------------------------- 652 // Java methods called from the native side 653 //-------------------- 654 @SuppressWarnings("unused") 655 private static void postEventFromNative(Object effect_ref, 656 int what, int arg1, int arg2, Object obj) { 657 Visualizer visu = (Visualizer)((WeakReference)effect_ref).get(); 658 if (visu == null) { 659 return; 660 } 661 662 if (visu.mNativeEventHandler != null) { 663 Message m = visu.mNativeEventHandler.obtainMessage(what, arg1, arg2, obj); 664 visu.mNativeEventHandler.sendMessage(m); 665 } 666 667 } 668 669} 670 671