1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18package android.filterpacks.videosrc; 19 20import android.content.Context; 21import android.content.res.AssetFileDescriptor; 22import android.filterfw.core.Filter; 23import android.filterfw.core.FilterContext; 24import android.filterfw.core.Frame; 25import android.filterfw.core.FrameFormat; 26import android.filterfw.core.FrameManager; 27import android.filterfw.core.GenerateFieldPort; 28import android.filterfw.core.GenerateFinalPort; 29import android.filterfw.core.GLFrame; 30import android.filterfw.core.KeyValueMap; 31import android.filterfw.core.MutableFrameFormat; 32import android.filterfw.core.NativeFrame; 33import android.filterfw.core.Program; 34import android.filterfw.core.ShaderProgram; 35import android.filterfw.format.ImageFormat; 36import android.graphics.SurfaceTexture; 37import android.media.MediaPlayer; 38import android.os.ConditionVariable; 39import android.opengl.Matrix; 40import android.view.Surface; 41 42import java.io.IOException; 43import java.io.FileDescriptor; 44import java.lang.IllegalArgumentException; 45import java.util.List; 46import java.util.Set; 47 48import android.util.Log; 49 50/** 51 * @hide 52 */ 53public class MediaSource extends Filter { 54 55 /** User-visible parameters */ 56 57 /** The source URL for the media source. Can be an http: link to a remote 58 * resource, or a file: link to a local media file 59 */ 60 @GenerateFieldPort(name = "sourceUrl", hasDefault = true) 61 private String mSourceUrl = ""; 62 63 /** An open asset file descriptor to a local media source. Default is null */ 64 @GenerateFieldPort(name = "sourceAsset", hasDefault = true) 65 private AssetFileDescriptor mSourceAsset = null; 66 67 /** Whether the media source is a URL or an asset file descriptor. Defaults 68 * to false. 69 */ 70 @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true) 71 private boolean mSelectedIsUrl = false; 72 73 /** Whether the filter will always wait for a new video frame, or whether it 74 * will output an old frame again if a new frame isn't available. Defaults 75 * to true. 76 */ 77 @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true) 78 private boolean mWaitForNewFrame = true; 79 80 /** Whether the media source should loop automatically or not. Defaults to 81 * true. 82 */ 83 @GenerateFieldPort(name = "loop", hasDefault = true) 84 private boolean mLooping = true; 85 86 /** Volume control. Currently sound is piped directly to the speakers, so 87 * this defaults to mute. 88 */ 89 @GenerateFieldPort(name = "volume", hasDefault = true) 90 private float mVolume = 0.f; 91 92 /** Orientation. This controls the output orientation of the video. Valid 93 * values are 0, 90, 180, 270 94 */ 95 @GenerateFieldPort(name = "orientation", hasDefault = true) 96 private int mOrientation = 0; 97 98 private MediaPlayer mMediaPlayer; 99 private GLFrame mMediaFrame; 100 private SurfaceTexture mSurfaceTexture; 101 private ShaderProgram mFrameExtractor; 102 private MutableFrameFormat mOutputFormat; 103 private int mWidth, mHeight; 104 105 // Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT 106 private static final int PREP_TIMEOUT = 100; // ms 107 private static final int PREP_TIMEOUT_REPEAT = 100; 108 private static final int NEWFRAME_TIMEOUT = 100; //ms 109 private static final int NEWFRAME_TIMEOUT_REPEAT = 10; 110 111 // This is an identity shader; not using the default identity 112 // shader because reading from a SurfaceTexture requires the 113 // GL_OES_EGL_image_external extension. 114 private final String mFrameShader = 115 "#extension GL_OES_EGL_image_external : require\n" + 116 "precision mediump float;\n" + 117 "uniform samplerExternalOES tex_sampler_0;\n" + 118 "varying vec2 v_texcoord;\n" + 119 "void main() {\n" + 120 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 121 "}\n"; 122 123 // The following transforms enable rotation of the decoded source. 124 // These are multiplied with the transform obtained from the 125 // SurfaceTexture to get the final transform to be set on the media source. 126 // Currently, given a device orientation, the MediaSource rotates in such a way 127 // that the source is displayed upright. A particular use case 128 // is "Background Replacement" feature in the Camera app 129 // where the MediaSource rotates the source to align with the camera feed and pass it 130 // on to the backdropper filter. The backdropper only does the blending 131 // and does not have to do any rotation 132 // (except for mirroring in case of front camera). 133 // TODO: Currently the rotations are spread over a bunch of stages in the 134 // pipeline. A cleaner design 135 // could be to cast away all the rotation in a separate filter or attach a transform 136 // to the frame so that MediaSource itself need not know about any rotation. 137 private static final float[] mSourceCoords_0 = { 1, 1, 0, 1, 138 0, 1, 0, 1, 139 1, 0, 0, 1, 140 0, 0, 0, 1 }; 141 private static final float[] mSourceCoords_270 = { 0, 1, 0, 1, 142 0, 0, 0, 1, 143 1, 1, 0, 1, 144 1, 0, 0, 1 }; 145 private static final float[] mSourceCoords_180 = { 0, 0, 0, 1, 146 1, 0, 0, 1, 147 0, 1, 0, 1, 148 1, 1, 0, 1 }; 149 private static final float[] mSourceCoords_90 = { 1, 0, 0, 1, 150 1, 1, 0, 1, 151 0, 0, 0, 1, 152 0, 1, 0, 1 }; 153 154 private boolean mGotSize; 155 private boolean mPrepared; 156 private boolean mPlaying; 157 private boolean mNewFrameAvailable; 158 private boolean mOrientationUpdated; 159 private boolean mPaused; 160 private boolean mCompleted; 161 162 private final boolean mLogVerbose; 163 private static final String TAG = "MediaSource"; 164 165 public MediaSource(String name) { 166 super(name); 167 mNewFrameAvailable = false; 168 169 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 170 } 171 172 @Override 173 public void setupPorts() { 174 // Add input port 175 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 176 FrameFormat.TARGET_GPU)); 177 } 178 179 private void createFormats() { 180 mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 181 FrameFormat.TARGET_GPU); 182 } 183 184 @Override 185 protected void prepare(FilterContext context) { 186 if (mLogVerbose) Log.v(TAG, "Preparing MediaSource"); 187 188 mFrameExtractor = new ShaderProgram(context, mFrameShader); 189 // SurfaceTexture defines (0,0) to be bottom-left. The filter framework 190 // defines (0,0) as top-left, so do the flip here. 191 mFrameExtractor.setSourceRect(0, 1, 1, -1); 192 193 createFormats(); 194 } 195 196 @Override 197 public void open(FilterContext context) { 198 if (mLogVerbose) { 199 Log.v(TAG, "Opening MediaSource"); 200 if (mSelectedIsUrl) { 201 Log.v(TAG, "Current URL is " + mSourceUrl); 202 } else { 203 Log.v(TAG, "Current source is Asset!"); 204 } 205 } 206 207 mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame( 208 mOutputFormat, 209 GLFrame.EXTERNAL_TEXTURE, 210 0); 211 212 mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); 213 214 if (!setupMediaPlayer(mSelectedIsUrl)) { 215 throw new RuntimeException("Error setting up MediaPlayer!"); 216 } 217 } 218 219 @Override 220 public void process(FilterContext context) { 221 // Note: process is synchronized by its caller in the Filter base class 222 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 223 224 if (mMediaPlayer == null) { 225 // Something went wrong in initialization or parameter updates 226 throw new NullPointerException("Unexpected null media player!"); 227 } 228 229 if (mCompleted) { 230 // Video playback is done, so close us down 231 closeOutputPort("video"); 232 return; 233 } 234 235 if (!mPlaying) { 236 int waitCount = 0; 237 if (mLogVerbose) Log.v(TAG, "Waiting for preparation to complete"); 238 while (!mGotSize || !mPrepared) { 239 try { 240 this.wait(PREP_TIMEOUT); 241 } catch (InterruptedException e) { 242 // ignoring 243 } 244 if (mCompleted) { 245 // Video playback is done, so close us down 246 closeOutputPort("video"); 247 return; 248 } 249 waitCount++; 250 if (waitCount == PREP_TIMEOUT_REPEAT) { 251 mMediaPlayer.release(); 252 throw new RuntimeException("MediaPlayer timed out while preparing!"); 253 } 254 } 255 if (mLogVerbose) Log.v(TAG, "Starting playback"); 256 mMediaPlayer.start(); 257 } 258 259 // Use last frame if paused, unless just starting playback, in which case 260 // we want at least one valid frame before pausing 261 if (!mPaused || !mPlaying) { 262 if (mWaitForNewFrame) { 263 if (mLogVerbose) Log.v(TAG, "Waiting for new frame"); 264 265 int waitCount = 0; 266 while (!mNewFrameAvailable) { 267 if (waitCount == NEWFRAME_TIMEOUT_REPEAT) { 268 if (mCompleted) { 269 // Video playback is done, so close us down 270 closeOutputPort("video"); 271 return; 272 } else { 273 throw new RuntimeException("Timeout waiting for new frame!"); 274 } 275 } 276 try { 277 this.wait(NEWFRAME_TIMEOUT); 278 } catch (InterruptedException e) { 279 if (mLogVerbose) Log.v(TAG, "interrupted"); 280 // ignoring 281 } 282 waitCount++; 283 } 284 mNewFrameAvailable = false; 285 if (mLogVerbose) Log.v(TAG, "Got new frame"); 286 } 287 288 mSurfaceTexture.updateTexImage(); 289 mOrientationUpdated = true; 290 } 291 if (mOrientationUpdated) { 292 float[] surfaceTransform = new float[16]; 293 mSurfaceTexture.getTransformMatrix(surfaceTransform); 294 295 float[] sourceCoords = new float[16]; 296 switch (mOrientation) { 297 default: 298 case 0: 299 Matrix.multiplyMM(sourceCoords, 0, 300 surfaceTransform, 0, 301 mSourceCoords_0, 0); 302 break; 303 case 90: 304 Matrix.multiplyMM(sourceCoords, 0, 305 surfaceTransform, 0, 306 mSourceCoords_90, 0); 307 break; 308 case 180: 309 Matrix.multiplyMM(sourceCoords, 0, 310 surfaceTransform, 0, 311 mSourceCoords_180, 0); 312 break; 313 case 270: 314 Matrix.multiplyMM(sourceCoords, 0, 315 surfaceTransform, 0, 316 mSourceCoords_270, 0); 317 break; 318 } 319 if (mLogVerbose) { 320 Log.v(TAG, "OrientationHint = " + mOrientation); 321 String temp = String.format("SetSourceRegion: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f", 322 sourceCoords[4], sourceCoords[5],sourceCoords[0], sourceCoords[1], 323 sourceCoords[12], sourceCoords[13],sourceCoords[8], sourceCoords[9]); 324 Log.v(TAG, temp); 325 } 326 mFrameExtractor.setSourceRegion(sourceCoords[4], sourceCoords[5], 327 sourceCoords[0], sourceCoords[1], 328 sourceCoords[12], sourceCoords[13], 329 sourceCoords[8], sourceCoords[9]); 330 mOrientationUpdated = false; 331 } 332 333 Frame output = context.getFrameManager().newFrame(mOutputFormat); 334 mFrameExtractor.process(mMediaFrame, output); 335 336 long timestamp = mSurfaceTexture.getTimestamp(); 337 if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s"); 338 output.setTimestamp(timestamp); 339 340 pushOutput("video", output); 341 output.release(); 342 343 mPlaying = true; 344 } 345 346 @Override 347 public void close(FilterContext context) { 348 if (mMediaPlayer.isPlaying()) { 349 mMediaPlayer.stop(); 350 } 351 mPrepared = false; 352 mGotSize = false; 353 mPlaying = false; 354 mPaused = false; 355 mCompleted = false; 356 mNewFrameAvailable = false; 357 358 mMediaPlayer.release(); 359 mMediaPlayer = null; 360 mSurfaceTexture.release(); 361 mSurfaceTexture = null; 362 if (mLogVerbose) Log.v(TAG, "MediaSource closed"); 363 } 364 365 @Override 366 public void tearDown(FilterContext context) { 367 if (mMediaFrame != null) { 368 mMediaFrame.release(); 369 } 370 } 371 372 // When updating the port values of the filter, users can update sourceIsUrl to switch 373 // between using URL objects or Assets. 374 // If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player 375 // uses Url objects/Asset. 376 // Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch 377 // sourceIsUrl next time. 378 @Override 379 public void fieldPortValueUpdated(String name, FilterContext context) { 380 if (mLogVerbose) Log.v(TAG, "Parameter update"); 381 if (name.equals("sourceUrl")) { 382 if (isOpen()) { 383 if (mLogVerbose) Log.v(TAG, "Opening new source URL"); 384 if (mSelectedIsUrl) { 385 setupMediaPlayer(mSelectedIsUrl); 386 } 387 } 388 } else if (name.equals("sourceAsset") ) { 389 if (isOpen()) { 390 if (mLogVerbose) Log.v(TAG, "Opening new source FD"); 391 if (!mSelectedIsUrl) { 392 setupMediaPlayer(mSelectedIsUrl); 393 } 394 } 395 } else if (name.equals("loop")) { 396 if (isOpen()) { 397 mMediaPlayer.setLooping(mLooping); 398 } 399 } else if (name.equals("sourceIsUrl")) { 400 if (isOpen()){ 401 if (mSelectedIsUrl){ 402 if (mLogVerbose) Log.v(TAG, "Opening new source URL"); 403 } else { 404 if (mLogVerbose) Log.v(TAG, "Opening new source Asset"); 405 } 406 setupMediaPlayer(mSelectedIsUrl); 407 } 408 } else if (name.equals("volume")) { 409 if (isOpen()) { 410 mMediaPlayer.setVolume(mVolume, mVolume); 411 } 412 } else if (name.equals("orientation") && mGotSize) { 413 if (mOrientation == 0 || mOrientation == 180) { 414 mOutputFormat.setDimensions(mWidth, mHeight); 415 } else { 416 mOutputFormat.setDimensions(mHeight, mWidth); 417 } 418 mOrientationUpdated = true; 419 } 420 } 421 422 synchronized public void pauseVideo(boolean pauseState) { 423 if (isOpen()) { 424 if (pauseState && !mPaused) { 425 mMediaPlayer.pause(); 426 } else if (!pauseState && mPaused) { 427 mMediaPlayer.start(); 428 } 429 } 430 mPaused = pauseState; 431 } 432 433 /** Creates a media player, sets it up, and calls prepare */ 434 synchronized private boolean setupMediaPlayer(boolean useUrl) { 435 mPrepared = false; 436 mGotSize = false; 437 mPlaying = false; 438 mPaused = false; 439 mCompleted = false; 440 mNewFrameAvailable = false; 441 442 if (mLogVerbose) Log.v(TAG, "Setting up playback."); 443 444 if (mMediaPlayer != null) { 445 // Clean up existing media players 446 if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer."); 447 mMediaPlayer.reset(); 448 } else { 449 // Create new media player 450 if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer."); 451 mMediaPlayer = new MediaPlayer(); 452 } 453 454 if (mMediaPlayer == null) { 455 throw new RuntimeException("Unable to create a MediaPlayer!"); 456 } 457 458 // Set up data sources, etc 459 try { 460 if (useUrl) { 461 if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl); 462 mMediaPlayer.setDataSource(mSourceUrl); 463 } else { 464 if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset); 465 mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength()); 466 } 467 } catch(IOException e) { 468 mMediaPlayer.release(); 469 mMediaPlayer = null; 470 if (useUrl) { 471 throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e); 472 } else { 473 throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e); 474 } 475 } catch(IllegalArgumentException e) { 476 mMediaPlayer.release(); 477 mMediaPlayer = null; 478 if (useUrl) { 479 throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e); 480 } else { 481 throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e); 482 } 483 } 484 485 mMediaPlayer.setLooping(mLooping); 486 mMediaPlayer.setVolume(mVolume, mVolume); 487 488 // Bind it to our media frame 489 Surface surface = new Surface(mSurfaceTexture); 490 mMediaPlayer.setSurface(surface); 491 surface.release(); 492 493 // Connect Media Player to callbacks 494 495 mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener); 496 mMediaPlayer.setOnPreparedListener(onPreparedListener); 497 mMediaPlayer.setOnCompletionListener(onCompletionListener); 498 499 // Connect SurfaceTexture to callback 500 mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener); 501 502 if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer."); 503 mMediaPlayer.prepareAsync(); 504 505 return true; 506 } 507 508 private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener = 509 new MediaPlayer.OnVideoSizeChangedListener() { 510 public void onVideoSizeChanged(MediaPlayer mp, int width, int height) { 511 if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height); 512 if (!mGotSize) { 513 if (mOrientation == 0 || mOrientation == 180) { 514 mOutputFormat.setDimensions(width, height); 515 } else { 516 mOutputFormat.setDimensions(height, width); 517 } 518 mWidth = width; 519 mHeight = height; 520 } else { 521 if (mOutputFormat.getWidth() != width || 522 mOutputFormat.getHeight() != height) { 523 Log.e(TAG, "Multiple video size change events received!"); 524 } 525 } 526 synchronized(MediaSource.this) { 527 mGotSize = true; 528 MediaSource.this.notify(); 529 } 530 } 531 }; 532 533 private MediaPlayer.OnPreparedListener onPreparedListener = 534 new MediaPlayer.OnPreparedListener() { 535 public void onPrepared(MediaPlayer mp) { 536 if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared"); 537 synchronized(MediaSource.this) { 538 mPrepared = true; 539 MediaSource.this.notify(); 540 } 541 } 542 }; 543 544 private MediaPlayer.OnCompletionListener onCompletionListener = 545 new MediaPlayer.OnCompletionListener() { 546 public void onCompletion(MediaPlayer mp) { 547 if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback"); 548 synchronized(MediaSource.this) { 549 mCompleted = true; 550 } 551 } 552 }; 553 554 private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener = 555 new SurfaceTexture.OnFrameAvailableListener() { 556 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 557 if (mLogVerbose) Log.v(TAG, "New frame from media player"); 558 synchronized(MediaSource.this) { 559 if (mLogVerbose) Log.v(TAG, "New frame: notify"); 560 mNewFrameAvailable = true; 561 MediaSource.this.notify(); 562 if (mLogVerbose) Log.v(TAG, "New frame: notify done"); 563 } 564 } 565 }; 566 567} 568