1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18package android.filterpacks.videosrc; 19 20import android.content.Context; 21import android.content.res.AssetFileDescriptor; 22import android.filterfw.core.Filter; 23import android.filterfw.core.FilterContext; 24import android.filterfw.core.Frame; 25import android.filterfw.core.FrameFormat; 26import android.filterfw.core.FrameManager; 27import android.filterfw.core.GenerateFieldPort; 28import android.filterfw.core.GenerateFinalPort; 29import android.filterfw.core.GLFrame; 30import android.filterfw.core.KeyValueMap; 31import android.filterfw.core.MutableFrameFormat; 32import android.filterfw.core.NativeFrame; 33import android.filterfw.core.Program; 34import android.filterfw.core.ShaderProgram; 35import android.filterfw.format.ImageFormat; 36import android.graphics.SurfaceTexture; 37import android.media.MediaPlayer; 38import android.net.Uri; 39import android.os.ConditionVariable; 40import android.opengl.Matrix; 41import android.view.Surface; 42 43import java.io.IOException; 44import java.io.FileDescriptor; 45import java.lang.IllegalArgumentException; 46import java.util.List; 47import java.util.Set; 48 49import android.util.Log; 50 51/** 52 * @hide 53 */ 54public class MediaSource extends Filter { 55 56 /** User-visible parameters */ 57 58 /** The source URL for the media source. Can be an http: link to a remote 59 * resource, or a file: link to a local media file 60 */ 61 @GenerateFieldPort(name = "sourceUrl", hasDefault = true) 62 private String mSourceUrl = ""; 63 64 /** An open asset file descriptor to a local media source. Default is null */ 65 @GenerateFieldPort(name = "sourceAsset", hasDefault = true) 66 private AssetFileDescriptor mSourceAsset = null; 67 68 /** The context for the MediaPlayer to resolve the sourceUrl. 69 * Make sure this is set before the sourceUrl to avoid unexpected result. 70 * If the sourceUrl is not a content URI, it is OK to keep this as null. */ 71 @GenerateFieldPort(name = "context", hasDefault = true) 72 private Context mContext = null; 73 74 /** Whether the media source is a URL or an asset file descriptor. Defaults 75 * to false. 76 */ 77 @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true) 78 private boolean mSelectedIsUrl = false; 79 80 /** Whether the filter will always wait for a new video frame, or whether it 81 * will output an old frame again if a new frame isn't available. Defaults 82 * to true. 83 */ 84 @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true) 85 private boolean mWaitForNewFrame = true; 86 87 /** Whether the media source should loop automatically or not. Defaults to 88 * true. 89 */ 90 @GenerateFieldPort(name = "loop", hasDefault = true) 91 private boolean mLooping = true; 92 93 /** Volume control. Currently sound is piped directly to the speakers, so 94 * this defaults to mute. 95 */ 96 @GenerateFieldPort(name = "volume", hasDefault = true) 97 private float mVolume = 0.f; 98 99 /** Orientation. This controls the output orientation of the video. Valid 100 * values are 0, 90, 180, 270 101 */ 102 @GenerateFieldPort(name = "orientation", hasDefault = true) 103 private int mOrientation = 0; 104 105 private MediaPlayer mMediaPlayer; 106 private GLFrame mMediaFrame; 107 private SurfaceTexture mSurfaceTexture; 108 private ShaderProgram mFrameExtractor; 109 private MutableFrameFormat mOutputFormat; 110 private int mWidth, mHeight; 111 112 // Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT 113 private static final int PREP_TIMEOUT = 100; // ms 114 private static final int PREP_TIMEOUT_REPEAT = 100; 115 private static final int NEWFRAME_TIMEOUT = 100; //ms 116 private static final int NEWFRAME_TIMEOUT_REPEAT = 10; 117 118 // This is an identity shader; not using the default identity 119 // shader because reading from a SurfaceTexture requires the 120 // GL_OES_EGL_image_external extension. 121 private final String mFrameShader = 122 "#extension GL_OES_EGL_image_external : require\n" + 123 "precision mediump float;\n" + 124 "uniform samplerExternalOES tex_sampler_0;\n" + 125 "varying vec2 v_texcoord;\n" + 126 "void main() {\n" + 127 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 128 "}\n"; 129 130 // The following transforms enable rotation of the decoded source. 131 // These are multiplied with the transform obtained from the 132 // SurfaceTexture to get the final transform to be set on the media source. 133 // Currently, given a device orientation, the MediaSource rotates in such a way 134 // that the source is displayed upright. A particular use case 135 // is "Background Replacement" feature in the Camera app 136 // where the MediaSource rotates the source to align with the camera feed and pass it 137 // on to the backdropper filter. The backdropper only does the blending 138 // and does not have to do any rotation 139 // (except for mirroring in case of front camera). 140 // TODO: Currently the rotations are spread over a bunch of stages in the 141 // pipeline. A cleaner design 142 // could be to cast away all the rotation in a separate filter or attach a transform 143 // to the frame so that MediaSource itself need not know about any rotation. 144 private static final float[] mSourceCoords_0 = { 1, 1, 0, 1, 145 0, 1, 0, 1, 146 1, 0, 0, 1, 147 0, 0, 0, 1 }; 148 private static final float[] mSourceCoords_270 = { 0, 1, 0, 1, 149 0, 0, 0, 1, 150 1, 1, 0, 1, 151 1, 0, 0, 1 }; 152 private static final float[] mSourceCoords_180 = { 0, 0, 0, 1, 153 1, 0, 0, 1, 154 0, 1, 0, 1, 155 1, 1, 0, 1 }; 156 private static final float[] mSourceCoords_90 = { 1, 0, 0, 1, 157 1, 1, 0, 1, 158 0, 0, 0, 1, 159 0, 1, 0, 1 }; 160 161 private boolean mGotSize; 162 private boolean mPrepared; 163 private boolean mPlaying; 164 private boolean mNewFrameAvailable; 165 private boolean mOrientationUpdated; 166 private boolean mPaused; 167 private boolean mCompleted; 168 169 private final boolean mLogVerbose; 170 private static final String TAG = "MediaSource"; 171 172 public MediaSource(String name) { 173 super(name); 174 mNewFrameAvailable = false; 175 176 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 177 } 178 179 @Override 180 public void setupPorts() { 181 // Add input port 182 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 183 FrameFormat.TARGET_GPU)); 184 } 185 186 private void createFormats() { 187 mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 188 FrameFormat.TARGET_GPU); 189 } 190 191 @Override 192 protected void prepare(FilterContext context) { 193 if (mLogVerbose) Log.v(TAG, "Preparing MediaSource"); 194 195 mFrameExtractor = new ShaderProgram(context, mFrameShader); 196 // SurfaceTexture defines (0,0) to be bottom-left. The filter framework 197 // defines (0,0) as top-left, so do the flip here. 198 mFrameExtractor.setSourceRect(0, 1, 1, -1); 199 200 createFormats(); 201 } 202 203 @Override 204 public void open(FilterContext context) { 205 if (mLogVerbose) { 206 Log.v(TAG, "Opening MediaSource"); 207 if (mSelectedIsUrl) { 208 Log.v(TAG, "Current URL is " + mSourceUrl); 209 } else { 210 Log.v(TAG, "Current source is Asset!"); 211 } 212 } 213 214 mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame( 215 mOutputFormat, 216 GLFrame.EXTERNAL_TEXTURE, 217 0); 218 219 mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); 220 221 if (!setupMediaPlayer(mSelectedIsUrl)) { 222 throw new RuntimeException("Error setting up MediaPlayer!"); 223 } 224 } 225 226 @Override 227 public void process(FilterContext context) { 228 // Note: process is synchronized by its caller in the Filter base class 229 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 230 231 if (mMediaPlayer == null) { 232 // Something went wrong in initialization or parameter updates 233 throw new NullPointerException("Unexpected null media player!"); 234 } 235 236 if (mCompleted) { 237 // Video playback is done, so close us down 238 closeOutputPort("video"); 239 return; 240 } 241 242 if (!mPlaying) { 243 int waitCount = 0; 244 if (mLogVerbose) Log.v(TAG, "Waiting for preparation to complete"); 245 while (!mGotSize || !mPrepared) { 246 try { 247 this.wait(PREP_TIMEOUT); 248 } catch (InterruptedException e) { 249 // ignoring 250 } 251 if (mCompleted) { 252 // Video playback is done, so close us down 253 closeOutputPort("video"); 254 return; 255 } 256 waitCount++; 257 if (waitCount == PREP_TIMEOUT_REPEAT) { 258 mMediaPlayer.release(); 259 throw new RuntimeException("MediaPlayer timed out while preparing!"); 260 } 261 } 262 if (mLogVerbose) Log.v(TAG, "Starting playback"); 263 mMediaPlayer.start(); 264 } 265 266 // Use last frame if paused, unless just starting playback, in which case 267 // we want at least one valid frame before pausing 268 if (!mPaused || !mPlaying) { 269 if (mWaitForNewFrame) { 270 if (mLogVerbose) Log.v(TAG, "Waiting for new frame"); 271 272 int waitCount = 0; 273 while (!mNewFrameAvailable) { 274 if (waitCount == NEWFRAME_TIMEOUT_REPEAT) { 275 if (mCompleted) { 276 // Video playback is done, so close us down 277 closeOutputPort("video"); 278 return; 279 } else { 280 throw new RuntimeException("Timeout waiting for new frame!"); 281 } 282 } 283 try { 284 this.wait(NEWFRAME_TIMEOUT); 285 } catch (InterruptedException e) { 286 if (mLogVerbose) Log.v(TAG, "interrupted"); 287 // ignoring 288 } 289 waitCount++; 290 } 291 mNewFrameAvailable = false; 292 if (mLogVerbose) Log.v(TAG, "Got new frame"); 293 } 294 295 mSurfaceTexture.updateTexImage(); 296 mOrientationUpdated = true; 297 } 298 if (mOrientationUpdated) { 299 float[] surfaceTransform = new float[16]; 300 mSurfaceTexture.getTransformMatrix(surfaceTransform); 301 302 float[] sourceCoords = new float[16]; 303 switch (mOrientation) { 304 default: 305 case 0: 306 Matrix.multiplyMM(sourceCoords, 0, 307 surfaceTransform, 0, 308 mSourceCoords_0, 0); 309 break; 310 case 90: 311 Matrix.multiplyMM(sourceCoords, 0, 312 surfaceTransform, 0, 313 mSourceCoords_90, 0); 314 break; 315 case 180: 316 Matrix.multiplyMM(sourceCoords, 0, 317 surfaceTransform, 0, 318 mSourceCoords_180, 0); 319 break; 320 case 270: 321 Matrix.multiplyMM(sourceCoords, 0, 322 surfaceTransform, 0, 323 mSourceCoords_270, 0); 324 break; 325 } 326 if (mLogVerbose) { 327 Log.v(TAG, "OrientationHint = " + mOrientation); 328 String temp = String.format("SetSourceRegion: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f", 329 sourceCoords[4], sourceCoords[5],sourceCoords[0], sourceCoords[1], 330 sourceCoords[12], sourceCoords[13],sourceCoords[8], sourceCoords[9]); 331 Log.v(TAG, temp); 332 } 333 mFrameExtractor.setSourceRegion(sourceCoords[4], sourceCoords[5], 334 sourceCoords[0], sourceCoords[1], 335 sourceCoords[12], sourceCoords[13], 336 sourceCoords[8], sourceCoords[9]); 337 mOrientationUpdated = false; 338 } 339 340 Frame output = context.getFrameManager().newFrame(mOutputFormat); 341 mFrameExtractor.process(mMediaFrame, output); 342 343 long timestamp = mSurfaceTexture.getTimestamp(); 344 if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s"); 345 output.setTimestamp(timestamp); 346 347 pushOutput("video", output); 348 output.release(); 349 350 mPlaying = true; 351 } 352 353 @Override 354 public void close(FilterContext context) { 355 if (mMediaPlayer.isPlaying()) { 356 mMediaPlayer.stop(); 357 } 358 mPrepared = false; 359 mGotSize = false; 360 mPlaying = false; 361 mPaused = false; 362 mCompleted = false; 363 mNewFrameAvailable = false; 364 365 mMediaPlayer.release(); 366 mMediaPlayer = null; 367 mSurfaceTexture.release(); 368 mSurfaceTexture = null; 369 if (mLogVerbose) Log.v(TAG, "MediaSource closed"); 370 } 371 372 @Override 373 public void tearDown(FilterContext context) { 374 if (mMediaFrame != null) { 375 mMediaFrame.release(); 376 } 377 } 378 379 // When updating the port values of the filter, users can update sourceIsUrl to switch 380 // between using URL objects or Assets. 381 // If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player 382 // uses Url objects/Asset. 383 // Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch 384 // sourceIsUrl next time. 385 @Override 386 public void fieldPortValueUpdated(String name, FilterContext context) { 387 if (mLogVerbose) Log.v(TAG, "Parameter update"); 388 if (name.equals("sourceUrl")) { 389 if (isOpen()) { 390 if (mLogVerbose) Log.v(TAG, "Opening new source URL"); 391 if (mSelectedIsUrl) { 392 setupMediaPlayer(mSelectedIsUrl); 393 } 394 } 395 } else if (name.equals("sourceAsset") ) { 396 if (isOpen()) { 397 if (mLogVerbose) Log.v(TAG, "Opening new source FD"); 398 if (!mSelectedIsUrl) { 399 setupMediaPlayer(mSelectedIsUrl); 400 } 401 } 402 } else if (name.equals("loop")) { 403 if (isOpen()) { 404 mMediaPlayer.setLooping(mLooping); 405 } 406 } else if (name.equals("sourceIsUrl")) { 407 if (isOpen()){ 408 if (mSelectedIsUrl){ 409 if (mLogVerbose) Log.v(TAG, "Opening new source URL"); 410 } else { 411 if (mLogVerbose) Log.v(TAG, "Opening new source Asset"); 412 } 413 setupMediaPlayer(mSelectedIsUrl); 414 } 415 } else if (name.equals("volume")) { 416 if (isOpen()) { 417 mMediaPlayer.setVolume(mVolume, mVolume); 418 } 419 } else if (name.equals("orientation") && mGotSize) { 420 if (mOrientation == 0 || mOrientation == 180) { 421 mOutputFormat.setDimensions(mWidth, mHeight); 422 } else { 423 mOutputFormat.setDimensions(mHeight, mWidth); 424 } 425 mOrientationUpdated = true; 426 } 427 } 428 429 synchronized public void pauseVideo(boolean pauseState) { 430 if (isOpen()) { 431 if (pauseState && !mPaused) { 432 mMediaPlayer.pause(); 433 } else if (!pauseState && mPaused) { 434 mMediaPlayer.start(); 435 } 436 } 437 mPaused = pauseState; 438 } 439 440 /** Creates a media player, sets it up, and calls prepare */ 441 synchronized private boolean setupMediaPlayer(boolean useUrl) { 442 mPrepared = false; 443 mGotSize = false; 444 mPlaying = false; 445 mPaused = false; 446 mCompleted = false; 447 mNewFrameAvailable = false; 448 449 if (mLogVerbose) Log.v(TAG, "Setting up playback."); 450 451 if (mMediaPlayer != null) { 452 // Clean up existing media players 453 if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer."); 454 mMediaPlayer.reset(); 455 } else { 456 // Create new media player 457 if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer."); 458 mMediaPlayer = new MediaPlayer(); 459 } 460 461 if (mMediaPlayer == null) { 462 throw new RuntimeException("Unable to create a MediaPlayer!"); 463 } 464 465 // Set up data sources, etc 466 try { 467 if (useUrl) { 468 if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl); 469 if (mContext == null) { 470 mMediaPlayer.setDataSource(mSourceUrl); 471 } else { 472 mMediaPlayer.setDataSource(mContext, Uri.parse(mSourceUrl.toString())); 473 } 474 } else { 475 if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset); 476 mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength()); 477 } 478 } catch(IOException e) { 479 mMediaPlayer.release(); 480 mMediaPlayer = null; 481 if (useUrl) { 482 throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e); 483 } else { 484 throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e); 485 } 486 } catch(IllegalArgumentException e) { 487 mMediaPlayer.release(); 488 mMediaPlayer = null; 489 if (useUrl) { 490 throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e); 491 } else { 492 throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e); 493 } 494 } 495 496 mMediaPlayer.setLooping(mLooping); 497 mMediaPlayer.setVolume(mVolume, mVolume); 498 499 // Bind it to our media frame 500 Surface surface = new Surface(mSurfaceTexture); 501 mMediaPlayer.setSurface(surface); 502 surface.release(); 503 504 // Connect Media Player to callbacks 505 506 mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener); 507 mMediaPlayer.setOnPreparedListener(onPreparedListener); 508 mMediaPlayer.setOnCompletionListener(onCompletionListener); 509 510 // Connect SurfaceTexture to callback 511 mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener); 512 513 if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer."); 514 mMediaPlayer.prepareAsync(); 515 516 return true; 517 } 518 519 private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener = 520 new MediaPlayer.OnVideoSizeChangedListener() { 521 public void onVideoSizeChanged(MediaPlayer mp, int width, int height) { 522 if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height); 523 if (!mGotSize) { 524 if (mOrientation == 0 || mOrientation == 180) { 525 mOutputFormat.setDimensions(width, height); 526 } else { 527 mOutputFormat.setDimensions(height, width); 528 } 529 mWidth = width; 530 mHeight = height; 531 } else { 532 if (mOutputFormat.getWidth() != width || 533 mOutputFormat.getHeight() != height) { 534 Log.e(TAG, "Multiple video size change events received!"); 535 } 536 } 537 synchronized(MediaSource.this) { 538 mGotSize = true; 539 MediaSource.this.notify(); 540 } 541 } 542 }; 543 544 private MediaPlayer.OnPreparedListener onPreparedListener = 545 new MediaPlayer.OnPreparedListener() { 546 public void onPrepared(MediaPlayer mp) { 547 if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared"); 548 synchronized(MediaSource.this) { 549 mPrepared = true; 550 MediaSource.this.notify(); 551 } 552 } 553 }; 554 555 private MediaPlayer.OnCompletionListener onCompletionListener = 556 new MediaPlayer.OnCompletionListener() { 557 public void onCompletion(MediaPlayer mp) { 558 if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback"); 559 synchronized(MediaSource.this) { 560 mCompleted = true; 561 } 562 } 563 }; 564 565 private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener = 566 new SurfaceTexture.OnFrameAvailableListener() { 567 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 568 if (mLogVerbose) Log.v(TAG, "New frame from media player"); 569 synchronized(MediaSource.this) { 570 if (mLogVerbose) Log.v(TAG, "New frame: notify"); 571 mNewFrameAvailable = true; 572 MediaSource.this.notify(); 573 if (mLogVerbose) Log.v(TAG, "New frame: notify done"); 574 } 575 } 576 }; 577 578} 579