xref: /system/media/mca/filterpacks/videosrc/java/MediaSource.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package android.filterpacks.videosrc;

import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.filterfw.core.Filter;
import android.filterfw.core.FilterContext;
import android.filterfw.core.Frame;
import android.filterfw.core.FrameFormat;
import android.filterfw.core.FrameManager;
import android.filterfw.core.GenerateFieldPort;
import android.filterfw.core.GenerateFinalPort;
import android.filterfw.core.GLFrame;
import android.filterfw.core.KeyValueMap;
import android.filterfw.core.MutableFrameFormat;
import android.filterfw.core.NativeFrame;
import android.filterfw.core.Program;
import android.filterfw.core.ShaderProgram;
import android.filterfw.format.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.os.ConditionVariable;

import java.io.IOException;
import java.io.FileDescriptor;
import java.lang.IllegalArgumentException;
import java.util.List;
import java.util.Set;

import android.util.Log;

/**
 * @hide
 */
public class MediaSource extends Filter {

    /** User-visible parameters */

    /** The source URL for the media source. Can be an http: link to a remote
     * resource, or a file: link to a local media file */
    @GenerateFieldPort(name = "sourceUrl", hasDefault = true)
    private String mSourceUrl = "";

    /** An open asset file descriptor to a local media source. Default is null */
    @GenerateFieldPort(name = "sourceAsset", hasDefault = true)
    private AssetFileDescriptor mSourceAsset = null;

    /** Whether the media source is a URL or an asset file descriptor. Defaults to false. */
    @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true)
    private boolean mSelectedIsUrl = false;

    /** Whether the filter will always wait for a new video frame, or whether it
     * will output an old frame again if a new frame isn't available. Defaults to true.
     */
    @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
    private boolean mWaitForNewFrame = true;

    /** Whether the media source should loop automatically or not. Defaults to true. */
    @GenerateFieldPort(name = "loop", hasDefault = true)
    private boolean mLooping = true;

    /** Volume control. Currently sound is piped directly to the speakers, so this defaults to mute. */
    @GenerateFieldPort(name = "volume", hasDefault = true)
    private float mVolume = 0.f;

    private MediaPlayer mMediaPlayer;
    private GLFrame mMediaFrame;
    private SurfaceTexture mSurfaceTexture;
    private ShaderProgram mFrameExtractor;
    private MutableFrameFormat mOutputFormat;
    private ConditionVariable mNewFrameAvailable;
    private float[] mFrameTransform;

    private final String mFrameShader =
            "#extension GL_OES_EGL_image_external : require\n" +
            "precision mediump float;\n" +
            "uniform mat4 frame_transform;\n" +
            "uniform samplerExternalOES tex_sampler_0;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec2 transformed_texcoord = (frame_transform * vec4(v_texcoord, 0., 1.) ).xy;" +
            "  gl_FragColor = texture2D(tex_sampler_0, transformed_texcoord);\n" +
            "}\n";

    private boolean mGotSize;
    private boolean mPrepared;
    private boolean mPlaying;
    private boolean mPaused;
    private boolean mCompleted;

    private final boolean mLogVerbose;
    private static final String TAG = "MediaSource";

    public MediaSource(String name) {
        super(name);
        mNewFrameAvailable = new ConditionVariable();
        mFrameTransform = new float[16];

        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
    }

    @Override
    public void setupPorts() {
        // Add input port
        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                  FrameFormat.TARGET_GPU));
    }

    private void createFormats() {
        mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                           FrameFormat.TARGET_GPU);
    }

    @Override
    protected void prepare(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Preparing MediaSource");

        mFrameExtractor = new ShaderProgram(context, mFrameShader);
        // SurfaceTexture defines (0,0) to be bottom-left. The filter framework
        // defines (0,0) as top-left, so do the flip here.
        mFrameExtractor.setSourceRect(0, 1, 1, -1);

        createFormats();

        mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
                                                                       GLFrame.EXTERNAL_TEXTURE,
                                                                       0);
        mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());

    }

    @Override
    public void open(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Opening MediaSource");
        if (mLogVerbose) {
          if (mSelectedIsUrl) {
            Log.v(TAG, "Current URL is " + mSourceUrl);
          } else {
            Log.v(TAG, "Current source is Asset!");
          }
        }
        if (!setupMediaPlayer(mSelectedIsUrl)) {
          throw new RuntimeException("Error setting up MediaPlayer!");
        }
    }

    @Override
    public void process(FilterContext context) {
        // Note: process is synchronized by its caller in the Filter base class
        if (mLogVerbose) Log.v(TAG, "Processing new frame");

        if (mMediaPlayer == null) {
            // Something went wrong in initialization or parameter updates
            throw new NullPointerException("Unexpected null media player!");
        }

        if (mCompleted) {
            // Video playback is done, so close us down
            closeOutputPort("video");
            return;
        }

        if (!mPlaying) {
            int waitCount = 0;
            while (!mGotSize || !mPrepared) {
                try {
                    this.wait(100);
                } catch (InterruptedException e) {
                    // ignoring
                }
                if (mCompleted) {
                    // Video playback is done, so close us down
                    closeOutputPort("video");
                    return;
                }
                waitCount++;
                if (waitCount == 50) {
                    mMediaPlayer.release();
                    throw new RuntimeException("MediaPlayer timed out while preparing!");
                }
            }
            mMediaPlayer.start();
        }

        // Use last frame if paused, unless just starting playback, in which case
        // we want at least one valid frame before pausing
        if (!mPaused || !mPlaying) {
            if (mWaitForNewFrame) {
                boolean gotNewFrame;
                gotNewFrame = mNewFrameAvailable.block(1000);
                if (!gotNewFrame) {
                    if (mCompleted) {
                        // Video playback is done, so close us down
                        closeOutputPort("video");
                        return;
                    } else {
                        throw new RuntimeException("Timeout waiting for new frame!");
                    }
                }
                mNewFrameAvailable.close();
            }

            mSurfaceTexture.updateTexImage();

            mSurfaceTexture.getTransformMatrix(mFrameTransform);
            mFrameExtractor.setHostValue("frame_transform", mFrameTransform);
        }

        Frame output = context.getFrameManager().newFrame(mOutputFormat);
        mFrameExtractor.process(mMediaFrame, output);

        long timestamp = mSurfaceTexture.getTimestamp();
        if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
        output.setTimestamp(timestamp);

        pushOutput("video", output);
        output.release();

        mPlaying = true;
    }

    @Override
    public void close(FilterContext context) {
        if (mMediaPlayer.isPlaying()) {
            mMediaPlayer.stop();
        }
        mMediaPlayer.release();
        mMediaPlayer = null;
        if (mLogVerbose) Log.v(TAG, "MediaSource closed");
    }

    @Override
    public void tearDown(FilterContext context) {
        if (mMediaFrame != null) {
            mMediaFrame.release();
        }
    }

    // When updating the port values of the filter, users can update sourceIsUrl to switch
    //   between using URL objects or Assets.
    // If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player
    //   uses Url objects/Asset.
    // Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch
    //   sourceIsUrl next time.
    @Override
    public void fieldPortValueUpdated(String name, FilterContext context) {
        if (name.equals("sourceUrl")) {
           if (isOpen()) {
                if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                if (mSelectedIsUrl) {
                    setupMediaPlayer(mSelectedIsUrl);
                }
            }
        } else if (name.equals("sourceAsset") ) {
            if (isOpen()) {
                if (mLogVerbose) Log.v(TAG, "Opening new source FD");
                if (!mSelectedIsUrl) {
                    setupMediaPlayer(mSelectedIsUrl);
                }
            }
        } else if (name.equals("loop")) {
            if (isOpen()) {
                mMediaPlayer.setLooping(mLooping);
            }
        } else if (name.equals("sourceIsUrl")) {
            if (isOpen()){
                if (mSelectedIsUrl){
                    if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                } else {
                    if (mLogVerbose) Log.v(TAG, "Opening new source Asset");
                }
                setupMediaPlayer(mSelectedIsUrl);
            }
        } else if (name.equals("volume")) {
            if (isOpen()) {
                mMediaPlayer.setVolume(mVolume, mVolume);
            }
        }
    }

    synchronized public void pauseVideo(boolean pauseState) {
        if (isOpen()) {
            if (pauseState && !mPaused) {
                mMediaPlayer.pause();
            } else if (!pauseState && mPaused) {
                mMediaPlayer.start();
            }
        }
        mPaused = pauseState;
    }

    /** Creates a media player, sets it up, and calls prepare */
    synchronized private boolean setupMediaPlayer(boolean useUrl) {
        mPrepared = false;
        mGotSize = false;
        mPlaying = false;
        mPaused = false;
        mCompleted = false;

        if (mLogVerbose) Log.v(TAG, "Setting up playback.");

        if (mMediaPlayer != null) {
            // Clean up existing media player
            if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer.");
            mMediaPlayer.reset();
        } else {
            // Create new media player
            if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer.");
            mMediaPlayer = new MediaPlayer();
        }

        if (mMediaPlayer == null) {
            throw new RuntimeException("Unable to create a MediaPlayer!");
        }

        // Set up data sources, etc
        try {
            if (useUrl) {
                if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl);
                mMediaPlayer.setDataSource(mSourceUrl);
            } else {
                if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset);
                mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength());
            }
        } catch(IOException e) {
            mMediaPlayer.release();
            mMediaPlayer = null;
            if (useUrl) {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
            } else {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
            }
        } catch(IllegalArgumentException e) {
            mMediaPlayer.release();
            mMediaPlayer = null;
            if (useUrl) {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
            } else {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
            }
        }

        mMediaPlayer.setLooping(mLooping);
        mMediaPlayer.setVolume(mVolume, mVolume);

        // Bind it to our media frame
        mMediaPlayer.setTexture(mSurfaceTexture);

        // Connect Media Player to callbacks

        mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener);
        mMediaPlayer.setOnPreparedListener(onPreparedListener);
        mMediaPlayer.setOnCompletionListener(onCompletionListener);

        // Connect SurfaceTexture to callback
        mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener);

        if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer.");
        mMediaPlayer.prepareAsync();

        return true;
    }

    private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener =
            new MediaPlayer.OnVideoSizeChangedListener() {
        public void onVideoSizeChanged(MediaPlayer mp, int width, int height) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height);
            if (!mGotSize) {
                mOutputFormat.setDimensions(width, height);
            } else {
                if (mOutputFormat.getWidth() != width ||
                    mOutputFormat.getHeight() != height) {
                    Log.e(TAG, "Multiple video size change events received!");
                }
            }
            synchronized(this) {
                mGotSize = true;
                this.notify();
            }
        }
    };

    private MediaPlayer.OnPreparedListener onPreparedListener =
            new MediaPlayer.OnPreparedListener() {
        public void onPrepared(MediaPlayer mp) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared");
            synchronized(this) {
                mPrepared = true;
                this.notify();
            }
        }
    };

    private MediaPlayer.OnCompletionListener onCompletionListener =
            new MediaPlayer.OnCompletionListener() {
        public void onCompletion(MediaPlayer mp) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback");
            synchronized(this) {
                mCompleted = true;
            }
        }
    };

    private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener =
            new SurfaceTexture.OnFrameAvailableListener() {
        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
            if (mLogVerbose) Log.v(TAG, "New frame from media player");
            mNewFrameAvailable.open();
        }
    };

}