xref: /frameworks/base/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/decoder/GpuVideoTrackDecoder.java

/*
 * Copyright (C) 2012 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.media.filterfw.decoder;

import android.annotation.TargetApi;
import android.graphics.SurfaceTexture;
import android.graphics.SurfaceTexture.OnFrameAvailableListener;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaFormat;
import android.view.Surface;

import androidx.media.filterfw.FrameImage2D;
import androidx.media.filterfw.ImageShader;
import androidx.media.filterfw.TextureSource;

import java.io.IOException;
import java.nio.ByteBuffer;

/**
 * {@link TrackDecoder} that decodes a video track and renders the frames onto a
 * {@link SurfaceTexture}.
 *
 * This implementation uses the GPU for image operations such as copying
 * and color-space conversion.
 */
@TargetApi(16)
public class GpuVideoTrackDecoder extends VideoTrackDecoder {

    /**
     * Identity fragment shader for external textures.
     */
    private static final String COPY_FRAGMENT_SHADER =
            "#extension GL_OES_EGL_image_external : require\n" +
            "precision mediump float;\n" +
            "uniform samplerExternalOES tex_sampler_0;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
            "}\n";

    private final TextureSource mTextureSource;
    private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor.
    private final float[] mTransformMatrix;

    private final int mOutputWidth;
    private final int mOutputHeight;

    private ImageShader mImageShader;

    private long mCurrentPresentationTimeUs;

    public GpuVideoTrackDecoder(
            int trackIndex, MediaFormat format, Listener listener) {
        super(trackIndex, format, listener);

        // Create a surface texture to be used by the video track decoder.
        mTextureSource = TextureSource.newExternalTexture();
        mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId());
        mSurfaceTexture.detachFromGLContext();
        mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() {
            @Override
            public void onFrameAvailable(SurfaceTexture surfaceTexture) {
                markFrameAvailable();
            }
        });

        mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH);
        mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT);

        mTransformMatrix = new float[16];
    }

    @Override
    protected MediaCodec initMediaCodec(MediaFormat format) {
        MediaCodec mediaCodec;
        try {
            mediaCodec = MediaCodec.createDecoderByType(
                    format.getString(MediaFormat.KEY_MIME));
        } catch (IOException e) {
            throw new RuntimeException(
                    "failed to create decoder for "
                    + format.getString(MediaFormat.KEY_MIME), e);
        }
        Surface surface = new Surface(mSurfaceTexture);
        mediaCodec.configure(format, surface, null, 0);
        surface.release();
        return mediaCodec;
    }

    @Override
    protected boolean onDataAvailable(
            MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
        boolean textureAvailable = waitForFrameGrab();

        mCurrentPresentationTimeUs = info.presentationTimeUs;

        // Only render the next frame if we weren't interrupted.
        codec.releaseOutputBuffer(bufferIndex, textureAvailable);

        if (textureAvailable) {
            if (updateTexture()) {
                notifyListener();
            }
        }

        return false;
    }

    /**
     * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates
     * the internal {@link SurfaceTexture}.
     */
    private boolean updateTexture() {
        // Wait for the frame we just released to appear in the texture.
        synchronized (mFrameMonitor) {
            try {
                while (!mFrameAvailable) {
                    mFrameMonitor.wait();
                }
                mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId());
                mSurfaceTexture.updateTexImage();
                mSurfaceTexture.detachFromGLContext();
                return true;
            } catch (InterruptedException e) {
                return false;
            }
        }
    }

    @Override
    protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
        TextureSource targetTexture = TextureSource.newExternalTexture();
        mSurfaceTexture.attachToGLContext(targetTexture.getTextureId());
        mSurfaceTexture.getTransformMatrix(mTransformMatrix);

        ImageShader imageShader = getImageShader();
        imageShader.setSourceTransform(mTransformMatrix);

        int outputWidth = mOutputWidth;
        int outputHeight = mOutputHeight;
        if (rotation != 0) {
            float[] targetCoords = getRotationCoords(rotation);
            imageShader.setTargetCoords(targetCoords);
            if (needSwapDimension(rotation)) {
                outputWidth = mOutputHeight;
                outputHeight = mOutputWidth;
            }
        }
        outputVideoFrame.resize(new int[] { outputWidth, outputHeight });
        imageShader.process(
                targetTexture,
                outputVideoFrame.lockRenderTarget(),
                outputWidth,
                outputHeight);
        outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
        outputVideoFrame.unlock();
        targetTexture.release();

        mSurfaceTexture.detachFromGLContext();
    }

    @Override
    public void release() {
        super.release();
        synchronized (mFrameMonitor) {
            mTextureSource.release();
            mSurfaceTexture.release();
        }
    }

    /*
     * This method has to be called on the MFF processing thread.
     */
    private ImageShader getImageShader() {
        if (mImageShader == null) {
            mImageShader = new ImageShader(COPY_FRAGMENT_SHADER);
            mImageShader.setTargetRect(0f, 1f, 1f, -1f);
        }
        return mImageShader;
    }

    /**
     * Get the quad coords for rotation.
     * @param rotation applied to the frame, value is one of
     *   {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}
     * @return coords the calculated quad coords for the given rotation
     */
    private static float[] getRotationCoords(int rotation) {
         switch(rotation) {
             case MediaDecoder.ROTATE_90_RIGHT:
                 return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f };
             case MediaDecoder.ROTATE_180:
                 return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f };
             case MediaDecoder.ROTATE_90_LEFT:
                 return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f };
             case MediaDecoder.ROTATE_NONE:
                 return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f };
             default:
                 throw new IllegalArgumentException("Unsupported rotation angle.");
         }
     }

}