1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package androidx.media.filterfw.decoder;
18
19import android.annotation.TargetApi;
20import android.graphics.SurfaceTexture;
21import android.graphics.SurfaceTexture.OnFrameAvailableListener;
22import android.media.MediaCodec;
23import android.media.MediaCodec.BufferInfo;
24import android.media.MediaFormat;
25import android.view.Surface;
26
27import androidx.media.filterfw.FrameImage2D;
28import androidx.media.filterfw.ImageShader;
29import androidx.media.filterfw.TextureSource;
30
31import java.io.IOException;
32import java.nio.ByteBuffer;
33
34/**
35 * {@link TrackDecoder} that decodes a video track and renders the frames onto a
36 * {@link SurfaceTexture}.
37 *
38 * This implementation uses the GPU for image operations such as copying
39 * and color-space conversion.
40 */
41@TargetApi(16)
42public class GpuVideoTrackDecoder extends VideoTrackDecoder {
43
44    /**
45     * Identity fragment shader for external textures.
46     */
47    private static final String COPY_FRAGMENT_SHADER =
48            "#extension GL_OES_EGL_image_external : require\n" +
49            "precision mediump float;\n" +
50            "uniform samplerExternalOES tex_sampler_0;\n" +
51            "varying vec2 v_texcoord;\n" +
52            "void main() {\n" +
53            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
54            "}\n";
55
56    private final TextureSource mTextureSource;
57    private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor.
58    private final float[] mTransformMatrix;
59
60    private final int mOutputWidth;
61    private final int mOutputHeight;
62
63    private ImageShader mImageShader;
64
65    private long mCurrentPresentationTimeUs;
66
67    public GpuVideoTrackDecoder(
68            int trackIndex, MediaFormat format, Listener listener) {
69        super(trackIndex, format, listener);
70
71        // Create a surface texture to be used by the video track decoder.
72        mTextureSource = TextureSource.newExternalTexture();
73        mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId());
74        mSurfaceTexture.detachFromGLContext();
75        mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() {
76            @Override
77            public void onFrameAvailable(SurfaceTexture surfaceTexture) {
78                markFrameAvailable();
79            }
80        });
81
82        mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH);
83        mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
84
85        mTransformMatrix = new float[16];
86    }
87
88    @Override
89    protected MediaCodec initMediaCodec(MediaFormat format) {
90        MediaCodec mediaCodec;
91        try {
92            mediaCodec = MediaCodec.createDecoderByType(
93                    format.getString(MediaFormat.KEY_MIME));
94        } catch (IOException e) {
95            throw new RuntimeException(
96                    "failed to create decoder for "
97                    + format.getString(MediaFormat.KEY_MIME), e);
98        }
99        Surface surface = new Surface(mSurfaceTexture);
100        mediaCodec.configure(format, surface, null, 0);
101        surface.release();
102        return mediaCodec;
103    }
104
105    @Override
106    protected boolean onDataAvailable(
107            MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
108        boolean textureAvailable = waitForFrameGrab();
109
110        mCurrentPresentationTimeUs = info.presentationTimeUs;
111
112        // Only render the next frame if we weren't interrupted.
113        codec.releaseOutputBuffer(bufferIndex, textureAvailable);
114
115        if (textureAvailable) {
116            if (updateTexture()) {
117                notifyListener();
118            }
119        }
120
121        return false;
122    }
123
124    /**
125     * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates
126     * the internal {@link SurfaceTexture}.
127     */
128    private boolean updateTexture() {
129        // Wait for the frame we just released to appear in the texture.
130        synchronized (mFrameMonitor) {
131            try {
132                while (!mFrameAvailable) {
133                    mFrameMonitor.wait();
134                }
135                mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId());
136                mSurfaceTexture.updateTexImage();
137                mSurfaceTexture.detachFromGLContext();
138                return true;
139            } catch (InterruptedException e) {
140                return false;
141            }
142        }
143    }
144
145    @Override
146    protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
147        TextureSource targetTexture = TextureSource.newExternalTexture();
148        mSurfaceTexture.attachToGLContext(targetTexture.getTextureId());
149        mSurfaceTexture.getTransformMatrix(mTransformMatrix);
150
151        ImageShader imageShader = getImageShader();
152        imageShader.setSourceTransform(mTransformMatrix);
153
154        int outputWidth = mOutputWidth;
155        int outputHeight = mOutputHeight;
156        if (rotation != 0) {
157            float[] targetCoords = getRotationCoords(rotation);
158            imageShader.setTargetCoords(targetCoords);
159            if (needSwapDimension(rotation)) {
160                outputWidth = mOutputHeight;
161                outputHeight = mOutputWidth;
162            }
163        }
164        outputVideoFrame.resize(new int[] { outputWidth, outputHeight });
165        imageShader.process(
166                targetTexture,
167                outputVideoFrame.lockRenderTarget(),
168                outputWidth,
169                outputHeight);
170        outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
171        outputVideoFrame.unlock();
172        targetTexture.release();
173
174        mSurfaceTexture.detachFromGLContext();
175    }
176
177    @Override
178    public void release() {
179        super.release();
180        synchronized (mFrameMonitor) {
181            mTextureSource.release();
182            mSurfaceTexture.release();
183        }
184    }
185
186    /*
187     * This method has to be called on the MFF processing thread.
188     */
189    private ImageShader getImageShader() {
190        if (mImageShader == null) {
191            mImageShader = new ImageShader(COPY_FRAGMENT_SHADER);
192            mImageShader.setTargetRect(0f, 1f, 1f, -1f);
193        }
194        return mImageShader;
195    }
196
197    /**
198     * Get the quad coords for rotation.
199     * @param rotation applied to the frame, value is one of
200     *   {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}
201     * @return coords the calculated quad coords for the given rotation
202     */
203    private static float[] getRotationCoords(int rotation) {
204         switch(rotation) {
205             case MediaDecoder.ROTATE_90_RIGHT:
206                 return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f };
207             case MediaDecoder.ROTATE_180:
208                 return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f };
209             case MediaDecoder.ROTATE_90_LEFT:
210                 return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f };
211             case MediaDecoder.ROTATE_NONE:
212                 return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f };
213             default:
214                 throw new IllegalArgumentException("Unsupported rotation angle.");
215         }
216     }
217
218}
219