GpuVideoTrackDecoder.java revision 227b47625d7482b5b47ad0e4c70ce0a246236ade 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package androidx.media.filterfw.decoder;
18
19import android.annotation.TargetApi;
20import android.graphics.SurfaceTexture;
21import android.graphics.SurfaceTexture.OnFrameAvailableListener;
22import android.media.MediaCodec;
23import android.media.MediaCodec.BufferInfo;
24import android.media.MediaFormat;
25import android.view.Surface;
26
27import androidx.media.filterfw.FrameImage2D;
28import androidx.media.filterfw.ImageShader;
29import androidx.media.filterfw.TextureSource;
30
31import java.nio.ByteBuffer;
32
33/**
34 * {@link TrackDecoder} that decodes a video track and renders the frames onto a
35 * {@link SurfaceTexture}.
36 *
37 * This implementation uses the GPU for image operations such as copying
38 * and color-space conversion.
39 */
40@TargetApi(16)
41public class GpuVideoTrackDecoder extends VideoTrackDecoder {
42
43    /**
44     * Identity fragment shader for external textures.
45     */
46    private static final String COPY_FRAGMENT_SHADER =
47            "#extension GL_OES_EGL_image_external : require\n" +
48            "precision mediump float;\n" +
49            "uniform samplerExternalOES tex_sampler_0;\n" +
50            "varying vec2 v_texcoord;\n" +
51            "void main() {\n" +
52            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
53            "}\n";
54
55    private final TextureSource mTextureSource;
56    private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor.
57    private final float[] mTransformMatrix;
58
59    private final int mOutputWidth;
60    private final int mOutputHeight;
61
62    private ImageShader mImageShader;
63
64    private long mCurrentPresentationTimeUs;
65
66    public GpuVideoTrackDecoder(
67            int trackIndex, MediaFormat format, Listener listener) {
68        super(trackIndex, format, listener);
69
70        // Create a surface texture to be used by the video track decoder.
71        mTextureSource = TextureSource.newExternalTexture();
72        mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId());
73        mSurfaceTexture.detachFromGLContext();
74        mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() {
75            @Override
76            public void onFrameAvailable(SurfaceTexture surfaceTexture) {
77                markFrameAvailable();
78            }
79        });
80
81        mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH);
82        mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
83
84        mTransformMatrix = new float[16];
85    }
86
87    @Override
88    protected MediaCodec initMediaCodec(MediaFormat format) {
89        Surface surface = new Surface(mSurfaceTexture);
90        MediaCodec mediaCodec = MediaCodec.createDecoderByType(
91                format.getString(MediaFormat.KEY_MIME));
92        mediaCodec.configure(format, surface, null, 0);
93        surface.release();
94        return mediaCodec;
95    }
96
97    @Override
98    protected boolean onDataAvailable(
99            MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
100        boolean textureAvailable = waitForFrameGrab();
101
102        mCurrentPresentationTimeUs = info.presentationTimeUs;
103
104        // Only render the next frame if we weren't interrupted.
105        codec.releaseOutputBuffer(bufferIndex, textureAvailable);
106
107        if (textureAvailable) {
108            if (updateTexture()) {
109                notifyListener();
110            }
111        }
112
113        return false;
114    }
115
116    /**
117     * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates
118     * the internal {@link SurfaceTexture}.
119     */
120    private boolean updateTexture() {
121        // Wait for the frame we just released to appear in the texture.
122        synchronized (mFrameMonitor) {
123            try {
124                while (!mFrameAvailable) {
125                    mFrameMonitor.wait();
126                }
127                mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId());
128                mSurfaceTexture.updateTexImage();
129                mSurfaceTexture.detachFromGLContext();
130                return true;
131            } catch (InterruptedException e) {
132                return false;
133            }
134        }
135    }
136
137    @Override
138    protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
139        TextureSource targetTexture = TextureSource.newExternalTexture();
140        mSurfaceTexture.attachToGLContext(targetTexture.getTextureId());
141        mSurfaceTexture.getTransformMatrix(mTransformMatrix);
142
143        ImageShader imageShader = getImageShader();
144        imageShader.setSourceTransform(mTransformMatrix);
145
146        int outputWidth = mOutputWidth;
147        int outputHeight = mOutputHeight;
148        if (rotation != 0) {
149            float[] targetCoords = getRotationCoords(rotation);
150            imageShader.setTargetCoords(targetCoords);
151            if (needSwapDimension(rotation)) {
152                outputWidth = mOutputHeight;
153                outputHeight = mOutputWidth;
154            }
155        }
156        outputVideoFrame.resize(new int[] { outputWidth, outputHeight });
157        imageShader.process(
158                targetTexture,
159                outputVideoFrame.lockRenderTarget(),
160                outputWidth,
161                outputHeight);
162        outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
163        outputVideoFrame.unlock();
164        targetTexture.release();
165
166        mSurfaceTexture.detachFromGLContext();
167    }
168
169    @Override
170    public void release() {
171        super.release();
172        synchronized (mFrameMonitor) {
173            mTextureSource.release();
174            mSurfaceTexture.release();
175        }
176    }
177
178    /*
179     * This method has to be called on the MFF processing thread.
180     */
181    private ImageShader getImageShader() {
182        if (mImageShader == null) {
183            mImageShader = new ImageShader(COPY_FRAGMENT_SHADER);
184            mImageShader.setTargetRect(0f, 1f, 1f, -1f);
185        }
186        return mImageShader;
187    }
188
189    /**
190     * Get the quad coords for rotation.
191     * @param rotation applied to the frame, value is one of
192     *   {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}
193     * @return coords the calculated quad coords for the given rotation
194     */
195    private static float[] getRotationCoords(int rotation) {
196         switch(rotation) {
197             case MediaDecoder.ROTATE_90_RIGHT:
198                 return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f };
199             case MediaDecoder.ROTATE_180:
200                 return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f };
201             case MediaDecoder.ROTATE_90_LEFT:
202                 return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f };
203             case MediaDecoder.ROTATE_NONE:
204                 return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f };
205             default:
206                 throw new IllegalArgumentException("Unsupported rotation angle.");
207         }
208     }
209
210}
211