1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package androidx.media.filterfw.decoder; 18 19import android.annotation.TargetApi; 20import android.graphics.SurfaceTexture; 21import android.graphics.SurfaceTexture.OnFrameAvailableListener; 22import android.media.MediaCodec; 23import android.media.MediaCodec.BufferInfo; 24import android.media.MediaFormat; 25import android.view.Surface; 26 27import androidx.media.filterfw.FrameImage2D; 28import androidx.media.filterfw.ImageShader; 29import androidx.media.filterfw.TextureSource; 30 31import java.nio.ByteBuffer; 32 33/** 34 * {@link TrackDecoder} that decodes a video track and renders the frames onto a 35 * {@link SurfaceTexture}. 36 * 37 * This implementation uses the GPU for image operations such as copying 38 * and color-space conversion. 39 */ 40@TargetApi(16) 41public class GpuVideoTrackDecoder extends VideoTrackDecoder { 42 43 /** 44 * Identity fragment shader for external textures. 45 */ 46 private static final String COPY_FRAGMENT_SHADER = 47 "#extension GL_OES_EGL_image_external : require\n" + 48 "precision mediump float;\n" + 49 "uniform samplerExternalOES tex_sampler_0;\n" + 50 "varying vec2 v_texcoord;\n" + 51 "void main() {\n" + 52 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 53 "}\n"; 54 55 private final TextureSource mTextureSource; 56 private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor. 57 private final float[] mTransformMatrix; 58 59 private final int mOutputWidth; 60 private final int mOutputHeight; 61 62 private ImageShader mImageShader; 63 64 private long mCurrentPresentationTimeUs; 65 66 public GpuVideoTrackDecoder( 67 int trackIndex, MediaFormat format, Listener listener) { 68 super(trackIndex, format, listener); 69 70 // Create a surface texture to be used by the video track decoder. 71 mTextureSource = TextureSource.newExternalTexture(); 72 mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId()); 73 mSurfaceTexture.detachFromGLContext(); 74 mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() { 75 @Override 76 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 77 markFrameAvailable(); 78 } 79 }); 80 81 mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH); 82 mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 83 84 mTransformMatrix = new float[16]; 85 } 86 87 @Override 88 protected MediaCodec initMediaCodec(MediaFormat format) { 89 Surface surface = new Surface(mSurfaceTexture); 90 MediaCodec mediaCodec = MediaCodec.createDecoderByType( 91 format.getString(MediaFormat.KEY_MIME)); 92 mediaCodec.configure(format, surface, null, 0); 93 surface.release(); 94 return mediaCodec; 95 } 96 97 @Override 98 protected boolean onDataAvailable( 99 MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) { 100 boolean textureAvailable = waitForFrameGrab(); 101 102 mCurrentPresentationTimeUs = info.presentationTimeUs; 103 104 // Only render the next frame if we weren't interrupted. 105 codec.releaseOutputBuffer(bufferIndex, textureAvailable); 106 107 if (textureAvailable) { 108 if (updateTexture()) { 109 notifyListener(); 110 } 111 } 112 113 return false; 114 } 115 116 /** 117 * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates 118 * the internal {@link SurfaceTexture}. 119 */ 120 private boolean updateTexture() { 121 // Wait for the frame we just released to appear in the texture. 122 synchronized (mFrameMonitor) { 123 try { 124 while (!mFrameAvailable) { 125 mFrameMonitor.wait(); 126 } 127 mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId()); 128 mSurfaceTexture.updateTexImage(); 129 mSurfaceTexture.detachFromGLContext(); 130 return true; 131 } catch (InterruptedException e) { 132 return false; 133 } 134 } 135 } 136 137 @Override 138 protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) { 139 TextureSource targetTexture = TextureSource.newExternalTexture(); 140 mSurfaceTexture.attachToGLContext(targetTexture.getTextureId()); 141 mSurfaceTexture.getTransformMatrix(mTransformMatrix); 142 143 ImageShader imageShader = getImageShader(); 144 imageShader.setSourceTransform(mTransformMatrix); 145 146 int outputWidth = mOutputWidth; 147 int outputHeight = mOutputHeight; 148 if (rotation != 0) { 149 float[] targetCoords = getRotationCoords(rotation); 150 imageShader.setTargetCoords(targetCoords); 151 if (needSwapDimension(rotation)) { 152 outputWidth = mOutputHeight; 153 outputHeight = mOutputWidth; 154 } 155 } 156 outputVideoFrame.resize(new int[] { outputWidth, outputHeight }); 157 imageShader.process( 158 targetTexture, 159 outputVideoFrame.lockRenderTarget(), 160 outputWidth, 161 outputHeight); 162 outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000); 163 outputVideoFrame.unlock(); 164 targetTexture.release(); 165 166 mSurfaceTexture.detachFromGLContext(); 167 } 168 169 @Override 170 public void release() { 171 super.release(); 172 synchronized (mFrameMonitor) { 173 mTextureSource.release(); 174 mSurfaceTexture.release(); 175 } 176 } 177 178 /* 179 * This method has to be called on the MFF processing thread. 180 */ 181 private ImageShader getImageShader() { 182 if (mImageShader == null) { 183 mImageShader = new ImageShader(COPY_FRAGMENT_SHADER); 184 mImageShader.setTargetRect(0f, 1f, 1f, -1f); 185 } 186 return mImageShader; 187 } 188 189 /** 190 * Get the quad coords for rotation. 191 * @param rotation applied to the frame, value is one of 192 * {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT} 193 * @return coords the calculated quad coords for the given rotation 194 */ 195 private static float[] getRotationCoords(int rotation) { 196 switch(rotation) { 197 case MediaDecoder.ROTATE_90_RIGHT: 198 return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f }; 199 case MediaDecoder.ROTATE_180: 200 return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f }; 201 case MediaDecoder.ROTATE_90_LEFT: 202 return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f }; 203 case MediaDecoder.ROTATE_NONE: 204 return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f }; 205 default: 206 throw new IllegalArgumentException("Unsupported rotation angle."); 207 } 208 } 209 210} 211