1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package androidx.media.filterfw.decoder; 18 19import android.annotation.TargetApi; 20import android.graphics.SurfaceTexture; 21import android.graphics.SurfaceTexture.OnFrameAvailableListener; 22import android.media.MediaCodec; 23import android.media.MediaCodec.BufferInfo; 24import android.media.MediaFormat; 25import android.view.Surface; 26 27import androidx.media.filterfw.FrameImage2D; 28import androidx.media.filterfw.ImageShader; 29import androidx.media.filterfw.TextureSource; 30 31import java.io.IOException; 32import java.nio.ByteBuffer; 33 34/** 35 * {@link TrackDecoder} that decodes a video track and renders the frames onto a 36 * {@link SurfaceTexture}. 37 * 38 * This implementation uses the GPU for image operations such as copying 39 * and color-space conversion. 40 */ 41@TargetApi(16) 42public class GpuVideoTrackDecoder extends VideoTrackDecoder { 43 44 /** 45 * Identity fragment shader for external textures. 46 */ 47 private static final String COPY_FRAGMENT_SHADER = 48 "#extension GL_OES_EGL_image_external : require\n" + 49 "precision mediump float;\n" + 50 "uniform samplerExternalOES tex_sampler_0;\n" + 51 "varying vec2 v_texcoord;\n" + 52 "void main() {\n" + 53 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 54 "}\n"; 55 56 private final TextureSource mTextureSource; 57 private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor. 58 private final float[] mTransformMatrix; 59 60 private final int mOutputWidth; 61 private final int mOutputHeight; 62 63 private ImageShader mImageShader; 64 65 private long mCurrentPresentationTimeUs; 66 67 public GpuVideoTrackDecoder( 68 int trackIndex, MediaFormat format, Listener listener) { 69 super(trackIndex, format, listener); 70 71 // Create a surface texture to be used by the video track decoder. 72 mTextureSource = TextureSource.newExternalTexture(); 73 mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId()); 74 mSurfaceTexture.detachFromGLContext(); 75 mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() { 76 @Override 77 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 78 markFrameAvailable(); 79 } 80 }); 81 82 mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH); 83 mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 84 85 mTransformMatrix = new float[16]; 86 } 87 88 @Override 89 protected MediaCodec initMediaCodec(MediaFormat format) { 90 MediaCodec mediaCodec; 91 try { 92 mediaCodec = MediaCodec.createDecoderByType( 93 format.getString(MediaFormat.KEY_MIME)); 94 } catch (IOException e) { 95 throw new RuntimeException( 96 "failed to create decoder for " 97 + format.getString(MediaFormat.KEY_MIME), e); 98 } 99 Surface surface = new Surface(mSurfaceTexture); 100 mediaCodec.configure(format, surface, null, 0); 101 surface.release(); 102 return mediaCodec; 103 } 104 105 @Override 106 protected boolean onDataAvailable( 107 MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) { 108 boolean textureAvailable = waitForFrameGrab(); 109 110 mCurrentPresentationTimeUs = info.presentationTimeUs; 111 112 // Only render the next frame if we weren't interrupted. 113 codec.releaseOutputBuffer(bufferIndex, textureAvailable); 114 115 if (textureAvailable) { 116 if (updateTexture()) { 117 notifyListener(); 118 } 119 } 120 121 return false; 122 } 123 124 /** 125 * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates 126 * the internal {@link SurfaceTexture}. 127 */ 128 private boolean updateTexture() { 129 // Wait for the frame we just released to appear in the texture. 130 synchronized (mFrameMonitor) { 131 try { 132 while (!mFrameAvailable) { 133 mFrameMonitor.wait(); 134 } 135 mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId()); 136 mSurfaceTexture.updateTexImage(); 137 mSurfaceTexture.detachFromGLContext(); 138 return true; 139 } catch (InterruptedException e) { 140 return false; 141 } 142 } 143 } 144 145 @Override 146 protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) { 147 TextureSource targetTexture = TextureSource.newExternalTexture(); 148 mSurfaceTexture.attachToGLContext(targetTexture.getTextureId()); 149 mSurfaceTexture.getTransformMatrix(mTransformMatrix); 150 151 ImageShader imageShader = getImageShader(); 152 imageShader.setSourceTransform(mTransformMatrix); 153 154 int outputWidth = mOutputWidth; 155 int outputHeight = mOutputHeight; 156 if (rotation != 0) { 157 float[] targetCoords = getRotationCoords(rotation); 158 imageShader.setTargetCoords(targetCoords); 159 if (needSwapDimension(rotation)) { 160 outputWidth = mOutputHeight; 161 outputHeight = mOutputWidth; 162 } 163 } 164 outputVideoFrame.resize(new int[] { outputWidth, outputHeight }); 165 imageShader.process( 166 targetTexture, 167 outputVideoFrame.lockRenderTarget(), 168 outputWidth, 169 outputHeight); 170 outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000); 171 outputVideoFrame.unlock(); 172 targetTexture.release(); 173 174 mSurfaceTexture.detachFromGLContext(); 175 } 176 177 @Override 178 public void release() { 179 super.release(); 180 synchronized (mFrameMonitor) { 181 mTextureSource.release(); 182 mSurfaceTexture.release(); 183 } 184 } 185 186 /* 187 * This method has to be called on the MFF processing thread. 188 */ 189 private ImageShader getImageShader() { 190 if (mImageShader == null) { 191 mImageShader = new ImageShader(COPY_FRAGMENT_SHADER); 192 mImageShader.setTargetRect(0f, 1f, 1f, -1f); 193 } 194 return mImageShader; 195 } 196 197 /** 198 * Get the quad coords for rotation. 199 * @param rotation applied to the frame, value is one of 200 * {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT} 201 * @return coords the calculated quad coords for the given rotation 202 */ 203 private static float[] getRotationCoords(int rotation) { 204 switch(rotation) { 205 case MediaDecoder.ROTATE_90_RIGHT: 206 return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f }; 207 case MediaDecoder.ROTATE_180: 208 return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f }; 209 case MediaDecoder.ROTATE_90_LEFT: 210 return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f }; 211 case MediaDecoder.ROTATE_NONE: 212 return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f }; 213 default: 214 throw new IllegalArgumentException("Unsupported rotation angle."); 215 } 216 } 217 218} 219