1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package androidx.media.filterfw.decoder;
17
18import android.annotation.TargetApi;
19import android.graphics.SurfaceTexture;
20import android.media.MediaCodec;
21import android.media.MediaCodec.BufferInfo;
22import android.media.MediaCodecInfo;
23import android.media.MediaCodecInfo.CodecCapabilities;
24import android.media.MediaCodecList;
25import android.media.MediaFormat;
26import android.util.SparseIntArray;
27import androidx.media.filterfw.ColorSpace;
28import androidx.media.filterfw.Frame;
29import androidx.media.filterfw.FrameImage2D;
30import androidx.media.filterfw.PixelUtils;
31
32import java.io.IOException;
33import java.nio.ByteBuffer;
34import java.util.Arrays;
35import java.util.HashSet;
36import java.util.Set;
37import java.util.TreeMap;
38
39/**
40 * {@link TrackDecoder} that decodes a video track and renders the frames onto a
41 * {@link SurfaceTexture}.
42 *
43 * This implementation purely uses CPU based methods to decode and color-convert the frames.
44 */
45@TargetApi(16)
46public class CpuVideoTrackDecoder extends VideoTrackDecoder {
47
48    private static final int COLOR_FORMAT_UNSET = -1;
49
50    private final int mWidth;
51    private final int mHeight;
52
53    private int mColorFormat = COLOR_FORMAT_UNSET;
54    private long mCurrentPresentationTimeUs;
55    private ByteBuffer mDecodedBuffer;
56    private ByteBuffer mUnrotatedBytes;
57
58    protected CpuVideoTrackDecoder(int trackIndex, MediaFormat format, Listener listener) {
59        super(trackIndex, format, listener);
60
61        mWidth = format.getInteger(MediaFormat.KEY_WIDTH);
62        mHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
63    }
64
65    @Override
66    protected MediaCodec initMediaCodec(MediaFormat format) {
67        // Find a codec for our video that can output to one of our supported color-spaces
68        MediaCodec mediaCodec = findDecoderCodec(format, new int[] {
69                CodecCapabilities.COLOR_Format32bitARGB8888,
70                CodecCapabilities.COLOR_FormatYUV420Planar});
71        if (mediaCodec == null) {
72            throw new RuntimeException(
73                    "Could not find a suitable decoder for format: " + format + "!");
74        }
75        mediaCodec.configure(format, null, null, 0);
76        return mediaCodec;
77    }
78
79    @Override
80    protected boolean onDataAvailable(
81            MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
82
83        mCurrentPresentationTimeUs = info.presentationTimeUs;
84        mDecodedBuffer = buffers[bufferIndex];
85
86        if (mColorFormat == -1) {
87            mColorFormat = codec.getOutputFormat().getInteger(MediaFormat.KEY_COLOR_FORMAT);
88        }
89
90        markFrameAvailable();
91        notifyListener();
92
93        // Wait for the grab before we release this buffer.
94        waitForFrameGrab();
95
96        codec.releaseOutputBuffer(bufferIndex, false);
97
98        return false;
99    }
100
101    @Override
102    protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
103        // Calculate output dimensions
104        int outputWidth = mWidth;
105        int outputHeight = mHeight;
106        if (needSwapDimension(rotation)) {
107            outputWidth = mHeight;
108            outputHeight = mWidth;
109        }
110
111        // Create output frame
112        outputVideoFrame.resize(new int[] {outputWidth, outputHeight});
113        outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
114        ByteBuffer outBytes = outputVideoFrame.lockBytes(Frame.MODE_WRITE);
115
116        // Set data
117        if (rotation == MediaDecoder.ROTATE_NONE) {
118            convertImage(mDecodedBuffer, outBytes, mColorFormat, mWidth, mHeight);
119        } else {
120            if (mUnrotatedBytes == null) {
121                mUnrotatedBytes = ByteBuffer.allocateDirect(mWidth * mHeight * 4);
122            }
123            // TODO: This could be optimized by including the rotation in the color conversion.
124            convertImage(mDecodedBuffer, mUnrotatedBytes, mColorFormat, mWidth, mHeight);
125            copyRotate(mUnrotatedBytes, outBytes, rotation);
126        }
127        outputVideoFrame.unlock();
128    }
129
130    /**
131     * Copy the input data to the output data applying the specified rotation.
132     *
133     * @param input The input image data
134     * @param output Buffer for the output image data
135     * @param rotation The rotation to apply
136     */
137    private void copyRotate(ByteBuffer input, ByteBuffer output, int rotation) {
138        int offset;
139        int pixStride;
140        int rowStride;
141        switch (rotation) {
142            case MediaDecoder.ROTATE_NONE:
143                offset = 0;
144                pixStride = 1;
145                rowStride = mWidth;
146                break;
147            case MediaDecoder.ROTATE_90_LEFT:
148                offset = (mWidth - 1) * mHeight;
149                pixStride = -mHeight;
150                rowStride = 1;
151                break;
152            case MediaDecoder.ROTATE_90_RIGHT:
153                offset = mHeight - 1;
154                pixStride = mHeight;
155                rowStride = -1;
156                break;
157            case MediaDecoder.ROTATE_180:
158                offset = mWidth * mHeight - 1;
159                pixStride = -1;
160                rowStride = -mWidth;
161                break;
162            default:
163                throw new IllegalArgumentException("Unsupported rotation " + rotation + "!");
164        }
165        PixelUtils.copyPixels(input, output, mWidth, mHeight, offset, pixStride, rowStride);
166    }
167
168    /**
169     * Looks for a codec with the specified requirements.
170     *
171     * The set of codecs will be filtered down to those that meet the following requirements:
172     * <ol>
173     *   <li>The codec is a decoder.</li>
174     *   <li>The codec can decode a video of the specified format.</li>
175     *   <li>The codec can decode to one of the specified color formats.</li>
176     * </ol>
177     * If multiple codecs are found, the one with the preferred color-format is taken. Color format
178     * preference is determined by the order of their appearance in the color format array.
179     *
180     * @param format The format the codec must decode.
181     * @param requiredColorFormats Array of target color spaces ordered by preference.
182     * @return A codec that meets the requirements, or null if no such codec was found.
183     */
184    private static MediaCodec findDecoderCodec(MediaFormat format, int[] requiredColorFormats) {
185        TreeMap<Integer, String> candidateCodecs = new TreeMap<Integer, String>();
186        SparseIntArray colorPriorities = intArrayToPriorityMap(requiredColorFormats);
187        for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) {
188            // Get next codec
189            MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
190
191            // Check that this is a decoder
192            if (info.isEncoder()) {
193                continue;
194            }
195
196            // Check if this codec can decode the video in question
197            String requiredType = format.getString(MediaFormat.KEY_MIME);
198            String[] supportedTypes = info.getSupportedTypes();
199            Set<String> typeSet = new HashSet<String>(Arrays.asList(supportedTypes));
200
201            // Check if it can decode to one of the required color formats
202            if (typeSet.contains(requiredType)) {
203                CodecCapabilities capabilities = info.getCapabilitiesForType(requiredType);
204                for (int supportedColorFormat : capabilities.colorFormats) {
205                    if (colorPriorities.indexOfKey(supportedColorFormat) >= 0) {
206                        int priority = colorPriorities.get(supportedColorFormat);
207                        candidateCodecs.put(priority, info.getName());
208                    }
209                }
210            }
211        }
212
213        // Pick the best codec (with the highest color priority)
214        if (candidateCodecs.isEmpty()) {
215            return null;
216        } else {
217            String bestCodec = candidateCodecs.firstEntry().getValue();
218            try {
219                return MediaCodec.createByCodecName(bestCodec);
220            } catch (IOException e) {
221                throw new RuntimeException(
222                        "failed to create codec for "
223                        + bestCodec, e);
224            }
225        }
226    }
227
228    private static SparseIntArray intArrayToPriorityMap(int[] values) {
229        SparseIntArray result = new SparseIntArray();
230        for (int priority = 0; priority < values.length; ++priority) {
231            result.append(values[priority], priority);
232        }
233        return result;
234    }
235
236    private static void convertImage(
237            ByteBuffer input, ByteBuffer output, int colorFormat, int width, int height) {
238        switch (colorFormat) {
239            case CodecCapabilities.COLOR_Format32bitARGB8888:
240                ColorSpace.convertArgb8888ToRgba8888(input, output, width, height);
241                break;
242            case CodecCapabilities.COLOR_FormatYUV420Planar:
243                ColorSpace.convertYuv420pToRgba8888(input, output, width, height);
244                break;
245            default:
246                throw new RuntimeException("Unsupported color format: " + colorFormat + "!");
247        }
248    }
249
250}
251