/*
* Copyright 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media.filterfw.decoder;
import android.annotation.TargetApi;
import android.graphics.SurfaceTexture;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.util.SparseIntArray;
import androidx.media.filterfw.ColorSpace;
import androidx.media.filterfw.Frame;
import androidx.media.filterfw.FrameImage2D;
import androidx.media.filterfw.PixelUtils;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeMap;
/**
* {@link TrackDecoder} that decodes a video track and renders the frames onto a
* {@link SurfaceTexture}.
*
* This implementation purely uses CPU based methods to decode and color-convert the frames.
*/
@TargetApi(16)
public class CpuVideoTrackDecoder extends VideoTrackDecoder {
private static final int COLOR_FORMAT_UNSET = -1;
private final int mWidth;
private final int mHeight;
private int mColorFormat = COLOR_FORMAT_UNSET;
private long mCurrentPresentationTimeUs;
private ByteBuffer mDecodedBuffer;
private ByteBuffer mUnrotatedBytes;
protected CpuVideoTrackDecoder(int trackIndex, MediaFormat format, Listener listener) {
super(trackIndex, format, listener);
mWidth = format.getInteger(MediaFormat.KEY_WIDTH);
mHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
}
@Override
protected MediaCodec initMediaCodec(MediaFormat format) {
// Find a codec for our video that can output to one of our supported color-spaces
MediaCodec mediaCodec = findDecoderCodec(format, new int[] {
CodecCapabilities.COLOR_Format32bitARGB8888,
CodecCapabilities.COLOR_FormatYUV420Planar});
if (mediaCodec == null) {
throw new RuntimeException(
"Could not find a suitable decoder for format: " + format + "!");
}
mediaCodec.configure(format, null, null, 0);
return mediaCodec;
}
@Override
protected boolean onDataAvailable(
MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
mCurrentPresentationTimeUs = info.presentationTimeUs;
mDecodedBuffer = buffers[bufferIndex];
if (mColorFormat == -1) {
mColorFormat = codec.getOutputFormat().getInteger(MediaFormat.KEY_COLOR_FORMAT);
}
markFrameAvailable();
notifyListener();
// Wait for the grab before we release this buffer.
waitForFrameGrab();
codec.releaseOutputBuffer(bufferIndex, false);
return false;
}
@Override
protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
// Calculate output dimensions
int outputWidth = mWidth;
int outputHeight = mHeight;
if (needSwapDimension(rotation)) {
outputWidth = mHeight;
outputHeight = mWidth;
}
// Create output frame
outputVideoFrame.resize(new int[] {outputWidth, outputHeight});
outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
ByteBuffer outBytes = outputVideoFrame.lockBytes(Frame.MODE_WRITE);
// Set data
if (rotation == MediaDecoder.ROTATE_NONE) {
convertImage(mDecodedBuffer, outBytes, mColorFormat, mWidth, mHeight);
} else {
if (mUnrotatedBytes == null) {
mUnrotatedBytes = ByteBuffer.allocateDirect(mWidth * mHeight * 4);
}
// TODO: This could be optimized by including the rotation in the color conversion.
convertImage(mDecodedBuffer, mUnrotatedBytes, mColorFormat, mWidth, mHeight);
copyRotate(mUnrotatedBytes, outBytes, rotation);
}
outputVideoFrame.unlock();
}
/**
* Copy the input data to the output data applying the specified rotation.
*
* @param input The input image data
* @param output Buffer for the output image data
* @param rotation The rotation to apply
*/
private void copyRotate(ByteBuffer input, ByteBuffer output, int rotation) {
int offset;
int pixStride;
int rowStride;
switch (rotation) {
case MediaDecoder.ROTATE_NONE:
offset = 0;
pixStride = 1;
rowStride = mWidth;
break;
case MediaDecoder.ROTATE_90_LEFT:
offset = (mWidth - 1) * mHeight;
pixStride = -mHeight;
rowStride = 1;
break;
case MediaDecoder.ROTATE_90_RIGHT:
offset = mHeight - 1;
pixStride = mHeight;
rowStride = -1;
break;
case MediaDecoder.ROTATE_180:
offset = mWidth * mHeight - 1;
pixStride = -1;
rowStride = -mWidth;
break;
default:
throw new IllegalArgumentException("Unsupported rotation " + rotation + "!");
}
PixelUtils.copyPixels(input, output, mWidth, mHeight, offset, pixStride, rowStride);
}
/**
* Looks for a codec with the specified requirements.
*
* The set of codecs will be filtered down to those that meet the following requirements:
*
* - The codec is a decoder.
* - The codec can decode a video of the specified format.
* - The codec can decode to one of the specified color formats.
*
* If multiple codecs are found, the one with the preferred color-format is taken. Color format
* preference is determined by the order of their appearance in the color format array.
*
* @param format The format the codec must decode.
* @param requiredColorFormats Array of target color spaces ordered by preference.
* @return A codec that meets the requirements, or null if no such codec was found.
*/
private static MediaCodec findDecoderCodec(MediaFormat format, int[] requiredColorFormats) {
TreeMap candidateCodecs = new TreeMap();
SparseIntArray colorPriorities = intArrayToPriorityMap(requiredColorFormats);
for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) {
// Get next codec
MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
// Check that this is a decoder
if (info.isEncoder()) {
continue;
}
// Check if this codec can decode the video in question
String requiredType = format.getString(MediaFormat.KEY_MIME);
String[] supportedTypes = info.getSupportedTypes();
Set typeSet = new HashSet(Arrays.asList(supportedTypes));
// Check if it can decode to one of the required color formats
if (typeSet.contains(requiredType)) {
CodecCapabilities capabilities = info.getCapabilitiesForType(requiredType);
for (int supportedColorFormat : capabilities.colorFormats) {
if (colorPriorities.indexOfKey(supportedColorFormat) >= 0) {
int priority = colorPriorities.get(supportedColorFormat);
candidateCodecs.put(priority, info.getName());
}
}
}
}
// Pick the best codec (with the highest color priority)
if (candidateCodecs.isEmpty()) {
return null;
} else {
String bestCodec = candidateCodecs.firstEntry().getValue();
try {
return MediaCodec.createByCodecName(bestCodec);
} catch (IOException e) {
throw new RuntimeException(
"failed to create codec for "
+ bestCodec, e);
}
}
}
private static SparseIntArray intArrayToPriorityMap(int[] values) {
SparseIntArray result = new SparseIntArray();
for (int priority = 0; priority < values.length; ++priority) {
result.append(values[priority], priority);
}
return result;
}
private static void convertImage(
ByteBuffer input, ByteBuffer output, int colorFormat, int width, int height) {
switch (colorFormat) {
case CodecCapabilities.COLOR_Format32bitARGB8888:
ColorSpace.convertArgb8888ToRgba8888(input, output, width, height);
break;
case CodecCapabilities.COLOR_FormatYUV420Planar:
ColorSpace.convertYuv420pToRgba8888(input, output, width, height);
break;
default:
throw new RuntimeException("Unsupported color format: " + colorFormat + "!");
}
}
}