/* * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.media.cts; import android.annotation.TargetApi; import android.content.res.AssetFileDescriptor; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaExtractor; import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Environment; import android.test.AndroidTestCase; import android.util.Log; import android.view.Surface; import com.android.cts.media.R; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicReference; /** * Test for the integration of MediaMuxer and MediaCodec's encoder. * *
It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write * them into a file. * *
It does not currently check whether the result file is correct, but makes sure that nothing * fails along the way. * *
It also tests the way the codec config buffers need to be passed from the MediaCodec to the * MediaMuxer. */ @TargetApi(18) public class ExtractDecodeEditEncodeMuxTest extends AndroidTestCase { private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); private static final boolean VERBOSE = false; // lots of logging /** How long to wait for the next buffer to become available. */ private static final int TIMEOUT_USEC = 10000; /** Where to output the test files. */ private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); // parameters for the video encoder // H.264 Advanced Video Coding private static final String OUTPUT_VIDEO_MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC; private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames private static final int OUTPUT_VIDEO_COLOR_FORMAT = MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; // parameters for the audio encoder // Advanced Audio Coding private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; private static final int OUTPUT_AUDIO_AAC_PROFILE = MediaCodecInfo.CodecProfileLevel.AACObjectHE; private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. /** * Used for editing the frames. * *
Swaps green and blue channels by storing an RBGA color in an RGBA buffer. */ private static final String FRAGMENT_SHADER = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "varying vec2 vTextureCoord;\n" + "uniform samplerExternalOES sTexture;\n" + "void main() {\n" + " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + "}\n"; /** Whether to copy the video from the test video. */ private boolean mCopyVideo; /** Whether to copy the audio from the test video. */ private boolean mCopyAudio; /** Whether to verify the audio format. */ private boolean mVerifyAudioFormat; /** Width of the output frames. */ private int mWidth = -1; /** Height of the output frames. */ private int mHeight = -1; /** The raw resource used as the input file. */ private int mSourceResId; /** The destination file for the encoded output. */ private String mOutputFile; public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { setSize(176, 144); setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); setCopyVideo(); TestWrapper.runTest(this); } public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { setSize(320, 240); setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); setCopyVideo(); TestWrapper.runTest(this); } public void testExtractDecodeEditEncodeMux720p() throws Throwable { setSize(1280, 720); setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); setCopyVideo(); TestWrapper.runTest(this); } public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { setSize(1280, 720); setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); setCopyAudio(); setVerifyAudioFormat(); TestWrapper.runTest(this); } public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { setSize(1280, 720); setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); setCopyAudio(); setCopyVideo(); setVerifyAudioFormat(); TestWrapper.runTest(this); } /** Wraps testExtractDecodeEditEncodeMux() */ private static class TestWrapper implements Runnable { private Throwable mThrowable; private ExtractDecodeEditEncodeMuxTest mTest; private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { mTest = test; } @Override public void run() { try { mTest.extractDecodeEditEncodeMux(); } catch (Throwable th) { mThrowable = th; } } /** * Entry point. */ public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { test.setOutputFile(); TestWrapper wrapper = new TestWrapper(test); Thread th = new Thread(wrapper, "codec test"); th.start(); th.join(); if (wrapper.mThrowable != null) { throw wrapper.mThrowable; } } } /** * Sets the test to copy the video stream. */ private void setCopyVideo() { mCopyVideo = true; } /** * Sets the test to copy the video stream. */ private void setCopyAudio() { mCopyAudio = true; } /** * Sets the test to verify the output audio format. */ private void setVerifyAudioFormat() { mVerifyAudioFormat = true; } /** * Sets the desired frame size. */ private void setSize(int width, int height) { if ((width % 16) != 0 || (height % 16) != 0) { Log.w(TAG, "WARNING: width or height not multiple of 16"); } mWidth = width; mHeight = height; } /** * Sets the raw resource used as the source video. */ private void setSource(int resId) { mSourceResId = resId; } /** * Sets the name of the output file based on the other parameters. * *
Must be called after {@link #setSize(int, int)} and {@link #setSource(int)}. */ private void setOutputFile() { StringBuilder sb = new StringBuilder(); sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); sb.append("/cts-media-"); sb.append(getClass().getSimpleName()); assertTrue("should have called setSource() first", mSourceResId != -1); sb.append('-'); sb.append(mSourceResId); if (mCopyVideo) { assertTrue("should have called setSize() first", mWidth != -1); assertTrue("should have called setSize() first", mHeight != -1); sb.append('-'); sb.append("video"); sb.append('-'); sb.append(mWidth); sb.append('x'); sb.append(mHeight); } if (mCopyAudio) { sb.append('-'); sb.append("audio"); } sb.append(".mp4"); mOutputFile = sb.toString(); } /** * Tests encoding and subsequently decoding video from frames generated into a buffer. *
* We encode several frames of a video test pattern using MediaCodec, then decode the output
* with MediaCodec and do some simple checks.
*/
private void extractDecodeEditEncodeMux() throws Exception {
// Exception that may be thrown during release.
Exception exception = null;
MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
// We avoid the device-specific limitations on width and height by using values
// that are multiples of 16, which all tested devices seem to be able to handle.
MediaFormat outputVideoFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, mWidth, mHeight);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
outputVideoFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT);
outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputVideoFormat.setInteger(
MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat);
if (videoEncoderName == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat);
return;
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName);
MediaFormat outputAudioFormat =
MediaFormat.createAudioFormat(
OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ,
OUTPUT_AUDIO_CHANNEL_COUNT);
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat);
if (audioEncoderName == null) {
// Don't fail CTS if they don't have an AAC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat);
return;
}
if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName);
MediaExtractor videoExtractor = null;
MediaExtractor audioExtractor = null;
OutputSurface outputSurface = null;
MediaCodec videoDecoder = null;
MediaCodec audioDecoder = null;
MediaCodec videoEncoder = null;
MediaCodec audioEncoder = null;
MediaMuxer muxer = null;
InputSurface inputSurface = null;
try {
if (mCopyVideo) {
videoExtractor = createExtractor();
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
assertTrue("missing video track in test video", videoInputTrack != -1);
MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
AtomicReference The surface to use as input is stored in the given reference.
*
* @param codecInfo of the codec to use
* @param format of the stream to be produced
* @param surfaceReference to store the surface to use as input
*/
private MediaCodec createVideoEncoder(
String codecName,
MediaFormat format,
AtomicReference The muxer is not started as it needs to be started only after all streams have been added.
*/
private MediaMuxer createMuxer() throws IOException {
return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isAudioFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
/**
* Does the actual work for extracting, decoding, encoding and muxing.
*/
private void doExtractDecodeEditEncodeMux(
MediaExtractor videoExtractor,
MediaExtractor audioExtractor,
MediaCodec videoDecoder,
MediaCodec videoEncoder,
MediaCodec audioDecoder,
MediaCodec audioEncoder,
MediaMuxer muxer,
InputSurface inputSurface,
OutputSurface outputSurface) {
ByteBuffer[] videoDecoderInputBuffers = null;
ByteBuffer[] videoDecoderOutputBuffers = null;
ByteBuffer[] videoEncoderOutputBuffers = null;
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null;
if (mCopyVideo) {
videoDecoderInputBuffers = videoDecoder.getInputBuffers();
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
ByteBuffer[] audioDecoderInputBuffers = null;
ByteBuffer[] audioDecoderOutputBuffers = null;
ByteBuffer[] audioEncoderInputBuffers = null;
ByteBuffer[] audioEncoderOutputBuffers = null;
MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null;
if (mCopyAudio) {
audioDecoderInputBuffers = audioDecoder.getInputBuffers();
audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
audioEncoderInputBuffers = audioEncoder.getInputBuffers();
audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
// We will get these from the decoders when notified of a format change.
MediaFormat decoderOutputVideoFormat = null;
MediaFormat decoderOutputAudioFormat = null;
// We will get these from the encoders when notified of a format change.
MediaFormat encoderOutputVideoFormat = null;
MediaFormat encoderOutputAudioFormat = null;
// We will determine these once we have the output format.
int outputVideoTrack = -1;
int outputAudioTrack = -1;
// Whether things are done on the video side.
boolean videoExtractorDone = false;
boolean videoDecoderDone = false;
boolean videoEncoderDone = false;
// Whether things are done on the audio side.
boolean audioExtractorDone = false;
boolean audioDecoderDone = false;
boolean audioEncoderDone = false;
// The audio decoder output buffer to process, -1 if none.
int pendingAudioDecoderOutputBufferIndex = -1;
boolean muxing = false;
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
int videoEncodedFrameCount = 0;
int audioExtractedFrameCount = 0;
int audioDecodedFrameCount = 0;
int audioEncodedFrameCount = 0;
while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) {
if (VERBOSE) {
Log.d(TAG, String.format(
"loop: "
+ "V(%b){"
+ "extracted:%d(done:%b) "
+ "decoded:%d(done:%b) "
+ "encoded:%d(done:%b)} "
+ "A(%b){"
+ "extracted:%d(done:%b) "
+ "decoded:%d(done:%b) "
+ "encoded:%d(done:%b) "
+ "pending:%d} "
+ "muxing:%b(V:%d,A:%d)",
mCopyVideo,
videoExtractedFrameCount, videoExtractorDone,
videoDecodedFrameCount, videoDecoderDone,
videoEncodedFrameCount, videoEncoderDone,
mCopyAudio,
audioExtractedFrameCount, audioExtractorDone,
audioDecodedFrameCount, audioDecoderDone,
audioEncodedFrameCount, audioEncoderDone,
pendingAudioDecoderOutputBufferIndex,
muxing, outputVideoTrack, outputAudioTrack));
}
// Extract video from file and feed to decoder.
// Do not extract video if we have determined the output format but we are not yet
// ready to mux the frames.
while (mCopyVideo && !videoExtractorDone
&& (encoderOutputVideoFormat == null || muxing)) {
int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
}
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "video extractor: returned buffer of size " + size);
Log.d(TAG, "video extractor: returned buffer for time " + presentationTime);
}
if (size >= 0) {
videoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
videoExtractor.getSampleFlags());
}
videoExtractorDone = !videoExtractor.advance();
if (videoExtractorDone) {
if (VERBOSE) Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Extract audio from file and feed to decoder.
// Do not extract audio if we have determined the output format but we are not yet
// ready to mux the frames.
while (mCopyAudio && !audioExtractorDone
&& (encoderOutputAudioFormat == null || muxing)) {
int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex);
}
ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex];
int size = audioExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = audioExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "audio extractor: returned buffer of size " + size);
Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime);
}
if (size >= 0) {
audioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
audioExtractor.getSampleFlags());
}
audioExtractorDone = !audioExtractor.advance();
if (audioExtractorDone) {
if (VERBOSE) Log.d(TAG, "audio extractor: EOS");
audioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
audioExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Poll output frames from the video decoder and feed the encoder.
while (mCopyVideo && !videoDecoderDone
&& (encoderOutputVideoFormat == null || muxing)) {
int decoderOutputBufferIndex =
videoDecoder.dequeueOutputBuffer(
videoDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
decoderOutputVideoFormat = videoDecoder.getOutputFormat();
if (VERBOSE) {
Log.d(TAG, "video decoder: output format changed: "
+ decoderOutputVideoFormat);
}
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned output buffer: "
+ decoderOutputBufferIndex);
Log.d(TAG, "video decoder: returned buffer of size "
+ videoDecoderOutputBufferInfo.size);
}
ByteBuffer decoderOutputBuffer =
videoDecoderOutputBuffers[decoderOutputBufferIndex];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned buffer for time "
+ videoDecoderOutputBufferInfo.presentationTimeUs);
}
boolean render = videoDecoderOutputBufferInfo.size != 0;
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
if (render) {
if (VERBOSE) Log.d(TAG, "output surface: await new image");
outputSurface.awaitNewImage();
// Edit the frame and send it to the encoder.
if (VERBOSE) Log.d(TAG, "output surface: draw image");
outputSurface.drawImage();
inputSurface.setPresentationTime(
videoDecoderOutputBufferInfo.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
inputSurface.swapBuffers();
if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
}
if ((videoDecoderOutputBufferInfo.flags
& MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: EOS");
videoDecoderDone = true;
videoEncoder.signalEndOfInputStream();
}
videoDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Poll output frames from the audio decoder.
// Do not poll if we already have a pending buffer to feed to the encoder.
while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1
&& (encoderOutputAudioFormat == null || muxing)) {
int decoderOutputBufferIndex =
audioDecoder.dequeueOutputBuffer(
audioDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed");
audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
decoderOutputAudioFormat = audioDecoder.getOutputFormat();
if (VERBOSE) {
Log.d(TAG, "audio decoder: output format changed: "
+ decoderOutputAudioFormat);
}
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned output buffer: "
+ decoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer of size "
+ audioDecoderOutputBufferInfo.size);
}
ByteBuffer decoderOutputBuffer =
audioDecoderOutputBuffers[decoderOutputBufferIndex];
if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer");
audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer for time "
+ audioDecoderOutputBufferInfo.presentationTimeUs);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: output buffer is now pending: "
+ pendingAudioDecoderOutputBufferIndex);
}
pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
audioDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Feed the pending decoded audio buffer to the audio encoder.
while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) {
if (VERBOSE) {
Log.d(TAG, "audio decoder: attempting to process pending buffer: "
+ pendingAudioDecoderOutputBufferIndex);
}
int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex);
}
ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex];
int size = audioDecoderOutputBufferInfo.size;
long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs;
if (VERBOSE) {
Log.d(TAG, "audio decoder: processing pending buffer: "
+ pendingAudioDecoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: pending buffer of size " + size);
Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime);
}
if (size >= 0) {
ByteBuffer decoderOutputBuffer =
audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex]
.duplicate();
decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset);
decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size);
encoderInputBuffer.position(0);
encoderInputBuffer.put(decoderOutputBuffer);
audioEncoder.queueInputBuffer(
encoderInputBufferIndex,
0,
size,
presentationTime,
audioDecoderOutputBufferInfo.flags);
}
audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false);
pendingAudioDecoderOutputBufferIndex = -1;
if ((audioDecoderOutputBufferInfo.flags
& MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: EOS");
audioDecoderDone = true;
}
// We enqueued a pending frame, let's try something else next.
break;
}
// Poll frames from the video encoder and send them to the muxer.
while (mCopyVideo && !videoEncoderDone
&& (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer(
videoEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
if (outputVideoTrack >= 0) {
fail("video encoder changed its output format again?");
}
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
break;
}
assertTrue("should have added track before processing output", muxing);
if (VERBOSE) {
Log.d(TAG, "video encoder: returned output buffer: "
+ encoderOutputBufferIndex);
Log.d(TAG, "video encoder: returned buffer of size "
+ videoEncoderOutputBufferInfo.size);
}
ByteBuffer encoderOutputBuffer =
videoEncoderOutputBuffers[encoderOutputBufferIndex];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
// Simply ignore codec config buffers.
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video encoder: returned buffer for time "
+ videoEncoderOutputBufferInfo.presentationTimeUs);
}
if (videoEncoderOutputBufferInfo.size != 0) {
muxer.writeSampleData(
outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
if (VERBOSE) Log.d(TAG, "video encoder: EOS");
videoEncoderDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
videoEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
// Poll frames from the audio encoder and send them to the muxer.
while (mCopyAudio && !audioEncoderDone
&& (encoderOutputAudioFormat == null || muxing)) {
int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer(
audioEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed");
audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output format changed");
if (outputAudioTrack >= 0) {
fail("audio encoder changed its output format again?");
}
encoderOutputAudioFormat = audioEncoder.getOutputFormat();
break;
}
assertTrue("should have added track before processing output", muxing);
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned output buffer: "
+ encoderOutputBufferIndex);
Log.d(TAG, "audio encoder: returned buffer of size "
+ audioEncoderOutputBufferInfo.size);
}
ByteBuffer encoderOutputBuffer =
audioEncoderOutputBuffers[encoderOutputBufferIndex];
if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer");
// Simply ignore codec config buffers.
audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned buffer for time "
+ audioEncoderOutputBufferInfo.presentationTimeUs);
}
if (audioEncoderOutputBufferInfo.size != 0) {
muxer.writeSampleData(
outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo);
}
if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: EOS");
audioEncoderDone = true;
}
audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
audioEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
if (!muxing
&& (!mCopyAudio || encoderOutputAudioFormat != null)
&& (!mCopyVideo || encoderOutputVideoFormat != null)) {
if (mCopyVideo) {
Log.d(TAG, "muxer: adding video track.");
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
}
if (mCopyAudio) {
Log.d(TAG, "muxer: adding audio track.");
outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat);
}
Log.d(TAG, "muxer: starting");
muxer.start();
muxing = true;
}
}
// Basic sanity checks.
if (mCopyVideo) {
assertEquals("encoded and decoded video frame counts should match",
videoDecodedFrameCount, videoEncodedFrameCount);
assertTrue("decoded frame count should be less than extracted frame count",
videoDecodedFrameCount <= videoExtractedFrameCount);
}
if (mCopyAudio) {
assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex);
}
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
private static boolean isAudioFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("audio/");
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
}