1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.media.cts;
18
19import android.annotation.TargetApi;
20import android.content.res.AssetFileDescriptor;
21import android.media.MediaCodec;
22import android.media.MediaCodecInfo;
23import android.media.MediaCodecList;
24import android.media.MediaExtractor;
25import android.media.MediaFormat;
26import android.media.MediaMuxer;
27import android.os.Environment;
28import android.test.AndroidTestCase;
29import android.util.Log;
30import android.view.Surface;
31
32import com.android.cts.media.R;
33
34import java.io.File;
35import java.io.IOException;
36import java.nio.ByteBuffer;
37import java.util.concurrent.atomic.AtomicReference;
38
39/**
40 * Test for the integration of MediaMuxer and MediaCodec's encoder.
41 *
42 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a
43 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write
44 * them into a file.
45 *
46 * <p>It does not currently check whether the result file is correct, but makes sure that nothing
47 * fails along the way.
48 *
49 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the
50 * MediaMuxer.
51 */
52@TargetApi(18)
53public class ExtractDecodeEditEncodeMuxTest extends AndroidTestCase {
54
55    private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName();
56    private static final boolean VERBOSE = false; // lots of logging
57
58    /** How long to wait for the next buffer to become available. */
59    private static final int TIMEOUT_USEC = 10000;
60
61    /** Where to output the test files. */
62    private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory();
63
64    // parameters for the video encoder
65                                                                // H.264 Advanced Video Coding
66    private static final String OUTPUT_VIDEO_MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
67    private static final int OUTPUT_VIDEO_BIT_RATE = 2000000;   // 2Mbps
68    private static final int OUTPUT_VIDEO_FRAME_RATE = 15;      // 15fps
69    private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames
70    private static final int OUTPUT_VIDEO_COLOR_FORMAT =
71            MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface;
72
73    // parameters for the audio encoder
74                                                                // Advanced Audio Coding
75    private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC;
76    private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2;    // Must match the input stream.
77    private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024;
78    private static final int OUTPUT_AUDIO_AAC_PROFILE =
79            MediaCodecInfo.CodecProfileLevel.AACObjectHE;
80    private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream.
81
82    /**
83     * Used for editing the frames.
84     *
85     * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer.
86     */
87    private static final String FRAGMENT_SHADER =
88            "#extension GL_OES_EGL_image_external : require\n" +
89            "precision mediump float;\n" +
90            "varying vec2 vTextureCoord;\n" +
91            "uniform samplerExternalOES sTexture;\n" +
92            "void main() {\n" +
93            "  gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" +
94            "}\n";
95
96    /** Whether to copy the video from the test video. */
97    private boolean mCopyVideo;
98    /** Whether to copy the audio from the test video. */
99    private boolean mCopyAudio;
100    /** Whether to verify the audio format. */
101    private boolean mVerifyAudioFormat;
102    /** Width of the output frames. */
103    private int mWidth = -1;
104    /** Height of the output frames. */
105    private int mHeight = -1;
106
107    /** The raw resource used as the input file. */
108    private int mSourceResId;
109
110    /** The destination file for the encoded output. */
111    private String mOutputFile;
112
113    public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable {
114        setSize(176, 144);
115        setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz);
116        setCopyVideo();
117        TestWrapper.runTest(this);
118    }
119
120    public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable {
121        setSize(320, 240);
122        setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz);
123        setCopyVideo();
124        TestWrapper.runTest(this);
125    }
126
127    public void testExtractDecodeEditEncodeMux720p() throws Throwable {
128        setSize(1280, 720);
129        setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz);
130        setCopyVideo();
131        TestWrapper.runTest(this);
132    }
133
134    public void testExtractDecodeEditEncodeMuxAudio() throws Throwable {
135        setSize(1280, 720);
136        setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz);
137        setCopyAudio();
138        setVerifyAudioFormat();
139        TestWrapper.runTest(this);
140    }
141
142    public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable {
143        setSize(1280, 720);
144        setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz);
145        setCopyAudio();
146        setCopyVideo();
147        setVerifyAudioFormat();
148        TestWrapper.runTest(this);
149    }
150
151    /** Wraps testExtractDecodeEditEncodeMux() */
152    private static class TestWrapper implements Runnable {
153        private Throwable mThrowable;
154        private ExtractDecodeEditEncodeMuxTest mTest;
155
156        private TestWrapper(ExtractDecodeEditEncodeMuxTest test) {
157            mTest = test;
158        }
159
160        @Override
161        public void run() {
162            try {
163                mTest.extractDecodeEditEncodeMux();
164            } catch (Throwable th) {
165                mThrowable = th;
166            }
167        }
168
169        /**
170         * Entry point.
171         */
172        public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable {
173            test.setOutputFile();
174            TestWrapper wrapper = new TestWrapper(test);
175            Thread th = new Thread(wrapper, "codec test");
176            th.start();
177            th.join();
178            if (wrapper.mThrowable != null) {
179                throw wrapper.mThrowable;
180            }
181        }
182    }
183
184    /**
185     * Sets the test to copy the video stream.
186     */
187    private void setCopyVideo() {
188        mCopyVideo = true;
189    }
190
191    /**
192     * Sets the test to copy the video stream.
193     */
194    private void setCopyAudio() {
195        mCopyAudio = true;
196    }
197
198    /**
199     * Sets the test to verify the output audio format.
200     */
201    private void setVerifyAudioFormat() {
202        mVerifyAudioFormat = true;
203    }
204
205    /**
206     * Sets the desired frame size.
207     */
208    private void setSize(int width, int height) {
209        if ((width % 16) != 0 || (height % 16) != 0) {
210            Log.w(TAG, "WARNING: width or height not multiple of 16");
211        }
212        mWidth = width;
213        mHeight = height;
214    }
215
216    /**
217     * Sets the raw resource used as the source video.
218     */
219    private void setSource(int resId) {
220        mSourceResId = resId;
221    }
222
223    /**
224     * Sets the name of the output file based on the other parameters.
225     *
226     * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(int)}.
227     */
228    private void setOutputFile() {
229        StringBuilder sb = new StringBuilder();
230        sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath());
231        sb.append("/cts-media-");
232        sb.append(getClass().getSimpleName());
233        assertTrue("should have called setSource() first", mSourceResId != -1);
234        sb.append('-');
235        sb.append(mSourceResId);
236        if (mCopyVideo) {
237            assertTrue("should have called setSize() first", mWidth != -1);
238            assertTrue("should have called setSize() first", mHeight != -1);
239            sb.append('-');
240            sb.append("video");
241            sb.append('-');
242            sb.append(mWidth);
243            sb.append('x');
244            sb.append(mHeight);
245        }
246        if (mCopyAudio) {
247            sb.append('-');
248            sb.append("audio");
249        }
250        sb.append(".mp4");
251        mOutputFile = sb.toString();
252    }
253
254    /**
255     * Tests encoding and subsequently decoding video from frames generated into a buffer.
256     * <p>
257     * We encode several frames of a video test pattern using MediaCodec, then decode the output
258     * with MediaCodec and do some simple checks.
259     */
260    private void extractDecodeEditEncodeMux() throws Exception {
261        // Exception that may be thrown during release.
262        Exception exception = null;
263
264        MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
265
266        // We avoid the device-specific limitations on width and height by using values
267        // that are multiples of 16, which all tested devices seem to be able to handle.
268        MediaFormat outputVideoFormat =
269                MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, mWidth, mHeight);
270
271        // Set some properties. Failing to specify some of these can cause the MediaCodec
272        // configure() call to throw an unhelpful exception.
273        outputVideoFormat.setInteger(
274                MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT);
275        outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
276        outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
277        outputVideoFormat.setInteger(
278                MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
279        if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
280
281        String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat);
282        if (videoEncoderName == null) {
283            // Don't fail CTS if they don't have an AVC codec (not here, anyway).
284            Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat);
285            return;
286        }
287        if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName);
288
289        MediaFormat outputAudioFormat =
290                MediaFormat.createAudioFormat(
291                        OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ,
292                        OUTPUT_AUDIO_CHANNEL_COUNT);
293        outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE);
294        outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
295
296        String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat);
297        if (audioEncoderName == null) {
298            // Don't fail CTS if they don't have an AAC codec (not here, anyway).
299            Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat);
300            return;
301        }
302        if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName);
303
304        MediaExtractor videoExtractor = null;
305        MediaExtractor audioExtractor = null;
306        OutputSurface outputSurface = null;
307        MediaCodec videoDecoder = null;
308        MediaCodec audioDecoder = null;
309        MediaCodec videoEncoder = null;
310        MediaCodec audioEncoder = null;
311        MediaMuxer muxer = null;
312
313        InputSurface inputSurface = null;
314
315        try {
316            if (mCopyVideo) {
317                videoExtractor = createExtractor();
318                int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
319                assertTrue("missing video track in test video", videoInputTrack != -1);
320                MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack);
321
322                // Create a MediaCodec for the desired codec, then configure it as an encoder with
323                // our desired properties. Request a Surface to use for input.
324                AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>();
325                videoEncoder = createVideoEncoder(
326                        videoEncoderName, outputVideoFormat, inputSurfaceReference);
327                inputSurface = new InputSurface(inputSurfaceReference.get());
328                inputSurface.makeCurrent();
329                // Create a MediaCodec for the decoder, based on the extractor's format.
330                outputSurface = new OutputSurface();
331                outputSurface.changeFragmentShader(FRAGMENT_SHADER);
332                videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface());
333            }
334
335            if (mCopyAudio) {
336                audioExtractor = createExtractor();
337                int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor);
338                assertTrue("missing audio track in test video", audioInputTrack != -1);
339                MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack);
340
341                // Create a MediaCodec for the desired codec, then configure it as an encoder with
342                // our desired properties. Request a Surface to use for input.
343                audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat);
344                // Create a MediaCodec for the decoder, based on the extractor's format.
345                audioDecoder = createAudioDecoder(mcl, inputFormat);
346            }
347
348            // Creates a muxer but do not start or add tracks just yet.
349            muxer = createMuxer();
350
351            doExtractDecodeEditEncodeMux(
352                    videoExtractor,
353                    audioExtractor,
354                    videoDecoder,
355                    videoEncoder,
356                    audioDecoder,
357                    audioEncoder,
358                    muxer,
359                    inputSurface,
360                    outputSurface);
361        } finally {
362            if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer");
363            // Try to release everything we acquired, even if one of the releases fails, in which
364            // case we save the first exception we got and re-throw at the end (unless something
365            // other exception has already been thrown). This guarantees the first exception thrown
366            // is reported as the cause of the error, everything is (attempted) to be released, and
367            // all other exceptions appear in the logs.
368            try {
369                if (videoExtractor != null) {
370                    videoExtractor.release();
371                }
372            } catch(Exception e) {
373                Log.e(TAG, "error while releasing videoExtractor", e);
374                if (exception == null) {
375                    exception = e;
376                }
377            }
378            try {
379                if (audioExtractor != null) {
380                    audioExtractor.release();
381                }
382            } catch(Exception e) {
383                Log.e(TAG, "error while releasing audioExtractor", e);
384                if (exception == null) {
385                    exception = e;
386                }
387            }
388            try {
389                if (videoDecoder != null) {
390                    videoDecoder.stop();
391                    videoDecoder.release();
392                }
393            } catch(Exception e) {
394                Log.e(TAG, "error while releasing videoDecoder", e);
395                if (exception == null) {
396                    exception = e;
397                }
398            }
399            try {
400                if (outputSurface != null) {
401                    outputSurface.release();
402                }
403            } catch(Exception e) {
404                Log.e(TAG, "error while releasing outputSurface", e);
405                if (exception == null) {
406                    exception = e;
407                }
408            }
409            try {
410                if (videoEncoder != null) {
411                    videoEncoder.stop();
412                    videoEncoder.release();
413                }
414            } catch(Exception e) {
415                Log.e(TAG, "error while releasing videoEncoder", e);
416                if (exception == null) {
417                    exception = e;
418                }
419            }
420            try {
421                if (audioDecoder != null) {
422                    audioDecoder.stop();
423                    audioDecoder.release();
424                }
425            } catch(Exception e) {
426                Log.e(TAG, "error while releasing audioDecoder", e);
427                if (exception == null) {
428                    exception = e;
429                }
430            }
431            try {
432                if (audioEncoder != null) {
433                    audioEncoder.stop();
434                    audioEncoder.release();
435                }
436            } catch(Exception e) {
437                Log.e(TAG, "error while releasing audioEncoder", e);
438                if (exception == null) {
439                    exception = e;
440                }
441            }
442            try {
443                if (muxer != null) {
444                    muxer.stop();
445                    muxer.release();
446                }
447            } catch(Exception e) {
448                Log.e(TAG, "error while releasing muxer", e);
449                if (exception == null) {
450                    exception = e;
451                }
452            }
453            try {
454                if (inputSurface != null) {
455                    inputSurface.release();
456                }
457            } catch(Exception e) {
458                Log.e(TAG, "error while releasing inputSurface", e);
459                if (exception == null) {
460                    exception = e;
461                }
462            }
463        }
464        if (exception != null) {
465            throw exception;
466        }
467
468        MediaExtractor mediaExtractor = null;
469        try {
470            mediaExtractor = new MediaExtractor();
471            mediaExtractor.setDataSource(mOutputFile);
472
473            assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0),
474                    mediaExtractor.getTrackCount());
475            if (mVerifyAudioFormat) {
476                boolean foundAudio = false;
477                for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
478                    MediaFormat trackFormat = mediaExtractor.getTrackFormat(i);
479                    if (isAudioFormat(trackFormat)) {
480                        foundAudio = true;
481                        int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ;
482
483                        // SBR mode halves the sample rate in the format.
484                        if (OUTPUT_AUDIO_AAC_PROFILE ==
485                                MediaCodecInfo.CodecProfileLevel.AACObjectHE) {
486                            expectedSampleRate /= 2;
487                        }
488                        assertEquals("sample rates should match", expectedSampleRate,
489                                trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE));
490                    }
491                }
492
493                assertTrue("output should have an audio track", foundAudio || !mCopyAudio);
494            }
495        } catch (IOException e) {
496            throw new IllegalStateException("exception verifying output file", e);
497        } finally {
498            if (mediaExtractor != null) {
499                mediaExtractor.release();
500            }
501        }
502
503        // TODO: Check the generated output file's video format and sample data.
504    }
505
506    /**
507     * Creates an extractor that reads its frames from {@link #mSourceResId}.
508     */
509    private MediaExtractor createExtractor() throws IOException {
510        MediaExtractor extractor;
511        AssetFileDescriptor srcFd = getContext().getResources().openRawResourceFd(mSourceResId);
512        extractor = new MediaExtractor();
513        extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(),
514                srcFd.getLength());
515        return extractor;
516    }
517
518    /**
519     * Creates a decoder for the given format, which outputs to the given surface.
520     *
521     * @param inputFormat the format of the stream to decode
522     * @param surface into which to decode the frames
523     */
524    private MediaCodec createVideoDecoder(
525            MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException {
526        MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat));
527        decoder.configure(inputFormat, surface, null, 0);
528        decoder.start();
529        return decoder;
530    }
531
532    /**
533     * Creates an encoder for the given format using the specified codec, taking input from a
534     * surface.
535     *
536     * <p>The surface to use as input is stored in the given reference.
537     *
538     * @param codecInfo of the codec to use
539     * @param format of the stream to be produced
540     * @param surfaceReference to store the surface to use as input
541     */
542    private MediaCodec createVideoEncoder(
543            String codecName,
544            MediaFormat format,
545            AtomicReference<Surface> surfaceReference)
546            throws IOException {
547        MediaCodec encoder = MediaCodec.createByCodecName(codecName);
548        encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
549        // Must be called before start() is.
550        surfaceReference.set(encoder.createInputSurface());
551        encoder.start();
552        return encoder;
553    }
554
555    /**
556     * Creates a decoder for the given format.
557     *
558     * @param inputFormat the format of the stream to decode
559     */
560    private MediaCodec createAudioDecoder(
561            MediaCodecList mcl, MediaFormat inputFormat) throws IOException {
562        MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat));
563        decoder.configure(inputFormat, null, null, 0);
564        decoder.start();
565        return decoder;
566    }
567
568    /**
569     * Creates an encoder for the given format using the specified codec.
570     *
571     * @param codecInfo of the codec to use
572     * @param format of the stream to be produced
573     */
574    private MediaCodec createAudioEncoder(String codecName, MediaFormat format)
575            throws IOException {
576        MediaCodec encoder = MediaCodec.createByCodecName(codecName);
577        encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
578        encoder.start();
579        return encoder;
580    }
581
582    /**
583     * Creates a muxer to write the encoded frames.
584     *
585     * <p>The muxer is not started as it needs to be started only after all streams have been added.
586     */
587    private MediaMuxer createMuxer() throws IOException {
588        return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
589    }
590
591    private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
592        for (int index = 0; index < extractor.getTrackCount(); ++index) {
593            if (VERBOSE) {
594                Log.d(TAG, "format for track " + index + " is "
595                        + getMimeTypeFor(extractor.getTrackFormat(index)));
596            }
597            if (isVideoFormat(extractor.getTrackFormat(index))) {
598                extractor.selectTrack(index);
599                return index;
600            }
601        }
602        return -1;
603    }
604
605    private int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
606        for (int index = 0; index < extractor.getTrackCount(); ++index) {
607            if (VERBOSE) {
608                Log.d(TAG, "format for track " + index + " is "
609                        + getMimeTypeFor(extractor.getTrackFormat(index)));
610            }
611            if (isAudioFormat(extractor.getTrackFormat(index))) {
612                extractor.selectTrack(index);
613                return index;
614            }
615        }
616        return -1;
617    }
618
619    /**
620     * Does the actual work for extracting, decoding, encoding and muxing.
621     */
622    private void doExtractDecodeEditEncodeMux(
623            MediaExtractor videoExtractor,
624            MediaExtractor audioExtractor,
625            MediaCodec videoDecoder,
626            MediaCodec videoEncoder,
627            MediaCodec audioDecoder,
628            MediaCodec audioEncoder,
629            MediaMuxer muxer,
630            InputSurface inputSurface,
631            OutputSurface outputSurface) {
632        ByteBuffer[] videoDecoderInputBuffers = null;
633        ByteBuffer[] videoDecoderOutputBuffers = null;
634        ByteBuffer[] videoEncoderOutputBuffers = null;
635        MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null;
636        MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null;
637        if (mCopyVideo) {
638            videoDecoderInputBuffers = videoDecoder.getInputBuffers();
639            videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
640            videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
641            videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
642            videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
643        }
644        ByteBuffer[] audioDecoderInputBuffers = null;
645        ByteBuffer[] audioDecoderOutputBuffers = null;
646        ByteBuffer[] audioEncoderInputBuffers = null;
647        ByteBuffer[] audioEncoderOutputBuffers = null;
648        MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null;
649        MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null;
650        if (mCopyAudio) {
651            audioDecoderInputBuffers = audioDecoder.getInputBuffers();
652            audioDecoderOutputBuffers =  audioDecoder.getOutputBuffers();
653            audioEncoderInputBuffers = audioEncoder.getInputBuffers();
654            audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
655            audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
656            audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
657        }
658        // We will get these from the decoders when notified of a format change.
659        MediaFormat decoderOutputVideoFormat = null;
660        MediaFormat decoderOutputAudioFormat = null;
661        // We will get these from the encoders when notified of a format change.
662        MediaFormat encoderOutputVideoFormat = null;
663        MediaFormat encoderOutputAudioFormat = null;
664        // We will determine these once we have the output format.
665        int outputVideoTrack = -1;
666        int outputAudioTrack = -1;
667        // Whether things are done on the video side.
668        boolean videoExtractorDone = false;
669        boolean videoDecoderDone = false;
670        boolean videoEncoderDone = false;
671        // Whether things are done on the audio side.
672        boolean audioExtractorDone = false;
673        boolean audioDecoderDone = false;
674        boolean audioEncoderDone = false;
675        // The audio decoder output buffer to process, -1 if none.
676        int pendingAudioDecoderOutputBufferIndex = -1;
677
678        boolean muxing = false;
679
680        int videoExtractedFrameCount = 0;
681        int videoDecodedFrameCount = 0;
682        int videoEncodedFrameCount = 0;
683
684        int audioExtractedFrameCount = 0;
685        int audioDecodedFrameCount = 0;
686        int audioEncodedFrameCount = 0;
687
688        while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) {
689            if (VERBOSE) {
690                Log.d(TAG, String.format(
691                        "loop: "
692
693                        + "V(%b){"
694                        + "extracted:%d(done:%b) "
695                        + "decoded:%d(done:%b) "
696                        + "encoded:%d(done:%b)} "
697
698                        + "A(%b){"
699                        + "extracted:%d(done:%b) "
700                        + "decoded:%d(done:%b) "
701                        + "encoded:%d(done:%b) "
702                        + "pending:%d} "
703
704                        + "muxing:%b(V:%d,A:%d)",
705
706                        mCopyVideo,
707                        videoExtractedFrameCount, videoExtractorDone,
708                        videoDecodedFrameCount, videoDecoderDone,
709                        videoEncodedFrameCount, videoEncoderDone,
710
711                        mCopyAudio,
712                        audioExtractedFrameCount, audioExtractorDone,
713                        audioDecodedFrameCount, audioDecoderDone,
714                        audioEncodedFrameCount, audioEncoderDone,
715                        pendingAudioDecoderOutputBufferIndex,
716
717                        muxing, outputVideoTrack, outputAudioTrack));
718            }
719
720            // Extract video from file and feed to decoder.
721            // Do not extract video if we have determined the output format but we are not yet
722            // ready to mux the frames.
723            while (mCopyVideo && !videoExtractorDone
724                    && (encoderOutputVideoFormat == null || muxing)) {
725                int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
726                if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
727                    if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
728                    break;
729                }
730                if (VERBOSE) {
731                    Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
732                }
733                ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex];
734                int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
735                long presentationTime = videoExtractor.getSampleTime();
736                if (VERBOSE) {
737                    Log.d(TAG, "video extractor: returned buffer of size " + size);
738                    Log.d(TAG, "video extractor: returned buffer for time " + presentationTime);
739                }
740                if (size >= 0) {
741                    videoDecoder.queueInputBuffer(
742                            decoderInputBufferIndex,
743                            0,
744                            size,
745                            presentationTime,
746                            videoExtractor.getSampleFlags());
747                }
748                videoExtractorDone = !videoExtractor.advance();
749                if (videoExtractorDone) {
750                    if (VERBOSE) Log.d(TAG, "video extractor: EOS");
751                    videoDecoder.queueInputBuffer(
752                            decoderInputBufferIndex,
753                            0,
754                            0,
755                            0,
756                            MediaCodec.BUFFER_FLAG_END_OF_STREAM);
757                }
758                videoExtractedFrameCount++;
759                // We extracted a frame, let's try something else next.
760                break;
761            }
762
763            // Extract audio from file and feed to decoder.
764            // Do not extract audio if we have determined the output format but we are not yet
765            // ready to mux the frames.
766            while (mCopyAudio && !audioExtractorDone
767                    && (encoderOutputAudioFormat == null || muxing)) {
768                int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC);
769                if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
770                    if (VERBOSE) Log.d(TAG, "no audio decoder input buffer");
771                    break;
772                }
773                if (VERBOSE) {
774                    Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex);
775                }
776                ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex];
777                int size = audioExtractor.readSampleData(decoderInputBuffer, 0);
778                long presentationTime = audioExtractor.getSampleTime();
779                if (VERBOSE) {
780                    Log.d(TAG, "audio extractor: returned buffer of size " + size);
781                    Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime);
782                }
783                if (size >= 0) {
784                    audioDecoder.queueInputBuffer(
785                            decoderInputBufferIndex,
786                            0,
787                            size,
788                            presentationTime,
789                            audioExtractor.getSampleFlags());
790                }
791                audioExtractorDone = !audioExtractor.advance();
792                if (audioExtractorDone) {
793                    if (VERBOSE) Log.d(TAG, "audio extractor: EOS");
794                    audioDecoder.queueInputBuffer(
795                            decoderInputBufferIndex,
796                            0,
797                            0,
798                            0,
799                            MediaCodec.BUFFER_FLAG_END_OF_STREAM);
800                }
801                audioExtractedFrameCount++;
802                // We extracted a frame, let's try something else next.
803                break;
804            }
805
806            // Poll output frames from the video decoder and feed the encoder.
807            while (mCopyVideo && !videoDecoderDone
808                    && (encoderOutputVideoFormat == null || muxing)) {
809                int decoderOutputBufferIndex =
810                        videoDecoder.dequeueOutputBuffer(
811                                videoDecoderOutputBufferInfo, TIMEOUT_USEC);
812                if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
813                    if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
814                    break;
815                }
816                if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
817                    if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
818                    videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
819                    break;
820                }
821                if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
822                    decoderOutputVideoFormat = videoDecoder.getOutputFormat();
823                    if (VERBOSE) {
824                        Log.d(TAG, "video decoder: output format changed: "
825                                + decoderOutputVideoFormat);
826                    }
827                    break;
828                }
829                if (VERBOSE) {
830                    Log.d(TAG, "video decoder: returned output buffer: "
831                            + decoderOutputBufferIndex);
832                    Log.d(TAG, "video decoder: returned buffer of size "
833                            + videoDecoderOutputBufferInfo.size);
834                }
835                ByteBuffer decoderOutputBuffer =
836                        videoDecoderOutputBuffers[decoderOutputBufferIndex];
837                if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
838                        != 0) {
839                    if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
840                    videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
841                    break;
842                }
843                if (VERBOSE) {
844                    Log.d(TAG, "video decoder: returned buffer for time "
845                            + videoDecoderOutputBufferInfo.presentationTimeUs);
846                }
847                boolean render = videoDecoderOutputBufferInfo.size != 0;
848                videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
849                if (render) {
850                    if (VERBOSE) Log.d(TAG, "output surface: await new image");
851                    outputSurface.awaitNewImage();
852                    // Edit the frame and send it to the encoder.
853                    if (VERBOSE) Log.d(TAG, "output surface: draw image");
854                    outputSurface.drawImage();
855                    inputSurface.setPresentationTime(
856                            videoDecoderOutputBufferInfo.presentationTimeUs * 1000);
857                    if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
858                    inputSurface.swapBuffers();
859                    if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
860                }
861                if ((videoDecoderOutputBufferInfo.flags
862                        & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
863                    if (VERBOSE) Log.d(TAG, "video decoder: EOS");
864                    videoDecoderDone = true;
865                    videoEncoder.signalEndOfInputStream();
866                }
867                videoDecodedFrameCount++;
868                // We extracted a pending frame, let's try something else next.
869                break;
870            }
871
872            // Poll output frames from the audio decoder.
873            // Do not poll if we already have a pending buffer to feed to the encoder.
874            while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1
875                    && (encoderOutputAudioFormat == null || muxing)) {
876                int decoderOutputBufferIndex =
877                        audioDecoder.dequeueOutputBuffer(
878                                audioDecoderOutputBufferInfo, TIMEOUT_USEC);
879                if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
880                    if (VERBOSE) Log.d(TAG, "no audio decoder output buffer");
881                    break;
882                }
883                if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
884                    if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed");
885                    audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
886                    break;
887                }
888                if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
889                    decoderOutputAudioFormat = audioDecoder.getOutputFormat();
890                    if (VERBOSE) {
891                        Log.d(TAG, "audio decoder: output format changed: "
892                                + decoderOutputAudioFormat);
893                    }
894                    break;
895                }
896                if (VERBOSE) {
897                    Log.d(TAG, "audio decoder: returned output buffer: "
898                            + decoderOutputBufferIndex);
899                }
900                if (VERBOSE) {
901                    Log.d(TAG, "audio decoder: returned buffer of size "
902                            + audioDecoderOutputBufferInfo.size);
903                }
904                ByteBuffer decoderOutputBuffer =
905                        audioDecoderOutputBuffers[decoderOutputBufferIndex];
906                if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
907                        != 0) {
908                    if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer");
909                    audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
910                    break;
911                }
912                if (VERBOSE) {
913                    Log.d(TAG, "audio decoder: returned buffer for time "
914                            + audioDecoderOutputBufferInfo.presentationTimeUs);
915                }
916                if (VERBOSE) {
917                    Log.d(TAG, "audio decoder: output buffer is now pending: "
918                            + pendingAudioDecoderOutputBufferIndex);
919                }
920                pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
921                audioDecodedFrameCount++;
922                // We extracted a pending frame, let's try something else next.
923                break;
924            }
925
926            // Feed the pending decoded audio buffer to the audio encoder.
927            while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) {
928                if (VERBOSE) {
929                    Log.d(TAG, "audio decoder: attempting to process pending buffer: "
930                            + pendingAudioDecoderOutputBufferIndex);
931                }
932                int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC);
933                if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
934                    if (VERBOSE) Log.d(TAG, "no audio encoder input buffer");
935                    break;
936                }
937                if (VERBOSE) {
938                    Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex);
939                }
940                ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex];
941                int size = audioDecoderOutputBufferInfo.size;
942                long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs;
943                if (VERBOSE) {
944                    Log.d(TAG, "audio decoder: processing pending buffer: "
945                            + pendingAudioDecoderOutputBufferIndex);
946                }
947                if (VERBOSE) {
948                    Log.d(TAG, "audio decoder: pending buffer of size " + size);
949                    Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime);
950                }
951                if (size >= 0) {
952                    ByteBuffer decoderOutputBuffer =
953                            audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex]
954                                    .duplicate();
955                    decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset);
956                    decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size);
957                    encoderInputBuffer.position(0);
958                    encoderInputBuffer.put(decoderOutputBuffer);
959
960                    audioEncoder.queueInputBuffer(
961                            encoderInputBufferIndex,
962                            0,
963                            size,
964                            presentationTime,
965                            audioDecoderOutputBufferInfo.flags);
966                }
967                audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false);
968                pendingAudioDecoderOutputBufferIndex = -1;
969                if ((audioDecoderOutputBufferInfo.flags
970                        & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
971                    if (VERBOSE) Log.d(TAG, "audio decoder: EOS");
972                    audioDecoderDone = true;
973                }
974                // We enqueued a pending frame, let's try something else next.
975                break;
976            }
977
978            // Poll frames from the video encoder and send them to the muxer.
979            while (mCopyVideo && !videoEncoderDone
980                    && (encoderOutputVideoFormat == null || muxing)) {
981                int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer(
982                        videoEncoderOutputBufferInfo, TIMEOUT_USEC);
983                if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
984                    if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
985                    break;
986                }
987                if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
988                    if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
989                    videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
990                    break;
991                }
992                if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
993                    if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
994                    if (outputVideoTrack >= 0) {
995                        fail("video encoder changed its output format again?");
996                    }
997                    encoderOutputVideoFormat = videoEncoder.getOutputFormat();
998                    break;
999                }
1000                assertTrue("should have added track before processing output", muxing);
1001                if (VERBOSE) {
1002                    Log.d(TAG, "video encoder: returned output buffer: "
1003                            + encoderOutputBufferIndex);
1004                    Log.d(TAG, "video encoder: returned buffer of size "
1005                            + videoEncoderOutputBufferInfo.size);
1006                }
1007                ByteBuffer encoderOutputBuffer =
1008                        videoEncoderOutputBuffers[encoderOutputBufferIndex];
1009                if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
1010                        != 0) {
1011                    if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
1012                    // Simply ignore codec config buffers.
1013                    videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
1014                    break;
1015                }
1016                if (VERBOSE) {
1017                    Log.d(TAG, "video encoder: returned buffer for time "
1018                            + videoEncoderOutputBufferInfo.presentationTimeUs);
1019                }
1020                if (videoEncoderOutputBufferInfo.size != 0) {
1021                    muxer.writeSampleData(
1022                            outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo);
1023                }
1024                if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
1025                        != 0) {
1026                    if (VERBOSE) Log.d(TAG, "video encoder: EOS");
1027                    videoEncoderDone = true;
1028                }
1029                videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
1030                videoEncodedFrameCount++;
1031                // We enqueued an encoded frame, let's try something else next.
1032                break;
1033            }
1034
1035            // Poll frames from the audio encoder and send them to the muxer.
1036            while (mCopyAudio && !audioEncoderDone
1037                    && (encoderOutputAudioFormat == null || muxing)) {
1038                int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer(
1039                        audioEncoderOutputBufferInfo, TIMEOUT_USEC);
1040                if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
1041                    if (VERBOSE) Log.d(TAG, "no audio encoder output buffer");
1042                    break;
1043                }
1044                if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
1045                    if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed");
1046                    audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
1047                    break;
1048                }
1049                if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1050                    if (VERBOSE) Log.d(TAG, "audio encoder: output format changed");
1051                    if (outputAudioTrack >= 0) {
1052                        fail("audio encoder changed its output format again?");
1053                    }
1054
1055                    encoderOutputAudioFormat = audioEncoder.getOutputFormat();
1056                    break;
1057                }
1058                assertTrue("should have added track before processing output", muxing);
1059                if (VERBOSE) {
1060                    Log.d(TAG, "audio encoder: returned output buffer: "
1061                            + encoderOutputBufferIndex);
1062                    Log.d(TAG, "audio encoder: returned buffer of size "
1063                            + audioEncoderOutputBufferInfo.size);
1064                }
1065                ByteBuffer encoderOutputBuffer =
1066                        audioEncoderOutputBuffers[encoderOutputBufferIndex];
1067                if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
1068                        != 0) {
1069                    if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer");
1070                    // Simply ignore codec config buffers.
1071                    audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
1072                    break;
1073                }
1074                if (VERBOSE) {
1075                    Log.d(TAG, "audio encoder: returned buffer for time "
1076                            + audioEncoderOutputBufferInfo.presentationTimeUs);
1077                }
1078                if (audioEncoderOutputBufferInfo.size != 0) {
1079                    muxer.writeSampleData(
1080                            outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo);
1081                }
1082                if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
1083                        != 0) {
1084                    if (VERBOSE) Log.d(TAG, "audio encoder: EOS");
1085                    audioEncoderDone = true;
1086                }
1087                audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
1088                audioEncodedFrameCount++;
1089                // We enqueued an encoded frame, let's try something else next.
1090                break;
1091            }
1092
1093            if (!muxing
1094                    && (!mCopyAudio || encoderOutputAudioFormat != null)
1095                    && (!mCopyVideo || encoderOutputVideoFormat != null)) {
1096                if (mCopyVideo) {
1097                    Log.d(TAG, "muxer: adding video track.");
1098                    outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
1099                }
1100                if (mCopyAudio) {
1101                    Log.d(TAG, "muxer: adding audio track.");
1102                    outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat);
1103                }
1104                Log.d(TAG, "muxer: starting");
1105                muxer.start();
1106                muxing = true;
1107            }
1108        }
1109
1110        // Basic sanity checks.
1111        if (mCopyVideo) {
1112            assertEquals("encoded and decoded video frame counts should match",
1113                    videoDecodedFrameCount, videoEncodedFrameCount);
1114            assertTrue("decoded frame count should be less than extracted frame count",
1115                    videoDecodedFrameCount <= videoExtractedFrameCount);
1116        }
1117        if (mCopyAudio) {
1118            assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex);
1119        }
1120    }
1121
1122    private static boolean isVideoFormat(MediaFormat format) {
1123        return getMimeTypeFor(format).startsWith("video/");
1124    }
1125
1126    private static boolean isAudioFormat(MediaFormat format) {
1127        return getMimeTypeFor(format).startsWith("audio/");
1128    }
1129
1130    private static String getMimeTypeFor(MediaFormat format) {
1131        return format.getString(MediaFormat.KEY_MIME);
1132    }
1133}
1134