1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package androidx.media.filterfw.decoder;
18
19import android.annotation.TargetApi;
20import android.content.Context;
21import android.media.MediaExtractor;
22import android.media.MediaFormat;
23import android.media.MediaMetadataRetriever;
24import android.net.Uri;
25import android.os.Build;
26import android.util.Log;
27import androidx.media.filterfw.FrameImage2D;
28import androidx.media.filterfw.FrameValue;
29import androidx.media.filterfw.RenderTarget;
30
31import java.util.concurrent.LinkedBlockingQueue;
32
33@TargetApi(16)
34public class MediaDecoder implements
35        Runnable,
36        TrackDecoder.Listener {
37
38    public interface Listener {
39        /**
40         * Notifies a listener when a decoded video frame is available. The listener should use
41         * {@link MediaDecoder#grabVideoFrame(FrameImage2D, int)} to grab the video data for this
42         * frame.
43         */
44        void onVideoFrameAvailable();
45
46        /**
47         * Notifies a listener when one or more audio samples are available. The listener should use
48         * {@link MediaDecoder#grabAudioSamples(FrameValue)} to grab the audio samples.
49         */
50        void onAudioSamplesAvailable();
51
52        /**
53         * Notifies a listener that decoding has started. This method is called on the decoder
54         * thread.
55         */
56        void onDecodingStarted();
57
58        /**
59         * Notifies a listener that decoding has stopped. This method is called on the decoder
60         * thread.
61         */
62        void onDecodingStopped();
63
64        /**
65         * Notifies a listener that an error occurred. If an error occurs, {@link MediaDecoder} is
66         * stopped and no more events are reported to this {@link Listener}'s callbacks.
67         * This method is called on the decoder thread.
68         */
69        void onError(Exception e);
70    }
71
72    public static final int ROTATE_NONE = 0;
73    public static final int ROTATE_90_RIGHT = 90;
74    public static final int ROTATE_180 = 180;
75    public static final int ROTATE_90_LEFT = 270;
76
77    private static final String LOG_TAG = "MediaDecoder";
78    private static final boolean DEBUG = false;
79
80    private static final int MAX_EVENTS = 32;
81    private static final int EVENT_START = 0;
82    private static final int EVENT_STOP = 1;
83    private static final int EVENT_EOF = 2;
84
85    private final Listener mListener;
86    private final Uri mUri;
87    private final Context mContext;
88
89    private final LinkedBlockingQueue<Integer> mEventQueue;
90
91    private final Thread mDecoderThread;
92
93    private MediaExtractor mMediaExtractor;
94
95    private RenderTarget mRenderTarget;
96
97    private int mDefaultRotation;
98    private int mVideoTrackIndex;
99    private int mAudioTrackIndex;
100
101    private VideoTrackDecoder mVideoTrackDecoder;
102    private AudioTrackDecoder mAudioTrackDecoder;
103
104    private boolean mStarted;
105
106    private long mStartMicros;
107
108    private boolean mOpenGLEnabled = true;
109
110    private boolean mSignaledEndOfInput;
111    private boolean mSeenEndOfAudioOutput;
112    private boolean mSeenEndOfVideoOutput;
113
114    public MediaDecoder(Context context, Uri uri, Listener listener) {
115        this(context, uri, 0, listener);
116    }
117
118    public MediaDecoder(Context context, Uri uri, long startMicros, Listener listener) {
119        if (context == null) {
120            throw new NullPointerException("context cannot be null");
121        }
122        mContext = context;
123
124        if (uri == null) {
125            throw new NullPointerException("uri cannot be null");
126        }
127        mUri = uri;
128
129        if (startMicros < 0) {
130            throw new IllegalArgumentException("startMicros cannot be negative");
131        }
132        mStartMicros = startMicros;
133
134        if (listener == null) {
135            throw new NullPointerException("listener cannot be null");
136        }
137        mListener = listener;
138
139        mEventQueue = new LinkedBlockingQueue<Integer>(MAX_EVENTS);
140        mDecoderThread = new Thread(this);
141    }
142
143    /**
144     * Set whether decoder may use OpenGL for decoding.
145     *
146     * This must be called before {@link #start()}.
147     *
148     * @param enabled flag whether to enable OpenGL decoding (default is true).
149     */
150    public void setOpenGLEnabled(boolean enabled) {
151        // If event-queue already has events, we have started already.
152        if (mEventQueue.isEmpty()) {
153            mOpenGLEnabled = enabled;
154        } else {
155            throw new IllegalStateException(
156                    "Must call setOpenGLEnabled() before calling start()!");
157        }
158    }
159
160    /**
161     * Returns whether OpenGL is enabled for decoding.
162     *
163     * @return whether OpenGL is enabled for decoding.
164     */
165    public boolean isOpenGLEnabled() {
166        return mOpenGLEnabled;
167    }
168
169    public void start() {
170        mEventQueue.offer(EVENT_START);
171        mDecoderThread.start();
172    }
173
174    public void stop() {
175        stop(true);
176    }
177
178    private void stop(boolean manual) {
179        if (manual) {
180            mEventQueue.offer(EVENT_STOP);
181            mDecoderThread.interrupt();
182        } else {
183            mEventQueue.offer(EVENT_EOF);
184        }
185    }
186
187    @Override
188    public void run() {
189        Integer event;
190        try {
191            while (true) {
192                event = mEventQueue.poll();
193                boolean shouldStop = false;
194                if (event != null) {
195                    switch (event) {
196                        case EVENT_START:
197                            onStart();
198                            break;
199                        case EVENT_EOF:
200                            if (mVideoTrackDecoder != null) {
201                                mVideoTrackDecoder.waitForFrameGrab();
202                            }
203                            // once the last frame has been grabbed, fall through and stop
204                        case EVENT_STOP:
205                            onStop(true);
206                            shouldStop = true;
207                            break;
208                    }
209                } else if (mStarted) {
210                    decode();
211                }
212                if (shouldStop) {
213                    break;
214                }
215
216            }
217        } catch (Exception e) {
218            mListener.onError(e);
219            onStop(false);
220        }
221    }
222
223    private void onStart() throws Exception {
224        if (mOpenGLEnabled) {
225            getRenderTarget().focus();
226        }
227
228        mMediaExtractor = new MediaExtractor();
229        mMediaExtractor.setDataSource(mContext, mUri, null);
230
231        mVideoTrackIndex = -1;
232        mAudioTrackIndex = -1;
233
234        for (int i = 0; i < mMediaExtractor.getTrackCount(); i++) {
235            MediaFormat format = mMediaExtractor.getTrackFormat(i);
236            if (DEBUG) {
237                Log.i(LOG_TAG, "Uri " + mUri + ", track " + i + ": " + format);
238            }
239            if (DecoderUtil.isVideoFormat(format) && mVideoTrackIndex == -1) {
240                mVideoTrackIndex = i;
241            } else if (DecoderUtil.isAudioFormat(format) && mAudioTrackIndex == -1) {
242                mAudioTrackIndex = i;
243            }
244        }
245
246        if (mVideoTrackIndex == -1 && mAudioTrackIndex == -1) {
247            throw new IllegalArgumentException(
248                    "Couldn't find a video or audio track in the provided file");
249        }
250
251        if (mVideoTrackIndex != -1) {
252            MediaFormat videoFormat = mMediaExtractor.getTrackFormat(mVideoTrackIndex);
253            mVideoTrackDecoder = mOpenGLEnabled
254                    ? new GpuVideoTrackDecoder(mVideoTrackIndex, videoFormat, this)
255                    : new CpuVideoTrackDecoder(mVideoTrackIndex, videoFormat, this);
256            mVideoTrackDecoder.init();
257            mMediaExtractor.selectTrack(mVideoTrackIndex);
258            if (Build.VERSION.SDK_INT >= 17) {
259                retrieveDefaultRotation();
260            }
261        }
262
263        if (mAudioTrackIndex != -1) {
264            MediaFormat audioFormat = mMediaExtractor.getTrackFormat(mAudioTrackIndex);
265            mAudioTrackDecoder = new AudioTrackDecoder(mAudioTrackIndex, audioFormat, this);
266            mAudioTrackDecoder.init();
267            mMediaExtractor.selectTrack(mAudioTrackIndex);
268        }
269
270        if (mStartMicros > 0) {
271            mMediaExtractor.seekTo(mStartMicros, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
272        }
273
274        mStarted = true;
275        mListener.onDecodingStarted();
276    }
277
278    @TargetApi(17)
279    private void retrieveDefaultRotation() {
280        MediaMetadataRetriever metadataRetriever = new MediaMetadataRetriever();
281        metadataRetriever.setDataSource(mContext, mUri);
282        String rotationString = metadataRetriever.extractMetadata(
283                MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION);
284        mDefaultRotation = rotationString == null ? 0 : Integer.parseInt(rotationString);
285    }
286
287    private void onStop(boolean notifyListener) {
288        mMediaExtractor.release();
289        mMediaExtractor = null;
290
291        if (mVideoTrackDecoder != null) {
292            mVideoTrackDecoder.release();
293            mVideoTrackDecoder = null;
294        }
295
296        if (mAudioTrackDecoder != null) {
297            mAudioTrackDecoder.release();
298            mAudioTrackDecoder = null;
299        }
300
301        if (mOpenGLEnabled) {
302            if (mRenderTarget != null) {
303                getRenderTarget().release();
304            }
305            RenderTarget.focusNone();
306        }
307
308        mVideoTrackIndex = -1;
309        mAudioTrackIndex = -1;
310
311        mEventQueue.clear();
312        mStarted = false;
313        if (notifyListener) {
314            mListener.onDecodingStopped();
315        }
316    }
317
318    private void decode() {
319        int sampleTrackIndex = mMediaExtractor.getSampleTrackIndex();
320        if (sampleTrackIndex >= 0) {
321            if (sampleTrackIndex == mVideoTrackIndex) {
322                mVideoTrackDecoder.feedInput(mMediaExtractor);
323            } else if (sampleTrackIndex == mAudioTrackIndex) {
324                mAudioTrackDecoder.feedInput(mMediaExtractor);
325            }
326        } else if (!mSignaledEndOfInput) {
327            if (mVideoTrackDecoder != null) {
328                mVideoTrackDecoder.signalEndOfInput();
329            }
330            if (mAudioTrackDecoder != null) {
331                mAudioTrackDecoder.signalEndOfInput();
332            }
333            mSignaledEndOfInput = true;
334        }
335
336        if (mVideoTrackDecoder != null) {
337            mVideoTrackDecoder.drainOutputBuffer();
338        }
339        if (mAudioTrackDecoder != null) {
340            mAudioTrackDecoder.drainOutputBuffer();
341        }
342    }
343
344    /**
345     * Fills the argument frame with the video data, using the rotation hint obtained from the
346     * file's metadata, if any.
347     *
348     * @see #grabVideoFrame(FrameImage2D, int)
349     */
350    public void grabVideoFrame(FrameImage2D outputVideoFrame) {
351        grabVideoFrame(outputVideoFrame, mDefaultRotation);
352    }
353
354    /**
355     * Fills the argument frame with the video data, the frame will be returned with the given
356     * rotation applied.
357     *
358     * @param outputVideoFrame the output video frame.
359     * @param videoRotation the rotation angle that is applied to the raw decoded frame.
360     *   Value is one of {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}.
361     */
362    public void grabVideoFrame(FrameImage2D outputVideoFrame, int videoRotation) {
363        if (mVideoTrackDecoder != null && outputVideoFrame != null) {
364            mVideoTrackDecoder.grabFrame(outputVideoFrame, videoRotation);
365        }
366    }
367
368    /**
369     * Fills the argument frame with the audio data.
370     *
371     * @param outputAudioFrame the output audio frame.
372     */
373    public void grabAudioSamples(FrameValue outputAudioFrame) {
374        if (mAudioTrackDecoder != null) {
375            if (outputAudioFrame != null) {
376                mAudioTrackDecoder.grabSample(outputAudioFrame);
377            } else {
378                mAudioTrackDecoder.clearBuffer();
379            }
380        }
381    }
382
383    /**
384     * Gets the duration, in nanoseconds.
385     */
386    public long getDuration() {
387        if (!mStarted) {
388            throw new IllegalStateException("MediaDecoder has not been started");
389        }
390
391        MediaFormat mediaFormat = mMediaExtractor.getTrackFormat(
392                mVideoTrackIndex != -1 ? mVideoTrackIndex : mAudioTrackIndex);
393        return mediaFormat.getLong(MediaFormat.KEY_DURATION) * 1000;
394    }
395
396    private RenderTarget getRenderTarget() {
397        if (mRenderTarget == null) {
398            mRenderTarget = RenderTarget.newTarget(1, 1);
399        }
400        return mRenderTarget;
401    }
402
403    @Override
404    public void onDecodedOutputAvailable(TrackDecoder decoder) {
405        if (decoder == mVideoTrackDecoder) {
406            mListener.onVideoFrameAvailable();
407        } else if (decoder == mAudioTrackDecoder) {
408            mListener.onAudioSamplesAvailable();
409        }
410    }
411
412    @Override
413    public void onEndOfStream(TrackDecoder decoder) {
414        if (decoder == mAudioTrackDecoder) {
415            mSeenEndOfAudioOutput = true;
416        } else if (decoder == mVideoTrackDecoder) {
417            mSeenEndOfVideoOutput = true;
418        }
419
420        if ((mAudioTrackDecoder == null || mSeenEndOfAudioOutput)
421                && (mVideoTrackDecoder == null || mSeenEndOfVideoOutput)) {
422            stop(false);
423        }
424    }
425
426}
427