1efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org/* 2efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * libjingle 35f93d0a140515e3b8cdd1b9a4c6f5871144e5deejlmiller@webrtc.org * Copyright 2014 Google Inc. 4efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 5efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * Redistribution and use in source and binary forms, with or without 6efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * modification, are permitted provided that the following conditions are met: 7efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 8efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 1. Redistributions of source code must retain the above copyright notice, 9efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * this list of conditions and the following disclaimer. 10efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 2. Redistributions in binary form must reproduce the above copyright notice, 11efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * this list of conditions and the following disclaimer in the documentation 12efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * and/or other materials provided with the distribution. 13efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 3. The name of the author may not be used to endorse or promote products 14efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * derived from this software without specific prior written permission. 15efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * 16efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org */ 27efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 28efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgpackage org.webrtc; 29efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 3068876f990ea1ea365d2d8155df261b38ec9fbeffPatrik Höglundimport android.graphics.SurfaceTexture; 31efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgimport android.media.MediaCodec; 32a09a99950ec40aef6421e4ba35eee7196b7a6e68buildbot@webrtc.orgimport android.media.MediaCodecInfo; 33996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.orgimport android.media.MediaCodecInfo.CodecCapabilities; 34efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgimport android.media.MediaCodecList; 35efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgimport android.media.MediaFormat; 36efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgimport android.os.Build; 37488e75f11b840dfbe636a9ea9bbc18252e7c59f0Perimport android.os.SystemClock; 38996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.orgimport android.view.Surface; 39996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 405975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liuimport org.webrtc.Logging; 415975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu 42efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.orgimport java.nio.ByteBuffer; 437e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvertimport java.util.Arrays; 44488e75f11b840dfbe636a9ea9bbc18252e7c59f0Perimport java.util.LinkedList; 4591b348c7029d843e06868ed12b728a809c53176cMagnus Jedvertimport java.util.List; 465c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznevimport java.util.concurrent.CountDownLatch; 47488e75f11b840dfbe636a9ea9bbc18252e7c59f0Perimport java.util.Queue; 48488e75f11b840dfbe636a9ea9bbc18252e7c59f0Perimport java.util.concurrent.TimeUnit; 498c425aa8f66fc2f06df402a0f2163cb53373856fmagjed 50efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org// Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. 51efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org// This class is an implementation detail of the Java PeerConnection API. 5268876f990ea1ea365d2d8155df261b38ec9fbeffPatrik Höglund@SuppressWarnings("deprecation") 53908e77bd00593a8bc5480fb1a07368b7c27778f3Alex Glaznevpublic class MediaCodecVideoDecoder { 54efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // This class is constructed, operated, and destroyed by its C++ incarnation, 55efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // so the class and its methods have non-public visibility. The API this 56efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // class exposes aims to mimic the webrtc::VideoDecoder API as closely as 57efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // possibly to minimize the amount of translation work necessary. 58efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 59efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private static final String TAG = "MediaCodecVideoDecoder"; 60efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 61b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org // Tracks webrtc::VideoCodecType. 62b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org public enum VideoCodecType { 63b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org VIDEO_CODEC_VP8, 64b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org VIDEO_CODEC_VP9, 65b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org VIDEO_CODEC_H264 66b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org } 67b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org 68996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org private static final int DEQUEUE_INPUT_TIMEOUT = 500000; // 500 ms timeout. 695c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // Timeout for codec releasing. 70c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev // Active running decoder instance. Set in initDecode() (called from native code) 71c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev // and reset to null in release() call. 72c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev private static MediaCodecVideoDecoder runningInstance = null; 735c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev private static MediaCodecVideoDecoderErrorCallback errorCallback = null; 745c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev private static int codecErrors = 0; 755c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 7669ddaefbb39975fde3e0bb43e233c357f514213cAlejandro Luebs private Thread mediaCodecThread; 77efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private MediaCodec mediaCodec; 78efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private ByteBuffer[] inputBuffers; 79efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private ByteBuffer[] outputBuffers; 80efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; 8169a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; 82b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org private static final String H264_MIME_TYPE = "video/avc"; 83efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // List of supported HW VP8 decoders. 84b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org private static final String[] supportedVp8HwCodecPrefixes = 854ddc9387bd064f89411a54f890f240ec17d86ed2glaznev@webrtc.org {"OMX.qcom.", "OMX.Nvidia.", "OMX.Exynos.", "OMX.Intel." }; 8669a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev // List of supported HW VP9 decoders. 8769a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev private static final String[] supportedVp9HwCodecPrefixes = 8817c0aff9eab8f2977c3ced1c9248cc10810f087eAlex Glaznev {"OMX.qcom.", "OMX.Exynos." }; 89b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org // List of supported HW H.264 decoders. 90b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org private static final String[] supportedH264HwCodecPrefixes = 910482dcc87376468be5f8a1a6d8f1d8a4f58267e8Alex Glaznev {"OMX.qcom.", "OMX.Intel." }; 92efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // NV12 color format supported by QCOM codec, but not declared in MediaCodec - 93efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h 94efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private static final int 95efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; 96efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // Allowable color formats supported by codec - in order of preference. 977e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert private static final List<Integer> supportedColorList = Arrays.asList( 98efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org CodecCapabilities.COLOR_FormatYUV420Planar, 99efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org CodecCapabilities.COLOR_FormatYUV420SemiPlanar, 100efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, 1017e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); 102efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int colorFormat; 103efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int width; 104efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int height; 105efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int stride; 106efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int sliceHeight; 107488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private boolean hasDecodedFirstFrame; 108488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final Queue<TimeStamps> decodeStartTimeMs = new LinkedList<TimeStamps>(); 109996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org private boolean useSurface; 110c01c25434ba92f6ea32cdfdcde77ec8278182851Per 111488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // The below variables are only used when decoding to a Surface. 112488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private TextureListener textureListener; 113488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Max number of output buffers queued before starting to drop decoded frames. 114488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private static final int MAX_QUEUED_OUTPUTBUFFERS = 3; 115488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private int droppedFrames; 116488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private Surface surface = null; 117488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final Queue<DecodedOutputBuffer> 118488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per dequeuedSurfaceOutputBuffers = new LinkedList<DecodedOutputBuffer>(); 119efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 1205c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev // MediaCodec error handler - invoked when critical error happens which may prevent 1215c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev // further use of media codec API. Now it means that one of media codec instances 1225c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev // is hanging and can no longer be used in the next call. 1235c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev public static interface MediaCodecVideoDecoderErrorCallback { 1245c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev void onMediaCodecVideoDecoderCriticalError(int codecErrors); 1255c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } 1265c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 1275c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorCallback) { 1285c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.d(TAG, "Set error callback"); 1295c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev MediaCodecVideoDecoder.errorCallback = errorCallback; 1305c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } 1315c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 13225cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org // Helper struct for findVp8Decoder() below. 133efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private static class DecoderProperties { 134996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org public DecoderProperties(String codecName, int colorFormat) { 135efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org this.codecName = codecName; 136efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org this.colorFormat = colorFormat; 137efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 138efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org public final String codecName; // OpenMax component name for VP8 codec. 139efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org public final int colorFormat; // Color format supported by codec. 140efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 141efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 142b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org private static DecoderProperties findDecoder( 143b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org String mime, String[] supportedCodecPrefixes) { 14425cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { 145efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return null; // MediaCodec.setParameters is missing. 14625cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org } 14769a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev Logging.d(TAG, "Trying to find HW decoder for mime " + mime); 148efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { 149efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); 150efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (info.isEncoder()) { 151efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org continue; 152efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 153efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org String name = null; 154efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org for (String mimeType : info.getSupportedTypes()) { 155b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org if (mimeType.equals(mime)) { 156efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org name = info.getName(); 157efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org break; 158efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 159efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 160efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (name == null) { 161b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org continue; // No HW support in this codec; try the next one. 162efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 16369a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev Logging.d(TAG, "Found candidate decoder " + name); 164996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 16525cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org // Check if this is supported decoder. 166996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org boolean supportedCodec = false; 16725cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org for (String codecPrefix : supportedCodecPrefixes) { 16825cc745d6b92962cec76abf30488e0a4cac36c98glaznev@webrtc.org if (name.startsWith(codecPrefix)) { 169996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org supportedCodec = true; 170996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org break; 171996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 172996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 173996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (!supportedCodec) { 174996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org continue; 175996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 176996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 177996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org // Check if codec supports either yuv420 or nv12. 178efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org CodecCapabilities capabilities = 179b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org info.getCapabilitiesForType(mime); 180efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org for (int colorFormat : capabilities.colorFormats) { 1815975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); 182efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 183996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org for (int supportedColorFormat : supportedColorList) { 184996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org for (int codecColorFormat : capabilities.colorFormats) { 185996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (codecColorFormat == supportedColorFormat) { 186b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org // Found supported HW decoder. 1875975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Found target decoder " + name + 188996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org ". Color: 0x" + Integer.toHexString(codecColorFormat)); 189996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org return new DecoderProperties(name, codecColorFormat); 190efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 191efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 192efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 193efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 19469a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev Logging.d(TAG, "No HW decoder found for mime " + mime); 195b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org return null; // No HW decoder. 196efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 197efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 198908e77bd00593a8bc5480fb1a07368b7c27778f3Alex Glaznev public static boolean isVp8HwSupported() { 199b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org return findDecoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null; 200b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org } 201b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org 20269a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev public static boolean isVp9HwSupported() { 20369a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev return findDecoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes) != null; 20469a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev } 20569a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev 206908e77bd00593a8bc5480fb1a07368b7c27778f3Alex Glaznev public static boolean isH264HwSupported() { 207b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org return findDecoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null; 208efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 209efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 210325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev public static void printStackTrace() { 211c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev if (runningInstance != null && runningInstance.mediaCodecThread != null) { 212c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev StackTraceElement[] mediaCodecStackTraces = runningInstance.mediaCodecThread.getStackTrace(); 213325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev if (mediaCodecStackTraces.length > 0) { 214325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev Logging.d(TAG, "MediaCodecVideoDecoder stacks trace:"); 215325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev for (StackTraceElement stackTrace : mediaCodecStackTraces) { 216325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev Logging.d(TAG, stackTrace.toString()); 217325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev } 218325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev } 219325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev } 220325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev } 221325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev 2227e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert private void checkOnMediaCodecThread() throws IllegalStateException { 223efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (mediaCodecThread.getId() != Thread.currentThread().getId()) { 2247e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert throw new IllegalStateException( 225efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + 226efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org " but is now called on " + Thread.currentThread()); 227efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 228efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 229efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 230488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output. 231488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private boolean initDecode( 232488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) { 23369ddaefbb39975fde3e0bb43e233c357f514213cAlejandro Luebs if (mediaCodecThread != null) { 234efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org throw new RuntimeException("Forgot to release()?"); 235efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 236488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per useSurface = (surfaceTextureHelper != null); 237b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org String mime = null; 238b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org String[] supportedCodecPrefixes = null; 239b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org if (type == VideoCodecType.VIDEO_CODEC_VP8) { 240b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org mime = VP8_MIME_TYPE; 241b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org supportedCodecPrefixes = supportedVp8HwCodecPrefixes; 24269a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { 24369a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev mime = VP9_MIME_TYPE; 24469a7fd50476a60ec3def8552993bebef83ed9c58Alex Glaznev supportedCodecPrefixes = supportedVp9HwCodecPrefixes; 245b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org } else if (type == VideoCodecType.VIDEO_CODEC_H264) { 246b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org mime = H264_MIME_TYPE; 247b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org supportedCodecPrefixes = supportedH264HwCodecPrefixes; 248b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org } else { 249b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org throw new RuntimeException("Non supported codec " + type); 250b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org } 251b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); 252efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (properties == null) { 253b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org throw new RuntimeException("Cannot find HW decoder for " + type); 254efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 2555975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + 256996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org ". Color: 0x" + Integer.toHexString(properties.colorFormat) + 257782671f7983e27be865f04f546fbfc3f1cb4e1b3Alex Glaznev ". Use Surface: " + useSurface); 258c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev runningInstance = this; // Decoder is now running and can be queried for stack traces. 259efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org mediaCodecThread = Thread.currentThread(); 260efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org try { 261efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org this.width = width; 262efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org this.height = height; 263efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org stride = width; 264efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org sliceHeight = height; 265996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 266996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (useSurface) { 267488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per textureListener = new TextureListener(surfaceTextureHelper); 268488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); 26980cf97cddd9c67fddb8cd9f78e7d560a2c0deec0Magnus Jedvert } 270996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 271b28474c7a0356f21b374f43a51602ed10f143bf4glaznev@webrtc.org MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); 272996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (!useSurface) { 273996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); 274996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 2755975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, " Format: " + format); 276528fc650d8546defdc0549a13b9c177d4981d7d1henrike@webrtc.org mediaCodec = 277528fc650d8546defdc0549a13b9c177d4981d7d1henrike@webrtc.org MediaCodecVideoEncoder.createByCodecName(properties.codecName); 278efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (mediaCodec == null) { 279325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev Logging.e(TAG, "Can not create media decoder"); 280efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return false; 281efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 2827e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert mediaCodec.configure(format, surface, null, 0); 283efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org mediaCodec.start(); 284efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org colorFormat = properties.colorFormat; 285efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org outputBuffers = mediaCodec.getOutputBuffers(); 286efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org inputBuffers = mediaCodec.getInputBuffers(); 287488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per decodeStartTimeMs.clear(); 288488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per hasDecodedFirstFrame = false; 289488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per dequeuedSurfaceOutputBuffers.clear(); 290488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per droppedFrames = 0; 2915975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Input buffers: " + inputBuffers.length + 292efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org ". Output buffers: " + outputBuffers.length); 293efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return true; 294efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } catch (IllegalStateException e) { 2955975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.e(TAG, "initDecode failed", e); 296efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return false; 297efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 298efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 299efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 300efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private void release() { 301488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames); 302efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org checkOnMediaCodecThread(); 3035c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 3045c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev // Run Mediacodec stop() and release() on separate thread since sometime 3055c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev // Mediacodec.stop() may hang. 3065c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev final CountDownLatch releaseDone = new CountDownLatch(1); 3075c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 3085c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Runnable runMediaCodecRelease = new Runnable() { 3095c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev @Override 3105c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev public void run() { 3115c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev try { 3125c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.d(TAG, "Java releaseDecoder on release thread"); 3135c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev mediaCodec.stop(); 3145c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev mediaCodec.release(); 3155c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.d(TAG, "Java releaseDecoder on release thread done"); 3165c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } catch (Exception e) { 3175c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.e(TAG, "Media decoder release failed", e); 3185c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } 3195c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev releaseDone.countDown(); 3205c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } 3215c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev }; 3225c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev new Thread(runMediaCodecRelease).start(); 3235c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 3245c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { 3255c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.e(TAG, "Media decoder release timeout"); 3265c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev codecErrors++; 3275c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev if (errorCallback != null) { 3285c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); 3295c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); 3305c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev } 331efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 3325c3da4b6e9bdc787a3c1f25c0ff0c16a4271fd26Alex Glaznev 333efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org mediaCodec = null; 334efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org mediaCodecThread = null; 335c6aec4b8edba5c003966a53f20b5e26cb8c7b8ceAlex Glaznev runningInstance = null; 336996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (useSurface) { 337996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org surface.release(); 3387e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert surface = null; 339488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per textureListener.release(); 340996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 341325d414e8cc0990328c49c1d1f4a977f82f90e6aAlex Glaznev Logging.d(TAG, "Java releaseDecoder done"); 342efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 343efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 344efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // Dequeue an input buffer and return its index, -1 if no input buffer is 345efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org // available, or -2 if the codec is no longer operative. 346efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org private int dequeueInputBuffer() { 347efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org checkOnMediaCodecThread(); 348efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org try { 349996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); 350efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } catch (IllegalStateException e) { 3515975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.e(TAG, "dequeueIntputBuffer failed", e); 352efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return -2; 353efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 354efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 355efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 356488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private boolean queueInputBuffer(int inputBufferIndex, int size, long presentationTimeStamUs, 357488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per long timeStampMs, long ntpTimeStamp) { 358efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org checkOnMediaCodecThread(); 359efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org try { 360efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org inputBuffers[inputBufferIndex].position(0); 361efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org inputBuffers[inputBufferIndex].limit(size); 362488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per decodeStartTimeMs.add(new TimeStamps(SystemClock.elapsedRealtime(), timeStampMs, 363488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per ntpTimeStamp)); 364488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, presentationTimeStamUs, 0); 365efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return true; 366efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 367efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org catch (IllegalStateException e) { 3685975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.e(TAG, "decode failed", e); 369efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org return false; 370efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 371efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 372efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 373488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private static class TimeStamps { 374488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public TimeStamps(long decodeStartTimeMs, long timeStampMs, long ntpTimeStampMs) { 375488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.decodeStartTimeMs = decodeStartTimeMs; 376488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.timeStampMs = timeStampMs; 377488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.ntpTimeStampMs = ntpTimeStampMs; 378488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 379488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long decodeStartTimeMs; // Time when this frame was queued for decoding. 380488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long timeStampMs; // Only used for bookkeeping in Java. Used in C++; 381488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long ntpTimeStampMs; // Only used for bookkeeping in Java. Used in C++; 382488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 383488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 384488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Helper struct for dequeueOutputBuffer() below. 385488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private static class DecodedOutputBuffer { 386488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public DecodedOutputBuffer(int index, int offset, int size, long timeStampMs, 387488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per long ntpTimeStampMs, long decodeTime, long endDecodeTime) { 388996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org this.index = index; 389996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org this.offset = offset; 390996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org this.size = size; 391488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.timeStampMs = timeStampMs; 392488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.ntpTimeStampMs = ntpTimeStampMs; 393488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.decodeTimeMs = decodeTime; 394488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.endDecodeTimeMs = endDecodeTime; 395996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 396996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 397996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org private final int index; 398996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org private final int offset; 399996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org private final int size; 400488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long timeStampMs; 401488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long ntpTimeStampMs; 402488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Number of ms it took to decode this frame. 403488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long decodeTimeMs; 404488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // System time when this frame finished decoding. 405488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long endDecodeTimeMs; 406996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org } 407996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org 408488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Helper struct for dequeueTextureBuffer() below. 40944bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed private static class DecodedTextureBuffer { 41044bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed private final int textureID; 411488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final float[] transformMatrix; 412488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long timeStampMs; 413488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long ntpTimeStampMs; 414488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long decodeTimeMs; 415488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Interval from when the frame finished decoding until this buffer has been created. 416488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Since there is only one texture, this interval depend on the time from when 417488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // a frame is decoded and provided to C++ and until that frame is returned to the MediaCodec 418488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // so that the texture can be updated with the next decoded frame. 419488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final long frameDelayMs; 420488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 421488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame 422488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // that was dropped. 423488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public DecodedTextureBuffer(int textureID, float[] transformMatrix, long timeStampMs, 424488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per long ntpTimeStampMs, long decodeTimeMs, long frameDelay) { 42544bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed this.textureID = textureID; 426488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.transformMatrix = transformMatrix; 427488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.timeStampMs = timeStampMs; 428488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.ntpTimeStampMs = ntpTimeStampMs; 429488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.decodeTimeMs = decodeTimeMs; 430488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.frameDelayMs = frameDelay; 431488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 432488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 433488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 434488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Poll based texture listener. 435488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private static class TextureListener 436488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per implements SurfaceTextureHelper.OnTextureFrameAvailableListener { 437488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final SurfaceTextureHelper surfaceTextureHelper; 438488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll(). 439488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private final Object newFrameLock = new Object(); 440488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // |bufferToRender| is non-null when waiting for transition between addBufferToRender() to 441488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // onTextureFrameAvailable(). 442488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private DecodedOutputBuffer bufferToRender; 443488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private DecodedTextureBuffer renderedBuffer; 444488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 445488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { 446488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per this.surfaceTextureHelper = surfaceTextureHelper; 447488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per surfaceTextureHelper.setListener(this); 448488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 449488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 450488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public void addBufferToRender(DecodedOutputBuffer buffer) { 451488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (bufferToRender != null) { 452488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per Logging.e(TAG, 453488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per "Unexpected addBufferToRender() called while waiting for a texture."); 454488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new IllegalStateException("Waiting for a texture."); 455488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 456488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per bufferToRender = buffer; 457488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 458488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 459488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public boolean isWaitingForTexture() { 460488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per synchronized (newFrameLock) { 461488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return bufferToRender != null; 462488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 463488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 464488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 465488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Callback from |surfaceTextureHelper|. May be called on an arbitrary thread. 466488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per @Override 467488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public void onTextureFrameAvailable( 468488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per int oesTextureId, float[] transformMatrix, long timestampNs) { 469488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per synchronized (newFrameLock) { 470488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (renderedBuffer != null) { 471488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per Logging.e(TAG, 472488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per "Unexpected onTextureFrameAvailable() called while already holding a texture."); 473488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new IllegalStateException("Already holding a texture."); 474488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 475488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // |timestampNs| is always zero on some Android versions. 476488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix, 477488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per bufferToRender.timeStampMs, bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs, 478488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs); 479488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per bufferToRender = null; 480488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per newFrameLock.notifyAll(); 481488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 482488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 483488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 484488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Dequeues and returns a DecodedTextureBuffer if available, or null otherwise. 485488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public DecodedTextureBuffer dequeueTextureBuffer(int timeoutMs) { 486488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per synchronized (newFrameLock) { 487488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (renderedBuffer == null && timeoutMs > 0 && isWaitingForTexture()) { 488488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per try { 489488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per newFrameLock.wait(timeoutMs); 490488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } catch(InterruptedException e) { 491488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Restore the interrupted status by reinterrupting the thread. 492488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per Thread.currentThread().interrupt(); 493488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 494488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 495488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per DecodedTextureBuffer returnedBuffer = renderedBuffer; 496488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per renderedBuffer = null; 497488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return returnedBuffer; 498488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 499488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 500488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 501488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per public void release() { 502488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // SurfaceTextureHelper.disconnect() will block until any onTextureFrameAvailable() in 503488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // progress is done. Therefore, the call to disconnect() must be outside any synchronized 504488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // statement that is also used in the onTextureFrameAvailable() above to avoid deadlocks. 505488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per surfaceTextureHelper.disconnect(); 506488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per synchronized (newFrameLock) { 507488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (renderedBuffer != null) { 508488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per surfaceTextureHelper.returnTextureFrame(); 509488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per renderedBuffer = null; 510488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 511488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 51244bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed } 51344bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed } 51444bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed 515488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Returns null if no decoded buffer is available, and otherwise a DecodedByteBuffer. 5167e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an 5177e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException 5187e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert // upon codec error. 519488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { 520efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org checkOnMediaCodecThread(); 521488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (decodeStartTimeMs.isEmpty()) { 522488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return null; 523488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 5247e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert // Drain the decoder until receiving a decoded buffer or hitting 5257e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert // MediaCodec.INFO_TRY_AGAIN_LATER. 5267e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 5277e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert while (true) { 528488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per final int result = mediaCodec.dequeueOutputBuffer( 529488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); 5307e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert switch (result) { 5317e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: 532efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org outputBuffers = mediaCodec.getOutputBuffers(); 5335975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.length); 534488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (hasDecodedFirstFrame) { 535488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new RuntimeException("Unexpected output buffer change event."); 536488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 5377e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert break; 5387e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: 539efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org MediaFormat format = mediaCodec.getOutputFormat(); 5405975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Decoder format changed: " + format.toString()); 541488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per int new_width = format.getInteger(MediaFormat.KEY_WIDTH); 542488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per int new_height = format.getInteger(MediaFormat.KEY_HEIGHT); 543488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (hasDecodedFirstFrame && (new_width != width || new_height != height)) { 544488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new RuntimeException("Unexpected size change. Configured " + width + "*" + 545488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per height + ". New " + new_width + "*" + new_height); 546488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 547efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org width = format.getInteger(MediaFormat.KEY_WIDTH); 548efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org height = format.getInteger(MediaFormat.KEY_HEIGHT); 549488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 550996784548d5024d97f795e330298ed71c68629e8glaznev@webrtc.org if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { 551efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); 5525975b3c5be44de32ee28c34ca577474e36259e6eJiayang Liu Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); 5537e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert if (!supportedColorList.contains(colorFormat)) { 5547e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert throw new IllegalStateException("Non supported color format: " + colorFormat); 555efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 556efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 557efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (format.containsKey("stride")) { 558efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org stride = format.getInteger("stride"); 559efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 560efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org if (format.containsKey("slice-height")) { 561efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org sliceHeight = format.getInteger("slice-height"); 562efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 5637e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); 564efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org stride = Math.max(width, stride); 565efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org sliceHeight = Math.max(height, sliceHeight); 5667e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert break; 567488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per case MediaCodec.INFO_TRY_AGAIN_LATER: 568488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return null; 5697e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert default: 570488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per hasDecodedFirstFrame = true; 571488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per TimeStamps timeStamps = decodeStartTimeMs.remove(); 572488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return new DecodedOutputBuffer(result, info.offset, info.size, timeStamps.timeStampMs, 573488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per timeStamps.ntpTimeStampMs, 574488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per SystemClock.elapsedRealtime() - timeStamps.decodeStartTimeMs, 575488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per SystemClock.elapsedRealtime()); 576488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 577488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 578488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 579488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 580488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Returns null if no decoded buffer is available, and otherwise a DecodedTextureBuffer. 581488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an 582488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException 583488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // upon codec error. If |dequeueTimeoutMs| > 0, the oldest decoded frame will be dropped if 584488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // a frame can't be returned. 585488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { 586488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per checkOnMediaCodecThread(); 587488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (!useSurface) { 588488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new IllegalStateException("dequeueTexture() called for byte buffer decoding."); 589488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 590488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); 591488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (outputBuffer != null) { 592488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per dequeuedSurfaceOutputBuffers.add(outputBuffer); 593488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 594488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 595488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per MaybeRenderDecodedTextureBuffer(); 596488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Check if there is texture ready now by waiting max |dequeueTimeoutMs|. 597488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per DecodedTextureBuffer renderedBuffer = textureListener.dequeueTextureBuffer(dequeueTimeoutMs); 598488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (renderedBuffer != null) { 599488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per MaybeRenderDecodedTextureBuffer(); 600488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return renderedBuffer; 601488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 602488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 603488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if ((dequeuedSurfaceOutputBuffers.size() 604488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per >= Math.min(MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length) 605488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per || (dequeueTimeoutMs > 0 && !dequeuedSurfaceOutputBuffers.isEmpty()))) { 606488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per ++droppedFrames; 607488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Drop the oldest frame still in dequeuedSurfaceOutputBuffers. 608488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // The oldest frame is owned by |textureListener| and can't be dropped since 609488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // mediaCodec.releaseOutputBuffer has already been called. 610488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per final DecodedOutputBuffer droppedFrame = dequeuedSurfaceOutputBuffers.remove(); 611488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (dequeueTimeoutMs > 0) { 6127baf79fb9ece918f8ad4768725529af5a367e0d7perkj // TODO(perkj): Re-add the below log when VideoRenderGUI has been removed or fixed to 6137baf79fb9ece918f8ad4768725529af5a367e0d7perkj // return the one and only texture even if it does not render. 6147baf79fb9ece918f8ad4768725529af5a367e0d7perkj // Logging.w(TAG, "Draining decoder. Dropping frame with TS: " 6157baf79fb9ece918f8ad4768725529af5a367e0d7perkj // + droppedFrame.timeStampMs + ". Total number of dropped frames: " + droppedFrames); 616488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } else { 617488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per Logging.w(TAG, "Too many output buffers. Dropping frame with TS: " 618488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per + droppedFrame.timeStampMs + ". Total number of dropped frames: " + droppedFrames); 6199cb8982e64f08d3d630bf7c3d2bcc78c10db88e2perkj } 620488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 621488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */); 622488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return new DecodedTextureBuffer(0, null, droppedFrame.timeStampMs, 623488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs, 624488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs); 625488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 626488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return null; 627488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per } 628488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per 629488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private void MaybeRenderDecodedTextureBuffer() { 630488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per if (dequeuedSurfaceOutputBuffers.isEmpty() || textureListener.isWaitingForTexture()) { 631488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per return; 632efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 633488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per // Get the first frame in the queue and render to the decoder output surface. 634488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per final DecodedOutputBuffer buffer = dequeuedSurfaceOutputBuffers.remove(); 635488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per textureListener.addBufferToRender(buffer); 636488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per mediaCodec.releaseOutputBuffer(buffer.index, true /* render */); 637efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 638efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org 63944bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for 64044bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed // non-surface decoding. 64144bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed // Throws IllegalStateException if the call is made on the wrong thread, if codec is configured 64244bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed // for surface decoding, or if |mediaCodec| is not in the Executing state. Throws 64344bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed // MediaCodec.CodecException upon codec error. 644488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per private void returnDecodedOutputBuffer(int index) 6457e319372abd4403565aa3d27a2d1bfa069969b19Magnus Jedvert throws IllegalStateException, MediaCodec.CodecException { 646efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org checkOnMediaCodecThread(); 64744bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed if (useSurface) { 648488e75f11b840dfbe636a9ea9bbc18252e7c59f0Per throw new IllegalStateException("returnDecodedOutputBuffer() called for surface decoding."); 64944bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed } 65044bf6f5f67ab6b6677e3cd1cceb954c2426f930amagjed mediaCodec.releaseOutputBuffer(index, false /* render */); 651efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org } 652efe4b9af49d2dbbf39a2f41b5818829a38fa0d5eglaznev@webrtc.org} 653