1/* 2 * libjingle 3 * Copyright 2013, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 29package org.webrtc; 30 31import android.media.MediaCodec; 32import android.media.MediaCodecInfo; 33import android.media.MediaCodecList; 34import android.media.MediaFormat; 35import android.media.MediaCodecInfo.CodecCapabilities; 36import android.os.Build; 37import android.os.Bundle; 38import android.util.Log; 39 40import java.nio.ByteBuffer; 41 42// Java-side of peerconnection_jni.cc:MediaCodecVideoEncoder. 43// This class is an implementation detail of the Java PeerConnection API. 44// MediaCodec is thread-hostile so this class must be operated on a single 45// thread. 46class MediaCodecVideoEncoder { 47 // This class is constructed, operated, and destroyed by its C++ incarnation, 48 // so the class and its methods have non-public visibility. The API this 49 // class exposes aims to mimic the webrtc::VideoEncoder API as closely as 50 // possibly to minimize the amount of translation work necessary. 51 52 private static final String TAG = "MediaCodecVideoEncoder"; 53 54 private static final int DEQUEUE_TIMEOUT = 0; // Non-blocking, no wait. 55 private Thread mediaCodecThread; 56 private MediaCodec mediaCodec; 57 private ByteBuffer[] outputBuffers; 58 private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; 59 // List of supported HW VP8 codecs. 60 private static final String[] supportedHwCodecPrefixes = 61 {"OMX.qcom.", "OMX.Nvidia." }; 62 // Bitrate mode 63 private static final int VIDEO_ControlRateConstant = 2; 64 // NV12 color format supported by QCOM codec, but not declared in MediaCodec - 65 // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h 66 private static final int 67 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; 68 // Allowable color formats supported by codec - in order of preference. 69 private static final int[] supportedColorList = { 70 CodecCapabilities.COLOR_FormatYUV420Planar, 71 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, 72 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, 73 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m 74 }; 75 private int colorFormat; 76 77 private MediaCodecVideoEncoder() {} 78 79 // Helper struct for findVp8HwEncoder() below. 80 private static class EncoderProperties { 81 EncoderProperties(String codecName, int colorFormat) { 82 this.codecName = codecName; 83 this.colorFormat = colorFormat; 84 } 85 public final String codecName; // OpenMax component name for VP8 codec. 86 public final int colorFormat; // Color format supported by codec. 87 } 88 89 private static EncoderProperties findVp8HwEncoder() { 90 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) 91 return null; // MediaCodec.setParameters is missing. 92 93 for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { 94 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); 95 if (!info.isEncoder()) { 96 continue; 97 } 98 String name = null; 99 for (String mimeType : info.getSupportedTypes()) { 100 if (mimeType.equals(VP8_MIME_TYPE)) { 101 name = info.getName(); 102 break; 103 } 104 } 105 if (name == null) { 106 continue; // No VP8 support in this codec; try the next one. 107 } 108 Log.d(TAG, "Found candidate encoder " + name); 109 CodecCapabilities capabilities = 110 info.getCapabilitiesForType(VP8_MIME_TYPE); 111 for (int colorFormat : capabilities.colorFormats) { 112 Log.d(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); 113 } 114 115 // Check if this is supported HW encoder 116 for (String hwCodecPrefix : supportedHwCodecPrefixes) { 117 if (!name.startsWith(hwCodecPrefix)) { 118 continue; 119 } 120 // Check if codec supports either yuv420 or nv12 121 for (int supportedColorFormat : supportedColorList) { 122 for (int codecColorFormat : capabilities.colorFormats) { 123 if (codecColorFormat == supportedColorFormat) { 124 // Found supported HW VP8 encoder 125 Log.d(TAG, "Found target encoder " + name + 126 ". Color: 0x" + Integer.toHexString(codecColorFormat)); 127 return new EncoderProperties(name, codecColorFormat); 128 } 129 } 130 } 131 } 132 } 133 return null; // No HW VP8 encoder. 134 } 135 136 private static boolean isPlatformSupported() { 137 return findVp8HwEncoder() != null; 138 } 139 140 private static int bitRate(int kbps) { 141 // webrtc "kilo" means 1000, not 1024. Apparently. 142 // (and the price for overshooting is frame-dropping as webrtc enforces its 143 // bandwidth estimation, which is unpleasant). 144 // Since the HW encoder in the N5 overshoots, we clamp to a bit less than 145 // the requested rate. Sad but true. Bug 3194. 146 return kbps * 950; 147 } 148 149 private void checkOnMediaCodecThread() { 150 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { 151 throw new RuntimeException( 152 "MediaCodecVideoEncoder previously operated on " + mediaCodecThread + 153 " but is now called on " + Thread.currentThread()); 154 } 155 } 156 157 // Return the array of input buffers, or null on failure. 158 private ByteBuffer[] initEncode(int width, int height, int kbps, int fps) { 159 Log.d(TAG, "initEncode: " + width + " x " + height + 160 ". @ " + kbps + " kbps. Fps: " + fps + 161 ". Color: 0x" + Integer.toHexString(colorFormat)); 162 if (mediaCodecThread != null) { 163 throw new RuntimeException("Forgot to release()?"); 164 } 165 EncoderProperties properties = findVp8HwEncoder(); 166 if (properties == null) { 167 throw new RuntimeException("Can not find HW VP8 encoder"); 168 } 169 mediaCodecThread = Thread.currentThread(); 170 try { 171 MediaFormat format = 172 MediaFormat.createVideoFormat(VP8_MIME_TYPE, width, height); 173 format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate(kbps)); 174 format.setInteger("bitrate-mode", VIDEO_ControlRateConstant); 175 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); 176 // Default WebRTC settings 177 format.setInteger(MediaFormat.KEY_FRAME_RATE, fps); 178 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 100); 179 Log.d(TAG, " Format: " + format); 180 mediaCodec = MediaCodec.createByCodecName(properties.codecName); 181 if (mediaCodec == null) { 182 return null; 183 } 184 mediaCodec.configure( 185 format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 186 mediaCodec.start(); 187 colorFormat = properties.colorFormat; 188 outputBuffers = mediaCodec.getOutputBuffers(); 189 ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers(); 190 Log.d(TAG, "Input buffers: " + inputBuffers.length + 191 ". Output buffers: " + outputBuffers.length); 192 return inputBuffers; 193 } catch (IllegalStateException e) { 194 Log.e(TAG, "initEncode failed", e); 195 return null; 196 } 197 } 198 199 private boolean encode( 200 boolean isKeyframe, int inputBuffer, int size, 201 long presentationTimestampUs) { 202 checkOnMediaCodecThread(); 203 try { 204 if (isKeyframe) { 205 // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could 206 // indicate this in queueInputBuffer() below and guarantee _this_ frame 207 // be encoded as a key frame, but sadly that flag is ignored. Instead, 208 // we request a key frame "soon". 209 Log.d(TAG, "Sync frame request"); 210 Bundle b = new Bundle(); 211 b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); 212 mediaCodec.setParameters(b); 213 } 214 mediaCodec.queueInputBuffer( 215 inputBuffer, 0, size, presentationTimestampUs, 0); 216 return true; 217 } 218 catch (IllegalStateException e) { 219 Log.e(TAG, "encode failed", e); 220 return false; 221 } 222 } 223 224 private void release() { 225 Log.d(TAG, "release"); 226 checkOnMediaCodecThread(); 227 try { 228 mediaCodec.stop(); 229 mediaCodec.release(); 230 } catch (IllegalStateException e) { 231 Log.e(TAG, "release failed", e); 232 } 233 mediaCodec = null; 234 mediaCodecThread = null; 235 } 236 237 private boolean setRates(int kbps, int frameRateIgnored) { 238 // frameRate argument is ignored - HW encoder is supposed to use 239 // video frame timestamps for bit allocation. 240 checkOnMediaCodecThread(); 241 Log.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRateIgnored); 242 try { 243 Bundle params = new Bundle(); 244 params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, bitRate(kbps)); 245 mediaCodec.setParameters(params); 246 return true; 247 } catch (IllegalStateException e) { 248 Log.e(TAG, "setRates failed", e); 249 return false; 250 } 251 } 252 253 // Dequeue an input buffer and return its index, -1 if no input buffer is 254 // available, or -2 if the codec is no longer operative. 255 private int dequeueInputBuffer() { 256 checkOnMediaCodecThread(); 257 try { 258 return mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT); 259 } catch (IllegalStateException e) { 260 Log.e(TAG, "dequeueIntputBuffer failed", e); 261 return -2; 262 } 263 } 264 265 // Helper struct for dequeueOutputBuffer() below. 266 private static class OutputBufferInfo { 267 public OutputBufferInfo( 268 int index, ByteBuffer buffer, boolean isKeyFrame, 269 long presentationTimestampUs) { 270 this.index = index; 271 this.buffer = buffer; 272 this.isKeyFrame = isKeyFrame; 273 this.presentationTimestampUs = presentationTimestampUs; 274 } 275 276 private final int index; 277 private final ByteBuffer buffer; 278 private final boolean isKeyFrame; 279 private final long presentationTimestampUs; 280 } 281 282 // Dequeue and return an output buffer, or null if no output is ready. Return 283 // a fake OutputBufferInfo with index -1 if the codec is no longer operable. 284 private OutputBufferInfo dequeueOutputBuffer() { 285 checkOnMediaCodecThread(); 286 try { 287 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 288 int result = mediaCodec.dequeueOutputBuffer(info, DEQUEUE_TIMEOUT); 289 if (result >= 0) { 290 // MediaCodec doesn't care about Buffer position/remaining/etc so we can 291 // mess with them to get a slice and avoid having to pass extra 292 // (BufferInfo-related) parameters back to C++. 293 ByteBuffer outputBuffer = outputBuffers[result].duplicate(); 294 outputBuffer.position(info.offset); 295 outputBuffer.limit(info.offset + info.size); 296 boolean isKeyFrame = 297 (info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0; 298 if (isKeyFrame) { 299 Log.d(TAG, "Sync frame generated"); 300 } 301 return new OutputBufferInfo( 302 result, outputBuffer.slice(), isKeyFrame, info.presentationTimeUs); 303 } else if (result == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 304 outputBuffers = mediaCodec.getOutputBuffers(); 305 return dequeueOutputBuffer(); 306 } else if (result == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 307 return dequeueOutputBuffer(); 308 } else if (result == MediaCodec.INFO_TRY_AGAIN_LATER) { 309 return null; 310 } 311 throw new RuntimeException("dequeueOutputBuffer: " + result); 312 } catch (IllegalStateException e) { 313 Log.e(TAG, "dequeueOutputBuffer failed", e); 314 return new OutputBufferInfo(-1, null, false, -1); 315 } 316 } 317 318 // Release a dequeued output buffer back to the codec for re-use. Return 319 // false if the codec is no longer operable. 320 private boolean releaseOutputBuffer(int index) { 321 checkOnMediaCodecThread(); 322 try { 323 mediaCodec.releaseOutputBuffer(index, false); 324 return true; 325 } catch (IllegalStateException e) { 326 Log.e(TAG, "releaseOutputBuffer failed", e); 327 return false; 328 } 329 } 330} 331