1/* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h" 13 14#if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) 15 16#include "libyuv/convert.h" 17#include "webrtc/base/checks.h" 18#include "webrtc/base/logging.h" 19#include "webrtc/common_video/include/video_frame_buffer.h" 20#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" 21#include "webrtc/video_frame.h" 22 23namespace internal { 24 25// Convenience function for creating a dictionary. 26inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, 27 CFTypeRef* values, 28 size_t size) { 29 return CFDictionaryCreate(nullptr, keys, values, size, 30 &kCFTypeDictionaryKeyCallBacks, 31 &kCFTypeDictionaryValueCallBacks); 32} 33 34// Struct that we pass to the decoder per frame to decode. We receive it again 35// in the decoder callback. 36struct FrameDecodeParams { 37 FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts) 38 : callback(cb), timestamp(ts) {} 39 webrtc::DecodedImageCallback* callback; 40 int64_t timestamp; 41}; 42 43// On decode we receive a CVPixelBuffer, which we need to convert to a frame 44// buffer for use in the rest of WebRTC. Unfortunately this involves a frame 45// copy. 46// TODO(tkchin): Stuff CVPixelBuffer into a TextureBuffer and pass that along 47// instead once the pipeline supports it. 48rtc::scoped_refptr<webrtc::VideoFrameBuffer> VideoFrameBufferForPixelBuffer( 49 CVPixelBufferRef pixel_buffer) { 50 RTC_DCHECK(pixel_buffer); 51 RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == 52 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); 53 size_t width = CVPixelBufferGetWidthOfPlane(pixel_buffer, 0); 54 size_t height = CVPixelBufferGetHeightOfPlane(pixel_buffer, 0); 55 // TODO(tkchin): Use a frame buffer pool. 56 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = 57 new rtc::RefCountedObject<webrtc::I420Buffer>(width, height); 58 CVPixelBufferLockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly); 59 const uint8_t* src_y = reinterpret_cast<const uint8_t*>( 60 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); 61 int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); 62 const uint8_t* src_uv = reinterpret_cast<const uint8_t*>( 63 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); 64 int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); 65 int ret = libyuv::NV12ToI420( 66 src_y, src_y_stride, src_uv, src_uv_stride, 67 buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane), 68 buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane), 69 buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane), 70 width, height); 71 CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly); 72 if (ret) { 73 LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret; 74 return nullptr; 75 } 76 return buffer; 77} 78 79// This is the callback function that VideoToolbox calls when decode is 80// complete. 81void VTDecompressionOutputCallback(void* decoder, 82 void* params, 83 OSStatus status, 84 VTDecodeInfoFlags info_flags, 85 CVImageBufferRef image_buffer, 86 CMTime timestamp, 87 CMTime duration) { 88 rtc::scoped_ptr<FrameDecodeParams> decode_params( 89 reinterpret_cast<FrameDecodeParams*>(params)); 90 if (status != noErr) { 91 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; 92 return; 93 } 94 // TODO(tkchin): Handle CVO properly. 95 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = 96 VideoFrameBufferForPixelBuffer(image_buffer); 97 webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, 0, 98 webrtc::kVideoRotation_0); 99 decode_params->callback->Decoded(decoded_frame); 100} 101 102} // namespace internal 103 104namespace webrtc { 105 106H264VideoToolboxDecoder::H264VideoToolboxDecoder() 107 : callback_(nullptr), 108 video_format_(nullptr), 109 decompression_session_(nullptr) {} 110 111H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { 112 DestroyDecompressionSession(); 113 SetVideoFormat(nullptr); 114} 115 116int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, 117 int number_of_cores) { 118 return WEBRTC_VIDEO_CODEC_OK; 119} 120 121int H264VideoToolboxDecoder::Decode( 122 const EncodedImage& input_image, 123 bool missing_frames, 124 const RTPFragmentationHeader* fragmentation, 125 const CodecSpecificInfo* codec_specific_info, 126 int64_t render_time_ms) { 127 RTC_DCHECK(input_image._buffer); 128 129 CMSampleBufferRef sample_buffer = nullptr; 130 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, 131 input_image._length, video_format_, 132 &sample_buffer)) { 133 return WEBRTC_VIDEO_CODEC_ERROR; 134 } 135 RTC_DCHECK(sample_buffer); 136 // Check if the video format has changed, and reinitialize decoder if needed. 137 CMVideoFormatDescriptionRef description = 138 CMSampleBufferGetFormatDescription(sample_buffer); 139 if (!CMFormatDescriptionEqual(description, video_format_)) { 140 SetVideoFormat(description); 141 ResetDecompressionSession(); 142 } 143 VTDecodeFrameFlags decode_flags = 144 kVTDecodeFrame_EnableAsynchronousDecompression; 145 rtc::scoped_ptr<internal::FrameDecodeParams> frame_decode_params; 146 frame_decode_params.reset( 147 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); 148 OSStatus status = VTDecompressionSessionDecodeFrame( 149 decompression_session_, sample_buffer, decode_flags, 150 frame_decode_params.release(), nullptr); 151 CFRelease(sample_buffer); 152 if (status != noErr) { 153 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; 154 return WEBRTC_VIDEO_CODEC_ERROR; 155 } 156 return WEBRTC_VIDEO_CODEC_OK; 157} 158 159int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( 160 DecodedImageCallback* callback) { 161 RTC_DCHECK(!callback_); 162 callback_ = callback; 163 return WEBRTC_VIDEO_CODEC_OK; 164} 165 166int H264VideoToolboxDecoder::Release() { 167 callback_ = nullptr; 168 return WEBRTC_VIDEO_CODEC_OK; 169} 170 171int H264VideoToolboxDecoder::Reset() { 172 ResetDecompressionSession(); 173 return WEBRTC_VIDEO_CODEC_OK; 174} 175 176int H264VideoToolboxDecoder::ResetDecompressionSession() { 177 DestroyDecompressionSession(); 178 179 // Need to wait for the first SPS to initialize decoder. 180 if (!video_format_) { 181 return WEBRTC_VIDEO_CODEC_OK; 182 } 183 184 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder 185 // create pixel buffers with GPU backed memory. The intent here is to pass 186 // the pixel buffers directly so we avoid a texture upload later during 187 // rendering. This currently is moot because we are converting back to an 188 // I420 frame after decode, but eventually we will be able to plumb 189 // CVPixelBuffers directly to the renderer. 190 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that 191 // we can pass CVPixelBuffers as native handles in decoder output. 192 static size_t const attributes_size = 3; 193 CFTypeRef keys[attributes_size] = { 194#if defined(WEBRTC_IOS) 195 kCVPixelBufferOpenGLESCompatibilityKey, 196#elif defined(WEBRTC_MAC) 197 kCVPixelBufferOpenGLCompatibilityKey, 198#endif 199 kCVPixelBufferIOSurfacePropertiesKey, 200 kCVPixelBufferPixelFormatTypeKey 201 }; 202 CFDictionaryRef io_surface_value = 203 internal::CreateCFDictionary(nullptr, nullptr, 0); 204 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; 205 CFNumberRef pixel_format = 206 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); 207 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, 208 pixel_format}; 209 CFDictionaryRef attributes = 210 internal::CreateCFDictionary(keys, values, attributes_size); 211 if (io_surface_value) { 212 CFRelease(io_surface_value); 213 io_surface_value = nullptr; 214 } 215 if (pixel_format) { 216 CFRelease(pixel_format); 217 pixel_format = nullptr; 218 } 219 VTDecompressionOutputCallbackRecord record = { 220 internal::VTDecompressionOutputCallback, this, 221 }; 222 OSStatus status = 223 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, 224 &record, &decompression_session_); 225 CFRelease(attributes); 226 if (status != noErr) { 227 DestroyDecompressionSession(); 228 return WEBRTC_VIDEO_CODEC_ERROR; 229 } 230 ConfigureDecompressionSession(); 231 232 return WEBRTC_VIDEO_CODEC_OK; 233} 234 235void H264VideoToolboxDecoder::ConfigureDecompressionSession() { 236 RTC_DCHECK(decompression_session_); 237#if defined(WEBRTC_IOS) 238 VTSessionSetProperty(decompression_session_, 239 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); 240#endif 241} 242 243void H264VideoToolboxDecoder::DestroyDecompressionSession() { 244 if (decompression_session_) { 245 VTDecompressionSessionInvalidate(decompression_session_); 246 decompression_session_ = nullptr; 247 } 248} 249 250void H264VideoToolboxDecoder::SetVideoFormat( 251 CMVideoFormatDescriptionRef video_format) { 252 if (video_format_ == video_format) { 253 return; 254 } 255 if (video_format_) { 256 CFRelease(video_format_); 257 } 258 video_format_ = video_format; 259 if (video_format_) { 260 CFRetain(video_format_); 261 } 262} 263 264const char* H264VideoToolboxDecoder::ImplementationName() const { 265 return "VideoToolbox"; 266} 267 268} // namespace webrtc 269 270#endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) 271