1/*
2 *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
12#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h"
13
14#if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
15
16#include "libyuv/convert.h"
17#include "webrtc/base/checks.h"
18#include "webrtc/base/logging.h"
19#include "webrtc/common_video/include/video_frame_buffer.h"
20#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
21#include "webrtc/video_frame.h"
22
23namespace internal {
24
25// Convenience function for creating a dictionary.
26inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
27                                          CFTypeRef* values,
28                                          size_t size) {
29  return CFDictionaryCreate(nullptr, keys, values, size,
30                            &kCFTypeDictionaryKeyCallBacks,
31                            &kCFTypeDictionaryValueCallBacks);
32}
33
34// Struct that we pass to the decoder per frame to decode. We receive it again
35// in the decoder callback.
36struct FrameDecodeParams {
37  FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts)
38      : callback(cb), timestamp(ts) {}
39  webrtc::DecodedImageCallback* callback;
40  int64_t timestamp;
41};
42
43// On decode we receive a CVPixelBuffer, which we need to convert to a frame
44// buffer for use in the rest of WebRTC. Unfortunately this involves a frame
45// copy.
46// TODO(tkchin): Stuff CVPixelBuffer into a TextureBuffer and pass that along
47// instead once the pipeline supports it.
48rtc::scoped_refptr<webrtc::VideoFrameBuffer> VideoFrameBufferForPixelBuffer(
49    CVPixelBufferRef pixel_buffer) {
50  RTC_DCHECK(pixel_buffer);
51  RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) ==
52             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
53  size_t width = CVPixelBufferGetWidthOfPlane(pixel_buffer, 0);
54  size_t height = CVPixelBufferGetHeightOfPlane(pixel_buffer, 0);
55  // TODO(tkchin): Use a frame buffer pool.
56  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
57      new rtc::RefCountedObject<webrtc::I420Buffer>(width, height);
58  CVPixelBufferLockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
59  const uint8_t* src_y = reinterpret_cast<const uint8_t*>(
60      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
61  int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
62  const uint8_t* src_uv = reinterpret_cast<const uint8_t*>(
63      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
64  int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
65  int ret = libyuv::NV12ToI420(
66      src_y, src_y_stride, src_uv, src_uv_stride,
67      buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
68      buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
69      buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
70      width, height);
71  CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
72  if (ret) {
73    LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret;
74    return nullptr;
75  }
76  return buffer;
77}
78
79// This is the callback function that VideoToolbox calls when decode is
80// complete.
81void VTDecompressionOutputCallback(void* decoder,
82                                   void* params,
83                                   OSStatus status,
84                                   VTDecodeInfoFlags info_flags,
85                                   CVImageBufferRef image_buffer,
86                                   CMTime timestamp,
87                                   CMTime duration) {
88  rtc::scoped_ptr<FrameDecodeParams> decode_params(
89      reinterpret_cast<FrameDecodeParams*>(params));
90  if (status != noErr) {
91    LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
92    return;
93  }
94  // TODO(tkchin): Handle CVO properly.
95  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
96      VideoFrameBufferForPixelBuffer(image_buffer);
97  webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, 0,
98                                   webrtc::kVideoRotation_0);
99  decode_params->callback->Decoded(decoded_frame);
100}
101
102}  // namespace internal
103
104namespace webrtc {
105
106H264VideoToolboxDecoder::H264VideoToolboxDecoder()
107    : callback_(nullptr),
108      video_format_(nullptr),
109      decompression_session_(nullptr) {}
110
111H264VideoToolboxDecoder::~H264VideoToolboxDecoder() {
112  DestroyDecompressionSession();
113  SetVideoFormat(nullptr);
114}
115
116int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec,
117                                        int number_of_cores) {
118  return WEBRTC_VIDEO_CODEC_OK;
119}
120
121int H264VideoToolboxDecoder::Decode(
122    const EncodedImage& input_image,
123    bool missing_frames,
124    const RTPFragmentationHeader* fragmentation,
125    const CodecSpecificInfo* codec_specific_info,
126    int64_t render_time_ms) {
127  RTC_DCHECK(input_image._buffer);
128
129  CMSampleBufferRef sample_buffer = nullptr;
130  if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer,
131                                        input_image._length, video_format_,
132                                        &sample_buffer)) {
133    return WEBRTC_VIDEO_CODEC_ERROR;
134  }
135  RTC_DCHECK(sample_buffer);
136  // Check if the video format has changed, and reinitialize decoder if needed.
137  CMVideoFormatDescriptionRef description =
138      CMSampleBufferGetFormatDescription(sample_buffer);
139  if (!CMFormatDescriptionEqual(description, video_format_)) {
140    SetVideoFormat(description);
141    ResetDecompressionSession();
142  }
143  VTDecodeFrameFlags decode_flags =
144      kVTDecodeFrame_EnableAsynchronousDecompression;
145  rtc::scoped_ptr<internal::FrameDecodeParams> frame_decode_params;
146  frame_decode_params.reset(
147      new internal::FrameDecodeParams(callback_, input_image._timeStamp));
148  OSStatus status = VTDecompressionSessionDecodeFrame(
149      decompression_session_, sample_buffer, decode_flags,
150      frame_decode_params.release(), nullptr);
151  CFRelease(sample_buffer);
152  if (status != noErr) {
153    LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
154    return WEBRTC_VIDEO_CODEC_ERROR;
155  }
156  return WEBRTC_VIDEO_CODEC_OK;
157}
158
159int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback(
160    DecodedImageCallback* callback) {
161  RTC_DCHECK(!callback_);
162  callback_ = callback;
163  return WEBRTC_VIDEO_CODEC_OK;
164}
165
166int H264VideoToolboxDecoder::Release() {
167  callback_ = nullptr;
168  return WEBRTC_VIDEO_CODEC_OK;
169}
170
171int H264VideoToolboxDecoder::Reset() {
172  ResetDecompressionSession();
173  return WEBRTC_VIDEO_CODEC_OK;
174}
175
176int H264VideoToolboxDecoder::ResetDecompressionSession() {
177  DestroyDecompressionSession();
178
179  // Need to wait for the first SPS to initialize decoder.
180  if (!video_format_) {
181    return WEBRTC_VIDEO_CODEC_OK;
182  }
183
184  // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
185  // create pixel buffers with GPU backed memory. The intent here is to pass
186  // the pixel buffers directly so we avoid a texture upload later during
187  // rendering. This currently is moot because we are converting back to an
188  // I420 frame after decode, but eventually we will be able to plumb
189  // CVPixelBuffers directly to the renderer.
190  // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
191  // we can pass CVPixelBuffers as native handles in decoder output.
192  static size_t const attributes_size = 3;
193  CFTypeRef keys[attributes_size] = {
194#if defined(WEBRTC_IOS)
195    kCVPixelBufferOpenGLESCompatibilityKey,
196#elif defined(WEBRTC_MAC)
197    kCVPixelBufferOpenGLCompatibilityKey,
198#endif
199    kCVPixelBufferIOSurfacePropertiesKey,
200    kCVPixelBufferPixelFormatTypeKey
201  };
202  CFDictionaryRef io_surface_value =
203      internal::CreateCFDictionary(nullptr, nullptr, 0);
204  int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
205  CFNumberRef pixel_format =
206      CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
207  CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
208                                       pixel_format};
209  CFDictionaryRef attributes =
210      internal::CreateCFDictionary(keys, values, attributes_size);
211  if (io_surface_value) {
212    CFRelease(io_surface_value);
213    io_surface_value = nullptr;
214  }
215  if (pixel_format) {
216    CFRelease(pixel_format);
217    pixel_format = nullptr;
218  }
219  VTDecompressionOutputCallbackRecord record = {
220      internal::VTDecompressionOutputCallback, this,
221  };
222  OSStatus status =
223      VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes,
224                                   &record, &decompression_session_);
225  CFRelease(attributes);
226  if (status != noErr) {
227    DestroyDecompressionSession();
228    return WEBRTC_VIDEO_CODEC_ERROR;
229  }
230  ConfigureDecompressionSession();
231
232  return WEBRTC_VIDEO_CODEC_OK;
233}
234
235void H264VideoToolboxDecoder::ConfigureDecompressionSession() {
236  RTC_DCHECK(decompression_session_);
237#if defined(WEBRTC_IOS)
238  VTSessionSetProperty(decompression_session_,
239                       kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
240#endif
241}
242
243void H264VideoToolboxDecoder::DestroyDecompressionSession() {
244  if (decompression_session_) {
245    VTDecompressionSessionInvalidate(decompression_session_);
246    decompression_session_ = nullptr;
247  }
248}
249
250void H264VideoToolboxDecoder::SetVideoFormat(
251    CMVideoFormatDescriptionRef video_format) {
252  if (video_format_ == video_format) {
253    return;
254  }
255  if (video_format_) {
256    CFRelease(video_format_);
257  }
258  video_format_ = video_format;
259  if (video_format_) {
260    CFRetain(video_format_);
261  }
262}
263
264const char* H264VideoToolboxDecoder::ImplementationName() const {
265  return "VideoToolbox";
266}
267
268}  // namespace webrtc
269
270#endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
271