1// libjingle 2// Copyright 2011 Google Inc. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are met: 6// 7// 1. Redistributions of source code must retain the above copyright notice, 8// this list of conditions and the following disclaimer. 9// 2. Redistributions in binary form must reproduce the above copyright notice, 10// this list of conditions and the following disclaimer in the documentation 11// and/or other materials provided with the distribution. 12// 3. The name of the author may not be used to endorse or promote products 13// derived from this software without specific prior written permission. 14// 15// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25// 26// Implementation of class WebRtcVideoCapturer. 27 28#include "talk/media/webrtc/webrtcvideocapturer.h" 29 30#ifdef HAVE_CONFIG_H 31#include <config.h> 32#endif 33 34#ifdef HAVE_WEBRTC_VIDEO 35#include "talk/media/webrtc/webrtcvideoframe.h" 36#include "talk/media/webrtc/webrtcvideoframefactory.h" 37#include "webrtc/base/criticalsection.h" 38#include "webrtc/base/logging.h" 39#include "webrtc/base/thread.h" 40#include "webrtc/base/timeutils.h" 41 42#include "webrtc/base/win32.h" // Need this to #include the impl files. 43#include "webrtc/modules/video_capture/include/video_capture_factory.h" 44 45namespace cricket { 46 47struct kVideoFourCCEntry { 48 uint32 fourcc; 49 webrtc::RawVideoType webrtc_type; 50}; 51 52// This indicates our format preferences and defines a mapping between 53// webrtc::RawVideoType (from video_capture_defines.h) to our FOURCCs. 54static kVideoFourCCEntry kSupportedFourCCs[] = { 55 { FOURCC_I420, webrtc::kVideoI420 }, // 12 bpp, no conversion. 56 { FOURCC_YV12, webrtc::kVideoYV12 }, // 12 bpp, no conversion. 57 { FOURCC_YUY2, webrtc::kVideoYUY2 }, // 16 bpp, fast conversion. 58 { FOURCC_UYVY, webrtc::kVideoUYVY }, // 16 bpp, fast conversion. 59 { FOURCC_NV12, webrtc::kVideoNV12 }, // 12 bpp, fast conversion. 60 { FOURCC_NV21, webrtc::kVideoNV21 }, // 12 bpp, fast conversion. 61 { FOURCC_MJPG, webrtc::kVideoMJPEG }, // compressed, slow conversion. 62 { FOURCC_ARGB, webrtc::kVideoARGB }, // 32 bpp, slow conversion. 63 { FOURCC_24BG, webrtc::kVideoRGB24 }, // 24 bpp, slow conversion. 64}; 65 66class WebRtcVcmFactory : public WebRtcVcmFactoryInterface { 67 public: 68 virtual webrtc::VideoCaptureModule* Create(int id, const char* device) { 69 return webrtc::VideoCaptureFactory::Create(id, device); 70 } 71 virtual webrtc::VideoCaptureModule::DeviceInfo* CreateDeviceInfo(int id) { 72 return webrtc::VideoCaptureFactory::CreateDeviceInfo(id); 73 } 74 virtual void DestroyDeviceInfo(webrtc::VideoCaptureModule::DeviceInfo* info) { 75 delete info; 76 } 77}; 78 79static bool CapabilityToFormat(const webrtc::VideoCaptureCapability& cap, 80 VideoFormat* format) { 81 uint32 fourcc = 0; 82 for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) { 83 if (kSupportedFourCCs[i].webrtc_type == cap.rawType) { 84 fourcc = kSupportedFourCCs[i].fourcc; 85 break; 86 } 87 } 88 if (fourcc == 0) { 89 return false; 90 } 91 92 format->fourcc = fourcc; 93 format->width = cap.width; 94 format->height = cap.height; 95 format->interval = VideoFormat::FpsToInterval(cap.maxFPS); 96 return true; 97} 98 99static bool FormatToCapability(const VideoFormat& format, 100 webrtc::VideoCaptureCapability* cap) { 101 webrtc::RawVideoType webrtc_type = webrtc::kVideoUnknown; 102 for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) { 103 if (kSupportedFourCCs[i].fourcc == format.fourcc) { 104 webrtc_type = kSupportedFourCCs[i].webrtc_type; 105 break; 106 } 107 } 108 if (webrtc_type == webrtc::kVideoUnknown) { 109 return false; 110 } 111 112 cap->width = format.width; 113 cap->height = format.height; 114 cap->maxFPS = VideoFormat::IntervalToFps(format.interval); 115 cap->expectedCaptureDelay = 0; 116 cap->rawType = webrtc_type; 117 cap->codecType = webrtc::kVideoCodecUnknown; 118 cap->interlaced = false; 119 return true; 120} 121 122/////////////////////////////////////////////////////////////////////////// 123// Implementation of class WebRtcVideoCapturer 124/////////////////////////////////////////////////////////////////////////// 125 126WebRtcVideoCapturer::WebRtcVideoCapturer() 127 : factory_(new WebRtcVcmFactory), 128 module_(NULL), 129 captured_frames_(0) { 130 set_frame_factory(new WebRtcVideoFrameFactory()); 131} 132 133WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory) 134 : factory_(factory), 135 module_(NULL), 136 captured_frames_(0) { 137 set_frame_factory(new WebRtcVideoFrameFactory()); 138} 139 140WebRtcVideoCapturer::~WebRtcVideoCapturer() { 141 if (module_) { 142 module_->Release(); 143 } 144} 145 146bool WebRtcVideoCapturer::Init(const Device& device) { 147 if (module_) { 148 LOG(LS_ERROR) << "The capturer is already initialized"; 149 return false; 150 } 151 152 webrtc::VideoCaptureModule::DeviceInfo* info = factory_->CreateDeviceInfo(0); 153 if (!info) { 154 return false; 155 } 156 157 // Find the desired camera, by name. 158 // In the future, comparing IDs will be more robust. 159 // TODO(juberti): Figure what's needed to allow this. 160 int num_cams = info->NumberOfDevices(); 161 char vcm_id[256] = ""; 162 bool found = false; 163 for (int index = 0; index < num_cams; ++index) { 164 char vcm_name[256]; 165 if (info->GetDeviceName(index, vcm_name, ARRAY_SIZE(vcm_name), 166 vcm_id, ARRAY_SIZE(vcm_id)) != -1) { 167 if (device.name == reinterpret_cast<char*>(vcm_name)) { 168 found = true; 169 break; 170 } 171 } 172 } 173 if (!found) { 174 LOG(LS_WARNING) << "Failed to find capturer for id: " << device.id; 175 factory_->DestroyDeviceInfo(info); 176 return false; 177 } 178 179 // Enumerate the supported formats. 180 // TODO(juberti): Find out why this starts/stops the camera... 181 std::vector<VideoFormat> supported; 182 int32_t num_caps = info->NumberOfCapabilities(vcm_id); 183 for (int32_t i = 0; i < num_caps; ++i) { 184 webrtc::VideoCaptureCapability cap; 185 if (info->GetCapability(vcm_id, i, cap) != -1) { 186 VideoFormat format; 187 if (CapabilityToFormat(cap, &format)) { 188 supported.push_back(format); 189 } else { 190 LOG(LS_WARNING) << "Ignoring unsupported WebRTC capture format " 191 << cap.rawType; 192 } 193 } 194 } 195 factory_->DestroyDeviceInfo(info); 196// TODO(fischman): Remove the following check 197// when capabilities for iOS are implemented 198// https://code.google.com/p/webrtc/issues/detail?id=2968 199#if !defined(IOS) 200 if (supported.empty()) { 201 LOG(LS_ERROR) << "Failed to find usable formats for id: " << device.id; 202 return false; 203 } 204#endif 205 module_ = factory_->Create(0, vcm_id); 206 if (!module_) { 207 LOG(LS_ERROR) << "Failed to create capturer for id: " << device.id; 208 return false; 209 } 210 211 // It is safe to change member attributes now. 212 module_->AddRef(); 213 SetId(device.id); 214 SetSupportedFormats(supported); 215 return true; 216} 217 218bool WebRtcVideoCapturer::Init(webrtc::VideoCaptureModule* module) { 219 if (module_) { 220 LOG(LS_ERROR) << "The capturer is already initialized"; 221 return false; 222 } 223 if (!module) { 224 LOG(LS_ERROR) << "Invalid VCM supplied"; 225 return false; 226 } 227 // TODO(juberti): Set id and formats. 228 (module_ = module)->AddRef(); 229 return true; 230} 231 232bool WebRtcVideoCapturer::GetBestCaptureFormat(const VideoFormat& desired, 233 VideoFormat* best_format) { 234 if (!best_format) { 235 return false; 236 } 237 238 if (!VideoCapturer::GetBestCaptureFormat(desired, best_format)) { 239 // We maybe using a manually injected VCM which doesn't support enum. 240 // Use the desired format as the best format. 241 best_format->width = desired.width; 242 best_format->height = desired.height; 243 best_format->fourcc = FOURCC_I420; 244 best_format->interval = desired.interval; 245 LOG(LS_INFO) << "Failed to find best capture format," 246 << " fall back to the requested format " 247 << best_format->ToString(); 248 } 249 return true; 250} 251 252CaptureState WebRtcVideoCapturer::Start(const VideoFormat& capture_format) { 253 if (!module_) { 254 LOG(LS_ERROR) << "The capturer has not been initialized"; 255 return CS_NO_DEVICE; 256 } 257 258 rtc::CritScope cs(&critical_section_stopping_); 259 // TODO(hellner): weird to return failure when it is in fact actually running. 260 if (IsRunning()) { 261 LOG(LS_ERROR) << "The capturer is already running"; 262 return CS_FAILED; 263 } 264 265 SetCaptureFormat(&capture_format); 266 267 webrtc::VideoCaptureCapability cap; 268 if (!FormatToCapability(capture_format, &cap)) { 269 LOG(LS_ERROR) << "Invalid capture format specified"; 270 return CS_FAILED; 271 } 272 273 std::string camera_id(GetId()); 274 uint32 start = rtc::Time(); 275 module_->RegisterCaptureDataCallback(*this); 276 if (module_->StartCapture(cap) != 0) { 277 LOG(LS_ERROR) << "Camera '" << camera_id << "' failed to start"; 278 return CS_FAILED; 279 } 280 281 LOG(LS_INFO) << "Camera '" << camera_id << "' started with format " 282 << capture_format.ToString() << ", elapsed time " 283 << rtc::TimeSince(start) << " ms"; 284 285 captured_frames_ = 0; 286 SetCaptureState(CS_RUNNING); 287 return CS_STARTING; 288} 289 290// Critical section blocks Stop from shutting down during callbacks from capture 291// thread to OnIncomingCapturedFrame. Note that the crit is try-locked in 292// OnFrameCaptured, as the lock ordering between this and the system component 293// controlling the camera is reversed: system frame -> OnIncomingCapturedFrame; 294// Stop -> system stop camera). 295void WebRtcVideoCapturer::Stop() { 296 rtc::CritScope cs(&critical_section_stopping_); 297 if (IsRunning()) { 298 rtc::Thread::Current()->Clear(this); 299 module_->StopCapture(); 300 module_->DeRegisterCaptureDataCallback(); 301 302 // TODO(juberti): Determine if the VCM exposes any drop stats we can use. 303 double drop_ratio = 0.0; 304 std::string camera_id(GetId()); 305 LOG(LS_INFO) << "Camera '" << camera_id << "' stopped after capturing " 306 << captured_frames_ << " frames and dropping " 307 << drop_ratio << "%"; 308 } 309 SetCaptureFormat(NULL); 310} 311 312bool WebRtcVideoCapturer::IsRunning() { 313 return (module_ != NULL && module_->CaptureStarted()); 314} 315 316bool WebRtcVideoCapturer::GetPreferredFourccs( 317 std::vector<uint32>* fourccs) { 318 if (!fourccs) { 319 return false; 320 } 321 322 fourccs->clear(); 323 for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) { 324 fourccs->push_back(kSupportedFourCCs[i].fourcc); 325 } 326 return true; 327} 328 329void WebRtcVideoCapturer::OnIncomingCapturedFrame(const int32_t id, 330 webrtc::I420VideoFrame& sample) { 331 // This would be a normal CritScope, except that it's possible that: 332 // (1) whatever system component producing this frame has taken a lock, and 333 // (2) Stop() probably calls back into that system component, which may take 334 // the same lock. Due to the reversed order, we have to try-lock in order to 335 // avoid a potential deadlock. Besides, if we can't enter because we're 336 // stopping, we may as well drop the frame. 337 rtc::TryCritScope cs(&critical_section_stopping_); 338 if (!cs.locked() || !IsRunning()) { 339 // Capturer has been stopped or is in the process of stopping. 340 return; 341 } 342 343 ++captured_frames_; 344 // Log the size and pixel aspect ratio of the first captured frame. 345 if (1 == captured_frames_) { 346 LOG(LS_INFO) << "Captured frame size " 347 << sample.width() << "x" << sample.height() 348 << ". Expected format " << GetCaptureFormat()->ToString(); 349 } 350 351 // Signal down stream components on captured frame. 352 // The CapturedFrame class doesn't support planes. We have to ExtractBuffer 353 // to one block for it. 354 int length = webrtc::CalcBufferSize(webrtc::kI420, 355 sample.width(), sample.height()); 356 capture_buffer_.resize(length); 357 // TODO(ronghuawu): Refactor the WebRtcCapturedFrame to avoid memory copy. 358 webrtc::ExtractBuffer(sample, length, &capture_buffer_[0]); 359 WebRtcCapturedFrame frame(sample, &capture_buffer_[0], length); 360 SignalFrameCaptured(this, &frame); 361} 362 363void WebRtcVideoCapturer::OnCaptureDelayChanged(const int32_t id, 364 const int32_t delay) { 365 LOG(LS_INFO) << "Capture delay changed to " << delay << " ms"; 366} 367 368// WebRtcCapturedFrame 369WebRtcCapturedFrame::WebRtcCapturedFrame(const webrtc::I420VideoFrame& sample, 370 void* buffer, 371 int length) { 372 width = sample.width(); 373 height = sample.height(); 374 fourcc = FOURCC_I420; 375 // TODO(hellner): Support pixel aspect ratio (for OSX). 376 pixel_width = 1; 377 pixel_height = 1; 378 // Convert units from VideoFrame RenderTimeMs to CapturedFrame (nanoseconds). 379 elapsed_time = sample.render_time_ms() * rtc::kNumNanosecsPerMillisec; 380 time_stamp = elapsed_time; 381 data_size = length; 382 data = buffer; 383} 384 385} // namespace cricket 386 387#endif // HAVE_WEBRTC_VIDEO 388