1// libjingle
2// Copyright 2011 Google Inc.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//  1. Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//  2. Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//  3. The name of the author may not be used to endorse or promote products
13//     derived from this software without specific prior written permission.
14//
15// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25//
26// Implementation of class WebRtcVideoCapturer.
27
28#include "talk/media/webrtc/webrtcvideocapturer.h"
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#ifdef HAVE_WEBRTC_VIDEO
35#include "talk/media/webrtc/webrtcvideoframe.h"
36#include "talk/media/webrtc/webrtcvideoframefactory.h"
37#include "webrtc/base/criticalsection.h"
38#include "webrtc/base/logging.h"
39#include "webrtc/base/thread.h"
40#include "webrtc/base/timeutils.h"
41
42#include "webrtc/base/win32.h"  // Need this to #include the impl files.
43#include "webrtc/modules/video_capture/include/video_capture_factory.h"
44
45namespace cricket {
46
47struct kVideoFourCCEntry {
48  uint32 fourcc;
49  webrtc::RawVideoType webrtc_type;
50};
51
52// This indicates our format preferences and defines a mapping between
53// webrtc::RawVideoType (from video_capture_defines.h) to our FOURCCs.
54static kVideoFourCCEntry kSupportedFourCCs[] = {
55  { FOURCC_I420, webrtc::kVideoI420 },   // 12 bpp, no conversion.
56  { FOURCC_YV12, webrtc::kVideoYV12 },   // 12 bpp, no conversion.
57  { FOURCC_YUY2, webrtc::kVideoYUY2 },   // 16 bpp, fast conversion.
58  { FOURCC_UYVY, webrtc::kVideoUYVY },   // 16 bpp, fast conversion.
59  { FOURCC_NV12, webrtc::kVideoNV12 },   // 12 bpp, fast conversion.
60  { FOURCC_NV21, webrtc::kVideoNV21 },   // 12 bpp, fast conversion.
61  { FOURCC_MJPG, webrtc::kVideoMJPEG },  // compressed, slow conversion.
62  { FOURCC_ARGB, webrtc::kVideoARGB },   // 32 bpp, slow conversion.
63  { FOURCC_24BG, webrtc::kVideoRGB24 },  // 24 bpp, slow conversion.
64};
65
66class WebRtcVcmFactory : public WebRtcVcmFactoryInterface {
67 public:
68  virtual webrtc::VideoCaptureModule* Create(int id, const char* device) {
69    return webrtc::VideoCaptureFactory::Create(id, device);
70  }
71  virtual webrtc::VideoCaptureModule::DeviceInfo* CreateDeviceInfo(int id) {
72    return webrtc::VideoCaptureFactory::CreateDeviceInfo(id);
73  }
74  virtual void DestroyDeviceInfo(webrtc::VideoCaptureModule::DeviceInfo* info) {
75    delete info;
76  }
77};
78
79static bool CapabilityToFormat(const webrtc::VideoCaptureCapability& cap,
80                               VideoFormat* format) {
81  uint32 fourcc = 0;
82  for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) {
83    if (kSupportedFourCCs[i].webrtc_type == cap.rawType) {
84      fourcc = kSupportedFourCCs[i].fourcc;
85      break;
86    }
87  }
88  if (fourcc == 0) {
89    return false;
90  }
91
92  format->fourcc = fourcc;
93  format->width = cap.width;
94  format->height = cap.height;
95  format->interval = VideoFormat::FpsToInterval(cap.maxFPS);
96  return true;
97}
98
99static bool FormatToCapability(const VideoFormat& format,
100                               webrtc::VideoCaptureCapability* cap) {
101  webrtc::RawVideoType webrtc_type = webrtc::kVideoUnknown;
102  for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) {
103    if (kSupportedFourCCs[i].fourcc == format.fourcc) {
104      webrtc_type = kSupportedFourCCs[i].webrtc_type;
105      break;
106    }
107  }
108  if (webrtc_type == webrtc::kVideoUnknown) {
109    return false;
110  }
111
112  cap->width = format.width;
113  cap->height = format.height;
114  cap->maxFPS = VideoFormat::IntervalToFps(format.interval);
115  cap->expectedCaptureDelay = 0;
116  cap->rawType = webrtc_type;
117  cap->codecType = webrtc::kVideoCodecUnknown;
118  cap->interlaced = false;
119  return true;
120}
121
122///////////////////////////////////////////////////////////////////////////
123// Implementation of class WebRtcVideoCapturer
124///////////////////////////////////////////////////////////////////////////
125
126WebRtcVideoCapturer::WebRtcVideoCapturer()
127    : factory_(new WebRtcVcmFactory),
128      module_(NULL),
129      captured_frames_(0) {
130  set_frame_factory(new WebRtcVideoFrameFactory());
131}
132
133WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory)
134    : factory_(factory),
135      module_(NULL),
136      captured_frames_(0) {
137  set_frame_factory(new WebRtcVideoFrameFactory());
138}
139
140WebRtcVideoCapturer::~WebRtcVideoCapturer() {
141  if (module_) {
142    module_->Release();
143  }
144}
145
146bool WebRtcVideoCapturer::Init(const Device& device) {
147  if (module_) {
148    LOG(LS_ERROR) << "The capturer is already initialized";
149    return false;
150  }
151
152  webrtc::VideoCaptureModule::DeviceInfo* info = factory_->CreateDeviceInfo(0);
153  if (!info) {
154    return false;
155  }
156
157  // Find the desired camera, by name.
158  // In the future, comparing IDs will be more robust.
159  // TODO(juberti): Figure what's needed to allow this.
160  int num_cams = info->NumberOfDevices();
161  char vcm_id[256] = "";
162  bool found = false;
163  for (int index = 0; index < num_cams; ++index) {
164    char vcm_name[256];
165    if (info->GetDeviceName(index, vcm_name, ARRAY_SIZE(vcm_name),
166                            vcm_id, ARRAY_SIZE(vcm_id)) != -1) {
167      if (device.name == reinterpret_cast<char*>(vcm_name)) {
168        found = true;
169        break;
170      }
171    }
172  }
173  if (!found) {
174    LOG(LS_WARNING) << "Failed to find capturer for id: " << device.id;
175    factory_->DestroyDeviceInfo(info);
176    return false;
177  }
178
179  // Enumerate the supported formats.
180  // TODO(juberti): Find out why this starts/stops the camera...
181  std::vector<VideoFormat> supported;
182  int32_t num_caps = info->NumberOfCapabilities(vcm_id);
183  for (int32_t i = 0; i < num_caps; ++i) {
184    webrtc::VideoCaptureCapability cap;
185    if (info->GetCapability(vcm_id, i, cap) != -1) {
186      VideoFormat format;
187      if (CapabilityToFormat(cap, &format)) {
188        supported.push_back(format);
189      } else {
190        LOG(LS_WARNING) << "Ignoring unsupported WebRTC capture format "
191                        << cap.rawType;
192      }
193    }
194  }
195  factory_->DestroyDeviceInfo(info);
196// TODO(fischman): Remove the following check
197// when capabilities for iOS are implemented
198// https://code.google.com/p/webrtc/issues/detail?id=2968
199#if !defined(IOS)
200  if (supported.empty()) {
201    LOG(LS_ERROR) << "Failed to find usable formats for id: " << device.id;
202    return false;
203  }
204#endif
205  module_ = factory_->Create(0, vcm_id);
206  if (!module_) {
207    LOG(LS_ERROR) << "Failed to create capturer for id: " << device.id;
208    return false;
209  }
210
211  // It is safe to change member attributes now.
212  module_->AddRef();
213  SetId(device.id);
214  SetSupportedFormats(supported);
215  return true;
216}
217
218bool WebRtcVideoCapturer::Init(webrtc::VideoCaptureModule* module) {
219  if (module_) {
220    LOG(LS_ERROR) << "The capturer is already initialized";
221    return false;
222  }
223  if (!module) {
224    LOG(LS_ERROR) << "Invalid VCM supplied";
225    return false;
226  }
227  // TODO(juberti): Set id and formats.
228  (module_ = module)->AddRef();
229  return true;
230}
231
232bool WebRtcVideoCapturer::GetBestCaptureFormat(const VideoFormat& desired,
233                                               VideoFormat* best_format) {
234  if (!best_format) {
235    return false;
236  }
237
238  if (!VideoCapturer::GetBestCaptureFormat(desired, best_format)) {
239    // We maybe using a manually injected VCM which doesn't support enum.
240    // Use the desired format as the best format.
241    best_format->width = desired.width;
242    best_format->height = desired.height;
243    best_format->fourcc = FOURCC_I420;
244    best_format->interval = desired.interval;
245    LOG(LS_INFO) << "Failed to find best capture format,"
246                 << " fall back to the requested format "
247                 << best_format->ToString();
248  }
249  return true;
250}
251
252CaptureState WebRtcVideoCapturer::Start(const VideoFormat& capture_format) {
253  if (!module_) {
254    LOG(LS_ERROR) << "The capturer has not been initialized";
255    return CS_NO_DEVICE;
256  }
257
258  rtc::CritScope cs(&critical_section_stopping_);
259  // TODO(hellner): weird to return failure when it is in fact actually running.
260  if (IsRunning()) {
261    LOG(LS_ERROR) << "The capturer is already running";
262    return CS_FAILED;
263  }
264
265  SetCaptureFormat(&capture_format);
266
267  webrtc::VideoCaptureCapability cap;
268  if (!FormatToCapability(capture_format, &cap)) {
269    LOG(LS_ERROR) << "Invalid capture format specified";
270    return CS_FAILED;
271  }
272
273  std::string camera_id(GetId());
274  uint32 start = rtc::Time();
275  module_->RegisterCaptureDataCallback(*this);
276  if (module_->StartCapture(cap) != 0) {
277    LOG(LS_ERROR) << "Camera '" << camera_id << "' failed to start";
278    return CS_FAILED;
279  }
280
281  LOG(LS_INFO) << "Camera '" << camera_id << "' started with format "
282               << capture_format.ToString() << ", elapsed time "
283               << rtc::TimeSince(start) << " ms";
284
285  captured_frames_ = 0;
286  SetCaptureState(CS_RUNNING);
287  return CS_STARTING;
288}
289
290// Critical section blocks Stop from shutting down during callbacks from capture
291// thread to OnIncomingCapturedFrame. Note that the crit is try-locked in
292// OnFrameCaptured, as the lock ordering between this and the system component
293// controlling the camera is reversed: system frame -> OnIncomingCapturedFrame;
294// Stop -> system stop camera).
295void WebRtcVideoCapturer::Stop() {
296  rtc::CritScope cs(&critical_section_stopping_);
297  if (IsRunning()) {
298    rtc::Thread::Current()->Clear(this);
299    module_->StopCapture();
300    module_->DeRegisterCaptureDataCallback();
301
302    // TODO(juberti): Determine if the VCM exposes any drop stats we can use.
303    double drop_ratio = 0.0;
304    std::string camera_id(GetId());
305    LOG(LS_INFO) << "Camera '" << camera_id << "' stopped after capturing "
306                 << captured_frames_ << " frames and dropping "
307                 << drop_ratio << "%";
308  }
309  SetCaptureFormat(NULL);
310}
311
312bool WebRtcVideoCapturer::IsRunning() {
313  return (module_ != NULL && module_->CaptureStarted());
314}
315
316bool WebRtcVideoCapturer::GetPreferredFourccs(
317    std::vector<uint32>* fourccs) {
318  if (!fourccs) {
319    return false;
320  }
321
322  fourccs->clear();
323  for (size_t i = 0; i < ARRAY_SIZE(kSupportedFourCCs); ++i) {
324    fourccs->push_back(kSupportedFourCCs[i].fourcc);
325  }
326  return true;
327}
328
329void WebRtcVideoCapturer::OnIncomingCapturedFrame(const int32_t id,
330    webrtc::I420VideoFrame& sample) {
331  // This would be a normal CritScope, except that it's possible that:
332  // (1) whatever system component producing this frame has taken a lock, and
333  // (2) Stop() probably calls back into that system component, which may take
334  // the same lock. Due to the reversed order, we have to try-lock in order to
335  // avoid a potential deadlock. Besides, if we can't enter because we're
336  // stopping, we may as well drop the frame.
337  rtc::TryCritScope cs(&critical_section_stopping_);
338  if (!cs.locked() || !IsRunning()) {
339    // Capturer has been stopped or is in the process of stopping.
340    return;
341  }
342
343  ++captured_frames_;
344  // Log the size and pixel aspect ratio of the first captured frame.
345  if (1 == captured_frames_) {
346    LOG(LS_INFO) << "Captured frame size "
347                 << sample.width() << "x" << sample.height()
348                 << ". Expected format " << GetCaptureFormat()->ToString();
349  }
350
351  // Signal down stream components on captured frame.
352  // The CapturedFrame class doesn't support planes. We have to ExtractBuffer
353  // to one block for it.
354  int length = webrtc::CalcBufferSize(webrtc::kI420,
355                                      sample.width(), sample.height());
356  capture_buffer_.resize(length);
357  // TODO(ronghuawu): Refactor the WebRtcCapturedFrame to avoid memory copy.
358  webrtc::ExtractBuffer(sample, length, &capture_buffer_[0]);
359  WebRtcCapturedFrame frame(sample, &capture_buffer_[0], length);
360  SignalFrameCaptured(this, &frame);
361}
362
363void WebRtcVideoCapturer::OnCaptureDelayChanged(const int32_t id,
364                                                const int32_t delay) {
365  LOG(LS_INFO) << "Capture delay changed to " << delay << " ms";
366}
367
368// WebRtcCapturedFrame
369WebRtcCapturedFrame::WebRtcCapturedFrame(const webrtc::I420VideoFrame& sample,
370                                         void* buffer,
371                                         int length) {
372  width = sample.width();
373  height = sample.height();
374  fourcc = FOURCC_I420;
375  // TODO(hellner): Support pixel aspect ratio (for OSX).
376  pixel_width = 1;
377  pixel_height = 1;
378  // Convert units from VideoFrame RenderTimeMs to CapturedFrame (nanoseconds).
379  elapsed_time = sample.render_time_ms() * rtc::kNumNanosecsPerMillisec;
380  time_stamp = elapsed_time;
381  data_size = length;
382  data = buffer;
383}
384
385}  // namespace cricket
386
387#endif  // HAVE_WEBRTC_VIDEO
388