1/*
2 * libjingle
3 * Copyright 2011 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include "talk/media/webrtc/webrtcvideocapturer.h"
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#ifdef HAVE_WEBRTC_VIDEO
35#include "talk/media/webrtc/webrtcvideoframe.h"
36#include "talk/media/webrtc/webrtcvideoframefactory.h"
37#include "webrtc/base/arraysize.h"
38#include "webrtc/base/bind.h"
39#include "webrtc/base/checks.h"
40#include "webrtc/base/criticalsection.h"
41#include "webrtc/base/logging.h"
42#include "webrtc/base/safe_conversions.h"
43#include "webrtc/base/thread.h"
44#include "webrtc/base/timeutils.h"
45
46#include "webrtc/base/win32.h"  // Need this to #include the impl files.
47#include "webrtc/modules/video_capture/video_capture_factory.h"
48#include "webrtc/system_wrappers/include/field_trial.h"
49
50namespace cricket {
51
52struct kVideoFourCCEntry {
53  uint32_t fourcc;
54  webrtc::RawVideoType webrtc_type;
55};
56
57// This indicates our format preferences and defines a mapping between
58// webrtc::RawVideoType (from video_capture_defines.h) to our FOURCCs.
59static kVideoFourCCEntry kSupportedFourCCs[] = {
60  { FOURCC_I420, webrtc::kVideoI420 },   // 12 bpp, no conversion.
61  { FOURCC_YV12, webrtc::kVideoYV12 },   // 12 bpp, no conversion.
62  { FOURCC_YUY2, webrtc::kVideoYUY2 },   // 16 bpp, fast conversion.
63  { FOURCC_UYVY, webrtc::kVideoUYVY },   // 16 bpp, fast conversion.
64  { FOURCC_NV12, webrtc::kVideoNV12 },   // 12 bpp, fast conversion.
65  { FOURCC_NV21, webrtc::kVideoNV21 },   // 12 bpp, fast conversion.
66  { FOURCC_MJPG, webrtc::kVideoMJPEG },  // compressed, slow conversion.
67  { FOURCC_ARGB, webrtc::kVideoARGB },   // 32 bpp, slow conversion.
68  { FOURCC_24BG, webrtc::kVideoRGB24 },  // 24 bpp, slow conversion.
69};
70
71class WebRtcVcmFactory : public WebRtcVcmFactoryInterface {
72 public:
73  virtual webrtc::VideoCaptureModule* Create(int id, const char* device) {
74    return webrtc::VideoCaptureFactory::Create(id, device);
75  }
76  virtual webrtc::VideoCaptureModule::DeviceInfo* CreateDeviceInfo(int id) {
77    return webrtc::VideoCaptureFactory::CreateDeviceInfo(id);
78  }
79  virtual void DestroyDeviceInfo(webrtc::VideoCaptureModule::DeviceInfo* info) {
80    delete info;
81  }
82};
83
84static bool CapabilityToFormat(const webrtc::VideoCaptureCapability& cap,
85                               VideoFormat* format) {
86  uint32_t fourcc = 0;
87  for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
88    if (kSupportedFourCCs[i].webrtc_type == cap.rawType) {
89      fourcc = kSupportedFourCCs[i].fourcc;
90      break;
91    }
92  }
93  if (fourcc == 0) {
94    return false;
95  }
96
97  format->fourcc = fourcc;
98  format->width = cap.width;
99  format->height = cap.height;
100  format->interval = VideoFormat::FpsToInterval(cap.maxFPS);
101  return true;
102}
103
104static bool FormatToCapability(const VideoFormat& format,
105                               webrtc::VideoCaptureCapability* cap) {
106  webrtc::RawVideoType webrtc_type = webrtc::kVideoUnknown;
107  for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
108    if (kSupportedFourCCs[i].fourcc == format.fourcc) {
109      webrtc_type = kSupportedFourCCs[i].webrtc_type;
110      break;
111    }
112  }
113  if (webrtc_type == webrtc::kVideoUnknown) {
114    return false;
115  }
116
117  cap->width = format.width;
118  cap->height = format.height;
119  cap->maxFPS = VideoFormat::IntervalToFps(format.interval);
120  cap->expectedCaptureDelay = 0;
121  cap->rawType = webrtc_type;
122  cap->codecType = webrtc::kVideoCodecUnknown;
123  cap->interlaced = false;
124  return true;
125}
126
127///////////////////////////////////////////////////////////////////////////
128// Implementation of class WebRtcVideoCapturer
129///////////////////////////////////////////////////////////////////////////
130
131WebRtcVideoCapturer::WebRtcVideoCapturer()
132    : factory_(new WebRtcVcmFactory),
133      module_(nullptr),
134      captured_frames_(0),
135      start_thread_(nullptr),
136      async_invoker_(nullptr) {
137  set_frame_factory(new WebRtcVideoFrameFactory());
138}
139
140WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory)
141    : factory_(factory),
142      module_(nullptr),
143      captured_frames_(0),
144      start_thread_(nullptr),
145      async_invoker_(nullptr) {
146  set_frame_factory(new WebRtcVideoFrameFactory());
147}
148
149WebRtcVideoCapturer::~WebRtcVideoCapturer() {
150  if (module_) {
151    module_->Release();
152  }
153}
154
155bool WebRtcVideoCapturer::Init(const Device& device) {
156  RTC_DCHECK(!start_thread_);
157  if (module_) {
158    LOG(LS_ERROR) << "The capturer is already initialized";
159    return false;
160  }
161
162  webrtc::VideoCaptureModule::DeviceInfo* info = factory_->CreateDeviceInfo(0);
163  if (!info) {
164    return false;
165  }
166
167  // Find the desired camera, by name.
168  // In the future, comparing IDs will be more robust.
169  // TODO(juberti): Figure what's needed to allow this.
170  int num_cams = info->NumberOfDevices();
171  char vcm_id[256] = "";
172  bool found = false;
173  for (int index = 0; index < num_cams; ++index) {
174    char vcm_name[256];
175    if (info->GetDeviceName(index, vcm_name, arraysize(vcm_name), vcm_id,
176                            arraysize(vcm_id)) != -1) {
177      if (device.name == reinterpret_cast<char*>(vcm_name)) {
178        found = true;
179        break;
180      }
181    }
182  }
183  if (!found) {
184    LOG(LS_WARNING) << "Failed to find capturer for id: " << device.id;
185    factory_->DestroyDeviceInfo(info);
186    return false;
187  }
188
189  // Enumerate the supported formats.
190  // TODO(juberti): Find out why this starts/stops the camera...
191  std::vector<VideoFormat> supported;
192  int32_t num_caps = info->NumberOfCapabilities(vcm_id);
193  for (int32_t i = 0; i < num_caps; ++i) {
194    webrtc::VideoCaptureCapability cap;
195    if (info->GetCapability(vcm_id, i, cap) != -1) {
196      VideoFormat format;
197      if (CapabilityToFormat(cap, &format)) {
198        supported.push_back(format);
199      } else {
200        LOG(LS_WARNING) << "Ignoring unsupported WebRTC capture format "
201                        << cap.rawType;
202      }
203    }
204  }
205  factory_->DestroyDeviceInfo(info);
206
207  if (supported.empty()) {
208    LOG(LS_ERROR) << "Failed to find usable formats for id: " << device.id;
209    return false;
210  }
211
212  module_ = factory_->Create(0, vcm_id);
213  if (!module_) {
214    LOG(LS_ERROR) << "Failed to create capturer for id: " << device.id;
215    return false;
216  }
217
218  // It is safe to change member attributes now.
219  module_->AddRef();
220  SetId(device.id);
221  SetSupportedFormats(supported);
222
223  // Ensure these 2 have the same value.
224  SetApplyRotation(module_->GetApplyRotation());
225
226  return true;
227}
228
229bool WebRtcVideoCapturer::Init(webrtc::VideoCaptureModule* module) {
230  RTC_DCHECK(!start_thread_);
231  if (module_) {
232    LOG(LS_ERROR) << "The capturer is already initialized";
233    return false;
234  }
235  if (!module) {
236    LOG(LS_ERROR) << "Invalid VCM supplied";
237    return false;
238  }
239  // TODO(juberti): Set id and formats.
240  (module_ = module)->AddRef();
241  return true;
242}
243
244bool WebRtcVideoCapturer::GetBestCaptureFormat(const VideoFormat& desired,
245                                               VideoFormat* best_format) {
246  if (!best_format) {
247    return false;
248  }
249
250  if (!VideoCapturer::GetBestCaptureFormat(desired, best_format)) {
251    // We maybe using a manually injected VCM which doesn't support enum.
252    // Use the desired format as the best format.
253    best_format->width = desired.width;
254    best_format->height = desired.height;
255    best_format->fourcc = FOURCC_I420;
256    best_format->interval = desired.interval;
257    LOG(LS_INFO) << "Failed to find best capture format,"
258                 << " fall back to the requested format "
259                 << best_format->ToString();
260  }
261  return true;
262}
263bool WebRtcVideoCapturer::SetApplyRotation(bool enable) {
264  // Can't take lock here as this will cause deadlock with
265  // OnIncomingCapturedFrame. In fact, the whole method, including methods it
266  // calls, can't take lock.
267  RTC_DCHECK(module_);
268
269  const std::string group_name =
270      webrtc::field_trial::FindFullName("WebRTC-CVO");
271
272  if (group_name == "Disabled") {
273    return true;
274  }
275
276  if (!VideoCapturer::SetApplyRotation(enable)) {
277    return false;
278  }
279  return module_->SetApplyRotation(enable);
280}
281
282CaptureState WebRtcVideoCapturer::Start(const VideoFormat& capture_format) {
283  if (!module_) {
284    LOG(LS_ERROR) << "The capturer has not been initialized";
285    return CS_NO_DEVICE;
286  }
287  if (start_thread_) {
288    LOG(LS_ERROR) << "The capturer is already running";
289    RTC_DCHECK(start_thread_->IsCurrent())
290        << "Trying to start capturer on different threads";
291    return CS_FAILED;
292  }
293
294  start_thread_ = rtc::Thread::Current();
295  RTC_DCHECK(!async_invoker_);
296  async_invoker_.reset(new rtc::AsyncInvoker());
297  captured_frames_ = 0;
298
299  SetCaptureFormat(&capture_format);
300
301  webrtc::VideoCaptureCapability cap;
302  if (!FormatToCapability(capture_format, &cap)) {
303    LOG(LS_ERROR) << "Invalid capture format specified";
304    return CS_FAILED;
305  }
306
307  uint32_t start = rtc::Time();
308  module_->RegisterCaptureDataCallback(*this);
309  if (module_->StartCapture(cap) != 0) {
310    LOG(LS_ERROR) << "Camera '" << GetId() << "' failed to start";
311    module_->DeRegisterCaptureDataCallback();
312    async_invoker_.reset();
313    SetCaptureFormat(nullptr);
314    start_thread_ = nullptr;
315    return CS_FAILED;
316  }
317
318  LOG(LS_INFO) << "Camera '" << GetId() << "' started with format "
319               << capture_format.ToString() << ", elapsed time "
320               << rtc::TimeSince(start) << " ms";
321
322  SetCaptureState(CS_RUNNING);
323  return CS_STARTING;
324}
325
326void WebRtcVideoCapturer::Stop() {
327  if (!start_thread_) {
328    LOG(LS_ERROR) << "The capturer is already stopped";
329    return;
330  }
331  RTC_DCHECK(start_thread_);
332  RTC_DCHECK(start_thread_->IsCurrent());
333  RTC_DCHECK(async_invoker_);
334  if (IsRunning()) {
335    // The module is responsible for OnIncomingCapturedFrame being called, if
336    // we stop it we will get no further callbacks.
337    module_->StopCapture();
338  }
339  module_->DeRegisterCaptureDataCallback();
340
341  // TODO(juberti): Determine if the VCM exposes any drop stats we can use.
342  double drop_ratio = 0.0;
343  LOG(LS_INFO) << "Camera '" << GetId() << "' stopped after capturing "
344               << captured_frames_ << " frames and dropping "
345               << drop_ratio << "%";
346
347  // Clear any pending async invokes (that OnIncomingCapturedFrame may have
348  // caused).
349  async_invoker_.reset();
350
351  SetCaptureFormat(NULL);
352  start_thread_ = nullptr;
353  SetCaptureState(CS_STOPPED);
354}
355
356bool WebRtcVideoCapturer::IsRunning() {
357  return (module_ != NULL && module_->CaptureStarted());
358}
359
360bool WebRtcVideoCapturer::GetPreferredFourccs(std::vector<uint32_t>* fourccs) {
361  if (!fourccs) {
362    return false;
363  }
364
365  fourccs->clear();
366  for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
367    fourccs->push_back(kSupportedFourCCs[i].fourcc);
368  }
369  return true;
370}
371
372void WebRtcVideoCapturer::OnIncomingCapturedFrame(
373    const int32_t id,
374    const webrtc::VideoFrame& sample) {
375  // This can only happen between Start() and Stop().
376  RTC_DCHECK(start_thread_);
377  RTC_DCHECK(async_invoker_);
378  if (start_thread_->IsCurrent()) {
379    SignalFrameCapturedOnStartThread(sample);
380  } else {
381    // This currently happens on with at least VideoCaptureModuleV4L2 and
382    // possibly other implementations of WebRTC's VideoCaptureModule.
383    // In order to maintain the threading contract with the upper layers and
384    // consistency with other capturers such as in Chrome, we need to do a
385    // thread hop.
386    // Note that Stop() can cause the async invoke call to be cancelled.
387    async_invoker_->AsyncInvoke<void>(
388        start_thread_,
389        // Note that Bind captures by value, so there's an intermediate copy
390        // of sample.
391        rtc::Bind(&WebRtcVideoCapturer::SignalFrameCapturedOnStartThread, this,
392                  sample));
393  }
394}
395
396void WebRtcVideoCapturer::OnCaptureDelayChanged(const int32_t id,
397                                                const int32_t delay) {
398  LOG(LS_INFO) << "Capture delay changed to " << delay << " ms";
399}
400
401void WebRtcVideoCapturer::SignalFrameCapturedOnStartThread(
402    const webrtc::VideoFrame& frame) {
403  // This can only happen between Start() and Stop().
404  RTC_DCHECK(start_thread_);
405  RTC_DCHECK(start_thread_->IsCurrent());
406  RTC_DCHECK(async_invoker_);
407
408  ++captured_frames_;
409  // Log the size and pixel aspect ratio of the first captured frame.
410  if (1 == captured_frames_) {
411    LOG(LS_INFO) << "Captured frame size "
412                 << frame.width() << "x" << frame.height()
413                 << ". Expected format " << GetCaptureFormat()->ToString();
414  }
415
416  // Signal down stream components on captured frame.
417  // The CapturedFrame class doesn't support planes. We have to ExtractBuffer
418  // to one block for it.
419  size_t length =
420      webrtc::CalcBufferSize(webrtc::kI420, frame.width(), frame.height());
421  capture_buffer_.resize(length);
422  // TODO(magjed): Refactor the WebRtcCapturedFrame to avoid memory copy or
423  // take over ownership of the buffer held by |frame| if that's possible.
424  webrtc::ExtractBuffer(frame, length, &capture_buffer_[0]);
425  WebRtcCapturedFrame webrtc_frame(frame, &capture_buffer_[0], length);
426  SignalFrameCaptured(this, &webrtc_frame);
427}
428
429// WebRtcCapturedFrame
430WebRtcCapturedFrame::WebRtcCapturedFrame(const webrtc::VideoFrame& sample,
431                                         void* buffer,
432                                         size_t length) {
433  width = sample.width();
434  height = sample.height();
435  fourcc = FOURCC_I420;
436  // TODO(hellner): Support pixel aspect ratio (for OSX).
437  pixel_width = 1;
438  pixel_height = 1;
439  // Convert units from VideoFrame RenderTimeMs to CapturedFrame (nanoseconds).
440  time_stamp = sample.render_time_ms() * rtc::kNumNanosecsPerMillisec;
441  data_size = rtc::checked_cast<uint32_t>(length);
442  data = buffer;
443  rotation = sample.rotation();
444}
445
446}  // namespace cricket
447
448#endif  // HAVE_WEBRTC_VIDEO
449