1/*
2 * libjingle
3 * Copyright 2012 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 *  1. Redistributions of source code must retain the above copyright notice,
9 *     this list of conditions and the following disclaimer.
10 *  2. Redistributions in binary form must reproduce the above copyright notice,
11 *     this list of conditions and the following disclaimer in the documentation
12 *     and/or other materials provided with the distribution.
13 *  3. The name of the author may not be used to endorse or promote products
14 *     derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <string>
29#include <vector>
30
31#include "talk/app/webrtc/remotevideocapturer.h"
32#include "talk/app/webrtc/test/fakeconstraints.h"
33#include "talk/app/webrtc/videosource.h"
34#include "talk/media/base/fakemediaengine.h"
35#include "talk/media/base/fakevideocapturer.h"
36#include "talk/media/base/fakevideorenderer.h"
37#include "talk/media/webrtc/webrtcvideoframe.h"
38#include "talk/session/media/channelmanager.h"
39#include "webrtc/base/gunit.h"
40
41using webrtc::FakeConstraints;
42using webrtc::VideoSource;
43using webrtc::MediaConstraintsInterface;
44using webrtc::MediaSourceInterface;
45using webrtc::ObserverInterface;
46using webrtc::VideoSourceInterface;
47
48namespace {
49
50// Max wait time for a test.
51const int kMaxWaitMs = 100;
52
53}  // anonymous namespace
54
55
56// TestVideoCapturer extends cricket::FakeVideoCapturer so it can be used for
57// testing without known camera formats.
58// It keeps its own lists of cricket::VideoFormats for the unit tests in this
59// file.
60class TestVideoCapturer : public cricket::FakeVideoCapturer {
61 public:
62  TestVideoCapturer() : test_without_formats_(false) {
63    std::vector<cricket::VideoFormat> formats;
64    formats.push_back(cricket::VideoFormat(1280, 720,
65        cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420));
66    formats.push_back(cricket::VideoFormat(640, 480,
67        cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420));
68    formats.push_back(cricket::VideoFormat(640, 400,
69            cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420));
70    formats.push_back(cricket::VideoFormat(320, 240,
71        cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420));
72    formats.push_back(cricket::VideoFormat(352, 288,
73            cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420));
74    ResetSupportedFormats(formats);
75  }
76
77  // This function is used for resetting the supported capture formats and
78  // simulating a cricket::VideoCapturer implementation that don't support
79  // capture format enumeration. This is used to simulate the current
80  // Chrome implementation.
81  void TestWithoutCameraFormats() {
82    test_without_formats_ = true;
83    std::vector<cricket::VideoFormat> formats;
84    ResetSupportedFormats(formats);
85  }
86
87  virtual cricket::CaptureState Start(
88      const cricket::VideoFormat& capture_format) {
89    if (test_without_formats_) {
90      std::vector<cricket::VideoFormat> formats;
91      formats.push_back(capture_format);
92      ResetSupportedFormats(formats);
93    }
94    return FakeVideoCapturer::Start(capture_format);
95  }
96
97  virtual bool GetBestCaptureFormat(const cricket::VideoFormat& desired,
98                                    cricket::VideoFormat* best_format) {
99    if (test_without_formats_) {
100      *best_format = desired;
101      return true;
102    }
103    return FakeVideoCapturer::GetBestCaptureFormat(desired,
104                                                   best_format);
105  }
106
107 private:
108  bool test_without_formats_;
109};
110
111class StateObserver : public ObserverInterface {
112 public:
113  explicit StateObserver(VideoSourceInterface* source)
114     : state_(source->state()),
115       source_(source) {
116  }
117  virtual void OnChanged() {
118    state_ = source_->state();
119  }
120  MediaSourceInterface::SourceState state() const { return state_; }
121
122 private:
123  MediaSourceInterface::SourceState state_;
124  rtc::scoped_refptr<VideoSourceInterface> source_;
125};
126
127class VideoSourceTest : public testing::Test {
128 protected:
129  VideoSourceTest()
130      : capturer_cleanup_(new TestVideoCapturer()),
131        capturer_(capturer_cleanup_.get()),
132        channel_manager_(new cricket::ChannelManager(
133          new cricket::FakeMediaEngine(), rtc::Thread::Current())) {
134  }
135
136  void SetUp() {
137    ASSERT_TRUE(channel_manager_->Init());
138  }
139
140  void CreateVideoSource() {
141    CreateVideoSource(NULL);
142  }
143
144  void CreateVideoSource(
145      const webrtc::MediaConstraintsInterface* constraints) {
146    // VideoSource take ownership of |capturer_|
147    source_ =
148        VideoSource::Create(channel_manager_.get(), capturer_cleanup_.release(),
149                            constraints, false);
150
151    ASSERT_TRUE(source_.get() != NULL);
152    EXPECT_EQ(capturer_, source_->GetVideoCapturer());
153
154    state_observer_.reset(new StateObserver(source_));
155    source_->RegisterObserver(state_observer_.get());
156    source_->AddSink(&renderer_);
157  }
158
159  rtc::scoped_ptr<TestVideoCapturer> capturer_cleanup_;
160  TestVideoCapturer* capturer_;
161  cricket::FakeVideoRenderer renderer_;
162  rtc::scoped_ptr<cricket::ChannelManager> channel_manager_;
163  rtc::scoped_ptr<StateObserver> state_observer_;
164  rtc::scoped_refptr<VideoSource> source_;
165};
166
167
168// Test that a VideoSource transition to kLive state when the capture
169// device have started and kEnded if it is stopped.
170// It also test that an output can receive video frames.
171TEST_F(VideoSourceTest, CapturerStartStop) {
172  // Initialize without constraints.
173  CreateVideoSource();
174  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
175                 kMaxWaitMs);
176
177  ASSERT_TRUE(capturer_->CaptureFrame());
178  EXPECT_EQ(1, renderer_.num_rendered_frames());
179
180  capturer_->Stop();
181  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
182                 kMaxWaitMs);
183}
184
185// Test that a VideoSource can be stopped and restarted.
186TEST_F(VideoSourceTest, StopRestart) {
187  // Initialize without constraints.
188  CreateVideoSource();
189  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
190                 kMaxWaitMs);
191
192  ASSERT_TRUE(capturer_->CaptureFrame());
193  EXPECT_EQ(1, renderer_.num_rendered_frames());
194
195  source_->Stop();
196  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
197                 kMaxWaitMs);
198
199  source_->Restart();
200  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
201                 kMaxWaitMs);
202
203  ASSERT_TRUE(capturer_->CaptureFrame());
204  EXPECT_EQ(2, renderer_.num_rendered_frames());
205
206  source_->Stop();
207}
208
209// Test start stop with a remote VideoSource - the video source that has a
210// RemoteVideoCapturer and takes video frames from FrameInput.
211TEST_F(VideoSourceTest, StartStopRemote) {
212  source_ = VideoSource::Create(channel_manager_.get(),
213                                new webrtc::RemoteVideoCapturer(), NULL, true);
214
215  ASSERT_TRUE(source_.get() != NULL);
216  EXPECT_TRUE(NULL != source_->GetVideoCapturer());
217
218  state_observer_.reset(new StateObserver(source_));
219  source_->RegisterObserver(state_observer_.get());
220  source_->AddSink(&renderer_);
221
222  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
223                 kMaxWaitMs);
224
225  cricket::VideoRenderer* frameinput = source_->FrameInput();
226  cricket::WebRtcVideoFrame test_frame;
227  frameinput->SetSize(1280, 720, 0);
228  frameinput->RenderFrame(&test_frame);
229  EXPECT_EQ(1, renderer_.num_rendered_frames());
230
231  source_->GetVideoCapturer()->Stop();
232  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
233                 kMaxWaitMs);
234}
235
236// Test that a VideoSource transition to kEnded if the capture device
237// fails.
238TEST_F(VideoSourceTest, CameraFailed) {
239  CreateVideoSource();
240  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
241                 kMaxWaitMs);
242
243  capturer_->SignalStateChange(capturer_, cricket::CS_FAILED);
244  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
245                 kMaxWaitMs);
246}
247
248// Test that the capture output is CIF if we set max constraints to CIF.
249// and the capture device support CIF.
250TEST_F(VideoSourceTest, MandatoryConstraintCif5Fps) {
251  FakeConstraints constraints;
252  constraints.AddMandatory(MediaConstraintsInterface::kMaxWidth, 352);
253  constraints.AddMandatory(MediaConstraintsInterface::kMaxHeight, 288);
254  constraints.AddMandatory(MediaConstraintsInterface::kMaxFrameRate, 5);
255
256  CreateVideoSource(&constraints);
257  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
258                 kMaxWaitMs);
259  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
260  ASSERT_TRUE(format != NULL);
261  EXPECT_EQ(352, format->width);
262  EXPECT_EQ(288, format->height);
263  EXPECT_EQ(30, format->framerate());
264}
265
266// Test that the capture output is 720P if the camera support it and the
267// optional constraint is set to 720P.
268TEST_F(VideoSourceTest, MandatoryMinVgaOptional720P) {
269  FakeConstraints constraints;
270  constraints.AddMandatory(MediaConstraintsInterface::kMinWidth, 640);
271  constraints.AddMandatory(MediaConstraintsInterface::kMinHeight, 480);
272  constraints.AddOptional(MediaConstraintsInterface::kMinWidth, 1280);
273  constraints.AddOptional(MediaConstraintsInterface::kMinAspectRatio,
274                          1280.0 / 720);
275
276  CreateVideoSource(&constraints);
277  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
278                 kMaxWaitMs);
279  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
280  ASSERT_TRUE(format != NULL);
281  EXPECT_EQ(1280, format->width);
282  EXPECT_EQ(720, format->height);
283  EXPECT_EQ(30, format->framerate());
284}
285
286// Test that the capture output have aspect ratio 4:3 if a mandatory constraint
287// require it even if an optional constraint request a higher resolution
288// that don't have this aspect ratio.
289TEST_F(VideoSourceTest, MandatoryAspectRatio4To3) {
290  FakeConstraints constraints;
291  constraints.AddMandatory(MediaConstraintsInterface::kMinWidth, 640);
292  constraints.AddMandatory(MediaConstraintsInterface::kMinHeight, 480);
293  constraints.AddMandatory(MediaConstraintsInterface::kMaxAspectRatio,
294                           640.0 / 480);
295  constraints.AddOptional(MediaConstraintsInterface::kMinWidth, 1280);
296
297  CreateVideoSource(&constraints);
298  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
299                 kMaxWaitMs);
300  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
301  ASSERT_TRUE(format != NULL);
302  EXPECT_EQ(640, format->width);
303  EXPECT_EQ(480, format->height);
304  EXPECT_EQ(30, format->framerate());
305}
306
307
308// Test that the source state transition to kEnded if the mandatory aspect ratio
309// is set higher than supported.
310TEST_F(VideoSourceTest, MandatoryAspectRatioTooHigh) {
311  FakeConstraints constraints;
312  constraints.AddMandatory(MediaConstraintsInterface::kMinAspectRatio, 2);
313  CreateVideoSource(&constraints);
314  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
315                 kMaxWaitMs);
316}
317
318// Test that the source ignores an optional aspect ratio that is higher than
319// supported.
320TEST_F(VideoSourceTest, OptionalAspectRatioTooHigh) {
321  FakeConstraints constraints;
322  constraints.AddOptional(MediaConstraintsInterface::kMinAspectRatio, 2);
323  CreateVideoSource(&constraints);
324  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
325                 kMaxWaitMs);
326  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
327  ASSERT_TRUE(format != NULL);
328  double aspect_ratio = static_cast<double>(format->width) / format->height;
329  EXPECT_LT(aspect_ratio, 2);
330}
331
332// Test that the source starts video with the default resolution if the
333// camera doesn't support capability enumeration and there are no constraints.
334TEST_F(VideoSourceTest, NoCameraCapability) {
335  capturer_->TestWithoutCameraFormats();
336
337  CreateVideoSource();
338  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
339                 kMaxWaitMs);
340  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
341  ASSERT_TRUE(format != NULL);
342  EXPECT_EQ(640, format->width);
343  EXPECT_EQ(480, format->height);
344  EXPECT_EQ(30, format->framerate());
345}
346
347// Test that the source can start the video and get the requested aspect ratio
348// if the camera doesn't support capability enumeration and the aspect ratio is
349// set.
350TEST_F(VideoSourceTest, NoCameraCapability16To9Ratio) {
351  capturer_->TestWithoutCameraFormats();
352
353  FakeConstraints constraints;
354  double requested_aspect_ratio = 640.0 / 360;
355  constraints.AddMandatory(MediaConstraintsInterface::kMinWidth, 640);
356  constraints.AddMandatory(MediaConstraintsInterface::kMinAspectRatio,
357                           requested_aspect_ratio);
358
359  CreateVideoSource(&constraints);
360  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
361                 kMaxWaitMs);
362  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
363  double aspect_ratio = static_cast<double>(format->width) / format->height;
364  EXPECT_LE(requested_aspect_ratio, aspect_ratio);
365}
366
367// Test that the source state transitions to kEnded if an unknown mandatory
368// constraint is found.
369TEST_F(VideoSourceTest, InvalidMandatoryConstraint) {
370  FakeConstraints constraints;
371  constraints.AddMandatory("weird key", 640);
372
373  CreateVideoSource(&constraints);
374  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
375                 kMaxWaitMs);
376}
377
378// Test that the source ignores an unknown optional constraint.
379TEST_F(VideoSourceTest, InvalidOptionalConstraint) {
380  FakeConstraints constraints;
381  constraints.AddOptional("weird key", 640);
382
383  CreateVideoSource(&constraints);
384  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
385                 kMaxWaitMs);
386}
387
388TEST_F(VideoSourceTest, SetValidOptionValues) {
389  FakeConstraints constraints;
390  constraints.AddMandatory(MediaConstraintsInterface::kNoiseReduction, "false");
391
392  CreateVideoSource(&constraints);
393
394  EXPECT_EQ(rtc::Optional<bool>(false),
395            source_->options()->video_noise_reduction);
396}
397
398TEST_F(VideoSourceTest, OptionNotSet) {
399  FakeConstraints constraints;
400  CreateVideoSource(&constraints);
401  EXPECT_EQ(rtc::Optional<bool>(), source_->options()->video_noise_reduction);
402}
403
404TEST_F(VideoSourceTest, MandatoryOptionOverridesOptional) {
405  FakeConstraints constraints;
406  constraints.AddMandatory(
407      MediaConstraintsInterface::kNoiseReduction, true);
408  constraints.AddOptional(
409      MediaConstraintsInterface::kNoiseReduction, false);
410
411  CreateVideoSource(&constraints);
412
413  EXPECT_EQ(rtc::Optional<bool>(true),
414            source_->options()->video_noise_reduction);
415}
416
417TEST_F(VideoSourceTest, InvalidOptionKeyOptional) {
418  FakeConstraints constraints;
419  constraints.AddOptional(
420      MediaConstraintsInterface::kNoiseReduction, false);
421  constraints.AddOptional("invalidKey", false);
422
423  CreateVideoSource(&constraints);
424
425  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
426      kMaxWaitMs);
427  EXPECT_EQ(rtc::Optional<bool>(false),
428            source_->options()->video_noise_reduction);
429}
430
431TEST_F(VideoSourceTest, InvalidOptionKeyMandatory) {
432  FakeConstraints constraints;
433  constraints.AddMandatory(
434      MediaConstraintsInterface::kNoiseReduction, false);
435  constraints.AddMandatory("invalidKey", false);
436
437  CreateVideoSource(&constraints);
438
439  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
440      kMaxWaitMs);
441  EXPECT_EQ(rtc::Optional<bool>(), source_->options()->video_noise_reduction);
442}
443
444TEST_F(VideoSourceTest, InvalidOptionValueOptional) {
445  FakeConstraints constraints;
446  constraints.AddOptional(
447      MediaConstraintsInterface::kNoiseReduction, "not a boolean");
448
449  CreateVideoSource(&constraints);
450
451  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
452      kMaxWaitMs);
453  EXPECT_EQ(rtc::Optional<bool>(), source_->options()->video_noise_reduction);
454}
455
456TEST_F(VideoSourceTest, InvalidOptionValueMandatory) {
457  FakeConstraints constraints;
458  // Optional constraints should be ignored if the mandatory constraints fail.
459  constraints.AddOptional(
460      MediaConstraintsInterface::kNoiseReduction, "false");
461  // Values are case-sensitive and must be all lower-case.
462  constraints.AddMandatory(
463      MediaConstraintsInterface::kNoiseReduction, "True");
464
465  CreateVideoSource(&constraints);
466
467  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
468      kMaxWaitMs);
469  EXPECT_EQ(rtc::Optional<bool>(), source_->options()->video_noise_reduction);
470}
471
472TEST_F(VideoSourceTest, MixedOptionsAndConstraints) {
473  FakeConstraints constraints;
474  constraints.AddMandatory(MediaConstraintsInterface::kMaxWidth, 352);
475  constraints.AddMandatory(MediaConstraintsInterface::kMaxHeight, 288);
476  constraints.AddOptional(MediaConstraintsInterface::kMaxFrameRate, 5);
477
478  constraints.AddMandatory(
479      MediaConstraintsInterface::kNoiseReduction, false);
480  constraints.AddOptional(
481      MediaConstraintsInterface::kNoiseReduction, true);
482
483  CreateVideoSource(&constraints);
484  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
485                 kMaxWaitMs);
486  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
487  ASSERT_TRUE(format != NULL);
488  EXPECT_EQ(352, format->width);
489  EXPECT_EQ(288, format->height);
490  EXPECT_EQ(30, format->framerate());
491
492  EXPECT_EQ(rtc::Optional<bool>(false),
493            source_->options()->video_noise_reduction);
494}
495
496// Tests that the source starts video with the default resolution for
497// screencast if no constraint is set.
498TEST_F(VideoSourceTest, ScreencastResolutionNoConstraint) {
499  capturer_->TestWithoutCameraFormats();
500  capturer_->SetScreencast(true);
501
502  CreateVideoSource();
503  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
504                 kMaxWaitMs);
505  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
506  ASSERT_TRUE(format != NULL);
507  EXPECT_EQ(640, format->width);
508  EXPECT_EQ(480, format->height);
509  EXPECT_EQ(30, format->framerate());
510}
511
512// Tests that the source starts video with the max width and height set by
513// constraints for screencast.
514TEST_F(VideoSourceTest, ScreencastResolutionWithConstraint) {
515  FakeConstraints constraints;
516  constraints.AddMandatory(MediaConstraintsInterface::kMaxWidth, 480);
517  constraints.AddMandatory(MediaConstraintsInterface::kMaxHeight, 270);
518
519  capturer_->TestWithoutCameraFormats();
520  capturer_->SetScreencast(true);
521
522  CreateVideoSource(&constraints);
523  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
524                 kMaxWaitMs);
525  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
526  ASSERT_TRUE(format != NULL);
527  EXPECT_EQ(480, format->width);
528  EXPECT_EQ(270, format->height);
529  EXPECT_EQ(30, format->framerate());
530}
531
532TEST_F(VideoSourceTest, MandatorySubOneFpsConstraints) {
533  FakeConstraints constraints;
534  constraints.AddMandatory(MediaConstraintsInterface::kMaxFrameRate, 0.5);
535
536  CreateVideoSource(&constraints);
537  EXPECT_EQ_WAIT(MediaSourceInterface::kEnded, state_observer_->state(),
538                 kMaxWaitMs);
539  ASSERT_TRUE(capturer_->GetCaptureFormat() == NULL);
540}
541
542TEST_F(VideoSourceTest, OptionalSubOneFpsConstraints) {
543  FakeConstraints constraints;
544  constraints.AddOptional(MediaConstraintsInterface::kMaxFrameRate, 0.5);
545
546  CreateVideoSource(&constraints);
547  EXPECT_EQ_WAIT(MediaSourceInterface::kLive, state_observer_->state(),
548                 kMaxWaitMs);
549  const cricket::VideoFormat* format = capturer_->GetCaptureFormat();
550  ASSERT_TRUE(format != NULL);
551  EXPECT_EQ(30, format->framerate());
552}
553
554