1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/renderer/media/media_stream_video_source.h"
6
7#include <algorithm>
8#include <limits>
9#include <string>
10
11#include "base/debug/trace_event.h"
12#include "base/logging.h"
13#include "base/strings/string_number_conversions.h"
14#include "content/child/child_process.h"
15#include "content/renderer/media/media_stream_constraints_util.h"
16#include "content/renderer/media/media_stream_video_track.h"
17#include "content/renderer/media/video_track_adapter.h"
18
19namespace content {
20
21// Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
23const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
24const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
25const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
26const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
27const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
28const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
29const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
30
31const char* kSupportedConstraints[] = {
32  MediaStreamVideoSource::kMaxAspectRatio,
33  MediaStreamVideoSource::kMinAspectRatio,
34  MediaStreamVideoSource::kMaxWidth,
35  MediaStreamVideoSource::kMinWidth,
36  MediaStreamVideoSource::kMaxHeight,
37  MediaStreamVideoSource::kMinHeight,
38  MediaStreamVideoSource::kMaxFrameRate,
39  MediaStreamVideoSource::kMinFrameRate,
40};
41
42const int MediaStreamVideoSource::kDefaultWidth = 640;
43const int MediaStreamVideoSource::kDefaultHeight = 480;
44const int MediaStreamVideoSource::kDefaultFrameRate = 30;
45const int MediaStreamVideoSource::kUnknownFrameRate = 0;
46
47namespace {
48
49// Google-specific key prefix. Constraints with this prefix are ignored if they
50// are unknown.
51const char kGooglePrefix[] = "goog";
52
53// Returns true if |constraint| has mandatory constraints.
54bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
55  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
56  constraints.getMandatoryConstraints(mandatory_constraints);
57  return !mandatory_constraints.isEmpty();
58}
59
60// Retrieve the desired max width and height from |constraints|. If not set,
61// the |desired_width| and |desired_height| are set to
62// std::numeric_limits<int>::max();
63// If either max width or height is set as a mandatory constraint, the optional
64// constraints are not checked.
65void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
66                                 int* desired_width, int* desired_height) {
67  *desired_width = std::numeric_limits<int>::max();
68  *desired_height = std::numeric_limits<int>::max();
69
70  bool mandatory = GetMandatoryConstraintValueAsInteger(
71      constraints,
72      MediaStreamVideoSource::kMaxWidth,
73      desired_width);
74  mandatory |= GetMandatoryConstraintValueAsInteger(
75      constraints,
76      MediaStreamVideoSource::kMaxHeight,
77      desired_height);
78  if (mandatory)
79    return;
80
81  GetOptionalConstraintValueAsInteger(constraints,
82                                      MediaStreamVideoSource::kMaxWidth,
83                                      desired_width);
84  GetOptionalConstraintValueAsInteger(constraints,
85                                      MediaStreamVideoSource::kMaxHeight,
86                                      desired_height);
87}
88
89// Retrieve the desired max and min aspect ratio from |constraints|. If not set,
90// the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
91// std::numeric_limits<double>::max();
92// If either min or max aspect ratio is set as a mandatory constraint, the
93// optional constraints are not checked.
94void GetDesiredMinAndMaxAspectRatio(
95    const blink::WebMediaConstraints& constraints,
96    double* min_aspect_ratio,
97    double* max_aspect_ratio) {
98  *min_aspect_ratio = 0;
99  *max_aspect_ratio = std::numeric_limits<double>::max();
100
101  bool mandatory = GetMandatoryConstraintValueAsDouble(
102      constraints,
103      MediaStreamVideoSource::kMinAspectRatio,
104      min_aspect_ratio);
105  mandatory |= GetMandatoryConstraintValueAsDouble(
106      constraints,
107      MediaStreamVideoSource::kMaxAspectRatio,
108      max_aspect_ratio);
109  if (mandatory)
110    return;
111
112  GetOptionalConstraintValueAsDouble(
113      constraints,
114      MediaStreamVideoSource::kMinAspectRatio,
115      min_aspect_ratio);
116  GetOptionalConstraintValueAsDouble(
117      constraints,
118      MediaStreamVideoSource::kMaxAspectRatio,
119      max_aspect_ratio);
120}
121
122// Returns true if |constraint| is fulfilled. |format| can be changed by a
123// constraint, e.g. the frame rate can be changed by setting maxFrameRate.
124bool UpdateFormatForConstraint(
125    const blink::WebMediaConstraint& constraint,
126    bool mandatory,
127    media::VideoCaptureFormat* format) {
128  DCHECK(format != NULL);
129
130  if (!format->IsValid())
131    return false;
132
133  std::string constraint_name = constraint.m_name.utf8();
134  std::string constraint_value = constraint.m_value.utf8();
135
136  if (constraint_name.find(kGooglePrefix) == 0) {
137    // These are actually options, not constraints, so they can be satisfied
138    // regardless of the format.
139    return true;
140  }
141
142  if (constraint_name == MediaStreamSource::kSourceId) {
143    // This is a constraint that doesn't affect the format.
144    return true;
145  }
146
147  // Ignore Chrome specific Tab capture constraints.
148  if (constraint_name == kMediaStreamSource ||
149      constraint_name == kMediaStreamSourceId)
150    return true;
151
152  if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
153      constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
154    // These constraints are handled by cropping if the camera outputs the wrong
155    // aspect ratio.
156    double value;
157    return base::StringToDouble(constraint_value, &value);
158  }
159
160  double value = 0.0;
161  if (!base::StringToDouble(constraint_value, &value)) {
162    DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
163                  <<  constraint_name << " Value:" << constraint_value;
164    return false;
165  }
166
167  if (constraint_name == MediaStreamVideoSource::kMinWidth) {
168    return (value <= format->frame_size.width());
169  } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
170    return value > 0.0;
171  } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
172    return (value <= format->frame_size.height());
173  } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
174     return value > 0.0;
175  } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
176    return (value > 0.0) && (value <= format->frame_rate);
177  } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
178    if (value <= 0.0) {
179      // The frame rate is set by constraint.
180      // Don't allow 0 as frame rate if it is a mandatory constraint.
181      // Set the frame rate to 1 if it is not mandatory.
182      if (mandatory) {
183        return false;
184      } else {
185        value = 1.0;
186      }
187    }
188    format->frame_rate =
189        (format->frame_rate > value) ? value : format->frame_rate;
190    return true;
191  } else {
192    LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
193                 <<  constraint_name << " Value:" << constraint_value;
194    return false;
195  }
196}
197
198// Removes media::VideoCaptureFormats from |formats| that don't meet
199// |constraint|.
200void FilterFormatsByConstraint(
201    const blink::WebMediaConstraint& constraint,
202    bool mandatory,
203    media::VideoCaptureFormats* formats) {
204  DVLOG(3) << "FilterFormatsByConstraint("
205           << "{ constraint.m_name = " << constraint.m_name.utf8()
206           << "  constraint.m_value = " << constraint.m_value.utf8()
207           << "  mandatory =  " << mandatory << "})";
208  media::VideoCaptureFormats::iterator format_it = formats->begin();
209  while (format_it != formats->end()) {
210    // Modify the format_it to fulfill the constraint if possible.
211    // Delete it otherwise.
212    if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it))) {
213      format_it = formats->erase(format_it);
214    } else {
215      ++format_it;
216    }
217  }
218}
219
220// Returns the media::VideoCaptureFormats that matches |constraints|.
221media::VideoCaptureFormats FilterFormats(
222    const blink::WebMediaConstraints& constraints,
223    const media::VideoCaptureFormats& supported_formats,
224    blink::WebString* unsatisfied_constraint) {
225  if (constraints.isNull()) {
226    return supported_formats;
227  }
228
229  double max_aspect_ratio;
230  double min_aspect_ratio;
231  GetDesiredMinAndMaxAspectRatio(constraints,
232                                 &min_aspect_ratio,
233                                 &max_aspect_ratio);
234
235  if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
236    DLOG(WARNING) << "Wrong requested aspect ratio.";
237    return media::VideoCaptureFormats();
238  }
239
240  int min_width = 0;
241  GetMandatoryConstraintValueAsInteger(constraints,
242                                       MediaStreamVideoSource::kMinWidth,
243                                       &min_width);
244  int min_height = 0;
245  GetMandatoryConstraintValueAsInteger(constraints,
246                                       MediaStreamVideoSource::kMinHeight,
247                                       &min_height);
248  int max_width;
249  int max_height;
250  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
251
252  if (min_width > max_width || min_height > max_height)
253    return media::VideoCaptureFormats();
254
255  double min_frame_rate = 0.0f;
256  double max_frame_rate = 0.0f;
257  if (GetConstraintValueAsDouble(constraints,
258                                 MediaStreamVideoSource::kMaxFrameRate,
259                                 &max_frame_rate) &&
260      GetConstraintValueAsDouble(constraints,
261                                 MediaStreamVideoSource::kMinFrameRate,
262                                 &min_frame_rate)) {
263    if (min_frame_rate > max_frame_rate) {
264      DLOG(WARNING) << "Wrong requested frame rate.";
265      return media::VideoCaptureFormats();
266    }
267  }
268
269  blink::WebVector<blink::WebMediaConstraint> mandatory;
270  blink::WebVector<blink::WebMediaConstraint> optional;
271  constraints.getMandatoryConstraints(mandatory);
272  constraints.getOptionalConstraints(optional);
273  media::VideoCaptureFormats candidates = supported_formats;
274  for (size_t i = 0; i < mandatory.size(); ++i) {
275    FilterFormatsByConstraint(mandatory[i], true, &candidates);
276    if (candidates.empty()) {
277      *unsatisfied_constraint = mandatory[i].m_name;
278      return candidates;
279    }
280  }
281
282  if (candidates.empty())
283    return candidates;
284
285  // Ok - all mandatory checked and we still have candidates.
286  // Let's try filtering using the optional constraints. The optional
287  // constraints must be filtered in the order they occur in |optional|.
288  // But if a constraint produce zero candidates, the constraint is ignored and
289  // the next constraint is tested.
290  // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
291  for (size_t i = 0; i < optional.size(); ++i) {
292    media::VideoCaptureFormats current_candidates = candidates;
293    FilterFormatsByConstraint(optional[i], false, &current_candidates);
294    if (!current_candidates.empty()) {
295      candidates = current_candidates;
296    }
297  }
298
299  // We have done as good as we can to filter the supported resolutions.
300  return candidates;
301}
302
303const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
304    const media::VideoCaptureFormats& formats,
305    int area) {
306  media::VideoCaptureFormats::const_iterator it = formats.begin();
307  media::VideoCaptureFormats::const_iterator best_it = formats.begin();
308  int best_diff = std::numeric_limits<int>::max();
309  for (; it != formats.end(); ++it) {
310    int diff = abs(area - it->frame_size.width() * it->frame_size.height());
311    if (diff < best_diff) {
312      best_diff = diff;
313      best_it = it;
314    }
315  }
316  return *best_it;
317}
318
319// Find the format that best matches the default video size.
320// This algorithm is chosen since a resolution must be picked even if no
321// constraints are provided. We don't just select the maximum supported
322// resolution since higher resolutions cost more in terms of complexity and
323// many cameras have lower frame rate and have more noise in the image at
324// their maximum supported resolution.
325void GetBestCaptureFormat(
326    const media::VideoCaptureFormats& formats,
327    const blink::WebMediaConstraints& constraints,
328    media::VideoCaptureFormat* capture_format) {
329  DCHECK(!formats.empty());
330
331  int max_width;
332  int max_height;
333  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
334
335  *capture_format = GetBestFormatBasedOnArea(
336      formats,
337      std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
338      std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
339}
340
341}  // anonymous namespace
342
343// static
344MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
345    const blink::WebMediaStreamSource& source) {
346  return static_cast<MediaStreamVideoSource*>(source.extraData());
347}
348
349// static
350bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
351  for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
352    if (kSupportedConstraints[i] == name)
353      return true;
354  }
355  return false;
356}
357
358MediaStreamVideoSource::MediaStreamVideoSource()
359    : state_(NEW),
360      track_adapter_(new VideoTrackAdapter(
361          ChildProcess::current()->io_message_loop_proxy())),
362      weak_factory_(this) {
363}
364
365MediaStreamVideoSource::~MediaStreamVideoSource() {
366  DCHECK(CalledOnValidThread());
367}
368
369void MediaStreamVideoSource::AddTrack(
370    MediaStreamVideoTrack* track,
371    const VideoCaptureDeliverFrameCB& frame_callback,
372    const blink::WebMediaConstraints& constraints,
373    const ConstraintsCallback& callback) {
374  DCHECK(CalledOnValidThread());
375  DCHECK(!constraints.isNull());
376  DCHECK(std::find(tracks_.begin(), tracks_.end(),
377                   track) == tracks_.end());
378  tracks_.push_back(track);
379
380  requested_constraints_.push_back(
381      RequestedConstraints(track, frame_callback, constraints, callback));
382
383  switch (state_) {
384    case NEW: {
385      // Tab capture and Screen capture needs the maximum requested height
386      // and width to decide on the resolution.
387      int max_requested_width = 0;
388      GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
389                                           &max_requested_width);
390
391      int max_requested_height = 0;
392      GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
393                                           &max_requested_height);
394
395      double max_requested_frame_rate;
396      if (!GetConstraintValueAsDouble(constraints, kMaxFrameRate,
397                                      &max_requested_frame_rate)) {
398        max_requested_frame_rate = kDefaultFrameRate;
399      }
400
401      state_ = RETRIEVING_CAPABILITIES;
402      GetCurrentSupportedFormats(
403          max_requested_width,
404          max_requested_height,
405          max_requested_frame_rate,
406          base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
407                     weak_factory_.GetWeakPtr()));
408
409      break;
410    }
411    case STARTING:
412    case RETRIEVING_CAPABILITIES: {
413      // The |callback| will be triggered once the source has started or
414      // the capabilities have been retrieved.
415      break;
416    }
417    case ENDED:
418    case STARTED: {
419      // Currently, reconfiguring the source is not supported.
420      FinalizeAddTrack();
421    }
422  }
423}
424
425void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
426  DCHECK(CalledOnValidThread());
427  std::vector<MediaStreamVideoTrack*>::iterator it =
428      std::find(tracks_.begin(), tracks_.end(), video_track);
429  DCHECK(it != tracks_.end());
430  tracks_.erase(it);
431
432  // Check if |video_track| is waiting for applying new constraints and remove
433  // the request in that case.
434  for (std::vector<RequestedConstraints>::iterator it =
435           requested_constraints_.begin();
436       it != requested_constraints_.end(); ++it) {
437    if (it->track == video_track) {
438      requested_constraints_.erase(it);
439      break;
440    }
441  }
442  // Call |frame_adapter_->RemoveTrack| here even if adding the track has
443  // failed and |frame_adapter_->AddCallback| has not been called.
444  track_adapter_->RemoveTrack(video_track);
445
446  if (tracks_.empty())
447    StopSource();
448}
449
450const scoped_refptr<base::MessageLoopProxy>&
451MediaStreamVideoSource::io_message_loop() const {
452  DCHECK(CalledOnValidThread());
453  return track_adapter_->io_message_loop();
454}
455
456void MediaStreamVideoSource::DoStopSource() {
457  DCHECK(CalledOnValidThread());
458  DVLOG(3) << "DoStopSource()";
459  if (state_ == ENDED)
460    return;
461  track_adapter_->StopFrameMonitoring();
462  StopSourceImpl();
463  state_ = ENDED;
464  SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
465}
466
467void MediaStreamVideoSource::OnSupportedFormats(
468    const media::VideoCaptureFormats& formats) {
469  DCHECK(CalledOnValidThread());
470  DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
471
472  supported_formats_ = formats;
473  if (!FindBestFormatWithConstraints(supported_formats_,
474                                     &current_format_)) {
475    SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
476    // This object can be deleted after calling FinalizeAddTrack. See comment
477    // in the header file.
478    FinalizeAddTrack();
479    return;
480  }
481
482  state_ = STARTING;
483  DVLOG(3) << "Starting the capturer with " << current_format_.ToString();
484
485  StartSourceImpl(
486      current_format_,
487      base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
488}
489
490bool MediaStreamVideoSource::FindBestFormatWithConstraints(
491    const media::VideoCaptureFormats& formats,
492    media::VideoCaptureFormat* best_format) {
493  DCHECK(CalledOnValidThread());
494  // Find the first constraints that we can fulfill.
495  for (std::vector<RequestedConstraints>::iterator request_it =
496           requested_constraints_.begin();
497       request_it != requested_constraints_.end(); ++request_it) {
498    const blink::WebMediaConstraints& requested_constraints =
499        request_it->constraints;
500
501    // If the source doesn't support capability enumeration it is still ok if
502    // no mandatory constraints have been specified. That just means that
503    // we will start with whatever format is native to the source.
504    if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
505      *best_format = media::VideoCaptureFormat();
506      return true;
507    }
508    blink::WebString unsatisfied_constraint;
509    media::VideoCaptureFormats filtered_formats =
510        FilterFormats(requested_constraints, formats, &unsatisfied_constraint);
511    if (filtered_formats.size() > 0) {
512      // A request with constraints that can be fulfilled.
513      GetBestCaptureFormat(filtered_formats,
514                           requested_constraints,
515                           best_format);
516      return true;
517    }
518  }
519  return false;
520}
521
522void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
523  DCHECK(CalledOnValidThread());
524  DVLOG(3) << "OnStartDone({result =" << result << "})";
525  if (result == MEDIA_DEVICE_OK) {
526    DCHECK_EQ(STARTING, state_);
527    state_ = STARTED;
528    SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
529
530    track_adapter_->StartFrameMonitoring(
531        current_format_.frame_rate,
532        base::Bind(&MediaStreamVideoSource::SetMutedState,
533                   weak_factory_.GetWeakPtr()));
534
535  } else {
536    StopSource();
537  }
538
539  // This object can be deleted after calling FinalizeAddTrack. See comment in
540  // the header file.
541  FinalizeAddTrack();
542}
543
544void MediaStreamVideoSource::FinalizeAddTrack() {
545  DCHECK(CalledOnValidThread());
546  media::VideoCaptureFormats formats;
547  formats.push_back(current_format_);
548
549  std::vector<RequestedConstraints> callbacks;
550  callbacks.swap(requested_constraints_);
551  for (std::vector<RequestedConstraints>::iterator it = callbacks.begin();
552       it != callbacks.end(); ++it) {
553    MediaStreamRequestResult result = MEDIA_DEVICE_OK;
554    blink::WebString unsatisfied_constraint;
555
556    if (HasMandatoryConstraints(it->constraints) &&
557        FilterFormats(it->constraints, formats,
558                      &unsatisfied_constraint).empty())
559      result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
560
561    if (state_ != STARTED && result == MEDIA_DEVICE_OK)
562      result = MEDIA_DEVICE_TRACK_START_FAILURE;
563
564    if (result == MEDIA_DEVICE_OK) {
565      int max_width;
566      int max_height;
567      GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
568      double max_aspect_ratio;
569      double min_aspect_ratio;
570      GetDesiredMinAndMaxAspectRatio(it->constraints,
571                                     &min_aspect_ratio,
572                                     &max_aspect_ratio);
573      double max_frame_rate = 0.0f;
574      GetConstraintValueAsDouble(it->constraints,
575                                 kMaxFrameRate, &max_frame_rate);
576
577      track_adapter_->AddTrack(it->track, it->frame_callback,
578                               max_width, max_height,
579                               min_aspect_ratio, max_aspect_ratio,
580                               max_frame_rate);
581    }
582
583    DVLOG(3) << "FinalizeAddTrack() result " << result;
584
585    if (!it->callback.is_null()) {
586      it->callback.Run(this, result, unsatisfied_constraint);
587    }
588  }
589}
590
591void MediaStreamVideoSource::SetReadyState(
592    blink::WebMediaStreamSource::ReadyState state) {
593  DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
594  DCHECK(CalledOnValidThread());
595  if (!owner().isNull())
596    owner().setReadyState(state);
597  for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
598       it != tracks_.end(); ++it) {
599    (*it)->OnReadyStateChanged(state);
600  }
601}
602
603void MediaStreamVideoSource::SetMutedState(bool muted_state) {
604  DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
605  DCHECK(CalledOnValidThread());
606  if (!owner().isNull()) {
607    owner().setReadyState(muted_state
608        ? blink::WebMediaStreamSource::ReadyStateMuted
609        : blink::WebMediaStreamSource::ReadyStateLive);
610  }
611}
612
613MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
614    MediaStreamVideoTrack* track,
615    const VideoCaptureDeliverFrameCB& frame_callback,
616    const blink::WebMediaConstraints& constraints,
617    const ConstraintsCallback& callback)
618    : track(track),
619      frame_callback(frame_callback),
620      constraints(constraints),
621      callback(callback) {
622}
623
624MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
625}
626
627}  // namespace content
628