1/* 2 * libjingle 3 * Copyright 2012, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include "talk/app/webrtc/localvideosource.h" 29 30#include <vector> 31 32#include "talk/app/webrtc/mediaconstraintsinterface.h" 33#include "talk/session/media/channelmanager.h" 34 35using cricket::CaptureState; 36using webrtc::MediaConstraintsInterface; 37using webrtc::MediaSourceInterface; 38 39namespace webrtc { 40 41// Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b 42// They are declared as static members in mediastreaminterface.h 43const char MediaConstraintsInterface::kMinAspectRatio[] = "minAspectRatio"; 44const char MediaConstraintsInterface::kMaxAspectRatio[] = "maxAspectRatio"; 45const char MediaConstraintsInterface::kMaxWidth[] = "maxWidth"; 46const char MediaConstraintsInterface::kMinWidth[] = "minWidth"; 47const char MediaConstraintsInterface::kMaxHeight[] = "maxHeight"; 48const char MediaConstraintsInterface::kMinHeight[] = "minHeight"; 49const char MediaConstraintsInterface::kMaxFrameRate[] = "maxFrameRate"; 50const char MediaConstraintsInterface::kMinFrameRate[] = "minFrameRate"; 51 52// Google-specific keys 53const char MediaConstraintsInterface::kNoiseReduction[] = "googNoiseReduction"; 54const char MediaConstraintsInterface::kLeakyBucket[] = "googLeakyBucket"; 55const char MediaConstraintsInterface::kTemporalLayeredScreencast[] = 56 "googTemporalLayeredScreencast"; 57 58} // namespace webrtc 59 60namespace { 61 62const double kRoundingTruncation = 0.0005; 63 64enum { 65 MSG_VIDEOCAPTURESTATECONNECT, 66 MSG_VIDEOCAPTURESTATEDISCONNECT, 67 MSG_VIDEOCAPTURESTATECHANGE, 68}; 69 70// Default resolution. If no constraint is specified, this is the resolution we 71// will use. 72static const cricket::VideoFormatPod kDefaultResolution = 73 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}; 74 75// List of formats used if the camera doesn't support capability enumeration. 76static const cricket::VideoFormatPod kVideoFormats[] = { 77 {1920, 1080, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 78 {1280, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 79 {960, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 80 {640, 360, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 81 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 82 {320, 240, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 83 {320, 180, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY} 84}; 85 86MediaSourceInterface::SourceState 87GetReadyState(cricket::CaptureState state) { 88 switch (state) { 89 case cricket::CS_STARTING: 90 return MediaSourceInterface::kInitializing; 91 case cricket::CS_RUNNING: 92 return MediaSourceInterface::kLive; 93 case cricket::CS_FAILED: 94 case cricket::CS_NO_DEVICE: 95 case cricket::CS_STOPPED: 96 return MediaSourceInterface::kEnded; 97 case cricket::CS_PAUSED: 98 return MediaSourceInterface::kMuted; 99 default: 100 ASSERT(false && "GetReadyState unknown state"); 101 } 102 return MediaSourceInterface::kEnded; 103} 104 105void SetUpperLimit(int new_limit, int* original_limit) { 106 if (*original_limit < 0 || new_limit < *original_limit) 107 *original_limit = new_limit; 108} 109 110// Updates |format_upper_limit| from |constraint|. 111// If constraint.maxFoo is smaller than format_upper_limit.foo, 112// set format_upper_limit.foo to constraint.maxFoo. 113void SetUpperLimitFromConstraint( 114 const MediaConstraintsInterface::Constraint& constraint, 115 cricket::VideoFormat* format_upper_limit) { 116 if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 117 int value = talk_base::FromString<int>(constraint.value); 118 SetUpperLimit(value, &(format_upper_limit->width)); 119 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 120 int value = talk_base::FromString<int>(constraint.value); 121 SetUpperLimit(value, &(format_upper_limit->height)); 122 } 123} 124 125// Fills |format_out| with the max width and height allowed by |constraints|. 126void FromConstraintsForScreencast( 127 const MediaConstraintsInterface::Constraints& constraints, 128 cricket::VideoFormat* format_out) { 129 typedef MediaConstraintsInterface::Constraints::const_iterator 130 ConstraintsIterator; 131 132 cricket::VideoFormat upper_limit(-1, -1, 0, 0); 133 for (ConstraintsIterator constraints_it = constraints.begin(); 134 constraints_it != constraints.end(); ++constraints_it) 135 SetUpperLimitFromConstraint(*constraints_it, &upper_limit); 136 137 if (upper_limit.width >= 0) 138 format_out->width = upper_limit.width; 139 if (upper_limit.height >= 0) 140 format_out->height = upper_limit.height; 141} 142 143// Returns true if |constraint| is fulfilled. |format_out| can differ from 144// |format_in| if the format is changed by the constraint. Ie - the frame rate 145// can be changed by setting maxFrameRate. 146bool NewFormatWithConstraints( 147 const MediaConstraintsInterface::Constraint& constraint, 148 const cricket::VideoFormat& format_in, 149 bool mandatory, 150 cricket::VideoFormat* format_out) { 151 ASSERT(format_out != NULL); 152 *format_out = format_in; 153 154 if (constraint.key == MediaConstraintsInterface::kMinWidth) { 155 int value = talk_base::FromString<int>(constraint.value); 156 return (value <= format_in.width); 157 } else if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 158 int value = talk_base::FromString<int>(constraint.value); 159 return (value >= format_in.width); 160 } else if (constraint.key == MediaConstraintsInterface::kMinHeight) { 161 int value = talk_base::FromString<int>(constraint.value); 162 return (value <= format_in.height); 163 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 164 int value = talk_base::FromString<int>(constraint.value); 165 return (value >= format_in.height); 166 } else if (constraint.key == MediaConstraintsInterface::kMinFrameRate) { 167 int value = talk_base::FromString<int>(constraint.value); 168 return (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)); 169 } else if (constraint.key == MediaConstraintsInterface::kMaxFrameRate) { 170 int value = talk_base::FromString<int>(constraint.value); 171 if (value == 0) { 172 if (mandatory) { 173 // TODO(ronghuawu): Convert the constraint value to float when sub-1fps 174 // is supported by the capturer. 175 return false; 176 } else { 177 value = 1; 178 } 179 } 180 if (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)) { 181 format_out->interval = cricket::VideoFormat::FpsToInterval(value); 182 return true; 183 } else { 184 return false; 185 } 186 } else if (constraint.key == MediaConstraintsInterface::kMinAspectRatio) { 187 double value = talk_base::FromString<double>(constraint.value); 188 // The aspect ratio in |constraint.value| has been converted to a string and 189 // back to a double, so it may have a rounding error. 190 // E.g if the value 1/3 is converted to a string, the string will not have 191 // infinite length. 192 // We add a margin of 0.0005 which is high enough to detect the same aspect 193 // ratio but small enough to avoid matching wrong aspect ratios. 194 double ratio = static_cast<double>(format_in.width) / format_in.height; 195 return (value <= ratio + kRoundingTruncation); 196 } else if (constraint.key == MediaConstraintsInterface::kMaxAspectRatio) { 197 double value = talk_base::FromString<double>(constraint.value); 198 double ratio = static_cast<double>(format_in.width) / format_in.height; 199 // Subtract 0.0005 to avoid rounding problems. Same as above. 200 const double kRoundingTruncation = 0.0005; 201 return (value >= ratio - kRoundingTruncation); 202 } else if (constraint.key == MediaConstraintsInterface::kNoiseReduction || 203 constraint.key == MediaConstraintsInterface::kLeakyBucket || 204 constraint.key == 205 MediaConstraintsInterface::kTemporalLayeredScreencast) { 206 // These are actually options, not constraints, so they can be satisfied 207 // regardless of the format. 208 return true; 209 } 210 LOG(LS_WARNING) << "Found unknown MediaStream constraint. Name:" 211 << constraint.key << " Value:" << constraint.value; 212 return false; 213} 214 215// Removes cricket::VideoFormats from |formats| that don't meet |constraint|. 216void FilterFormatsByConstraint( 217 const MediaConstraintsInterface::Constraint& constraint, 218 bool mandatory, 219 std::vector<cricket::VideoFormat>* formats) { 220 std::vector<cricket::VideoFormat>::iterator format_it = 221 formats->begin(); 222 while (format_it != formats->end()) { 223 // Modify the format_it to fulfill the constraint if possible. 224 // Delete it otherwise. 225 if (!NewFormatWithConstraints(constraint, (*format_it), 226 mandatory, &(*format_it))) { 227 format_it = formats->erase(format_it); 228 } else { 229 ++format_it; 230 } 231 } 232} 233 234// Returns a vector of cricket::VideoFormat that best match |constraints|. 235std::vector<cricket::VideoFormat> FilterFormats( 236 const MediaConstraintsInterface::Constraints& mandatory, 237 const MediaConstraintsInterface::Constraints& optional, 238 const std::vector<cricket::VideoFormat>& supported_formats) { 239 typedef MediaConstraintsInterface::Constraints::const_iterator 240 ConstraintsIterator; 241 std::vector<cricket::VideoFormat> candidates = supported_formats; 242 243 for (ConstraintsIterator constraints_it = mandatory.begin(); 244 constraints_it != mandatory.end(); ++constraints_it) 245 FilterFormatsByConstraint(*constraints_it, true, &candidates); 246 247 if (candidates.size() == 0) 248 return candidates; 249 250 // Ok - all mandatory checked and we still have a candidate. 251 // Let's try filtering using the optional constraints. 252 for (ConstraintsIterator constraints_it = optional.begin(); 253 constraints_it != optional.end(); ++constraints_it) { 254 std::vector<cricket::VideoFormat> current_candidates = candidates; 255 FilterFormatsByConstraint(*constraints_it, false, ¤t_candidates); 256 if (current_candidates.size() > 0) { 257 candidates = current_candidates; 258 } 259 } 260 261 // We have done as good as we can to filter the supported resolutions. 262 return candidates; 263} 264 265// Find the format that best matches the default video size. 266// Constraints are optional and since the performance of a video call 267// might be bad due to bitrate limitations, CPU, and camera performance, 268// it is better to select a resolution that is as close as possible to our 269// default and still meets the contraints. 270const cricket::VideoFormat& GetBestCaptureFormat( 271 const std::vector<cricket::VideoFormat>& formats) { 272 ASSERT(formats.size() > 0); 273 274 int default_area = kDefaultResolution.width * kDefaultResolution.height; 275 276 std::vector<cricket::VideoFormat>::const_iterator it = formats.begin(); 277 std::vector<cricket::VideoFormat>::const_iterator best_it = formats.begin(); 278 int best_diff = abs(default_area - it->width* it->height); 279 for (; it != formats.end(); ++it) { 280 int diff = abs(default_area - it->width* it->height); 281 if (diff < best_diff) { 282 best_diff = diff; 283 best_it = it; 284 } 285 } 286 return *best_it; 287} 288 289// Set |option| to the highest-priority value of |key| in the constraints. 290// Return false if the key is mandatory, and the value is invalid. 291bool ExtractOption(const MediaConstraintsInterface* all_constraints, 292 const std::string& key, cricket::Settable<bool>* option) { 293 size_t mandatory = 0; 294 bool value; 295 if (FindConstraint(all_constraints, key, &value, &mandatory)) { 296 option->Set(value); 297 return true; 298 } 299 300 return mandatory == 0; 301} 302 303// Search |all_constraints| for known video options. Apply all options that are 304// found with valid values, and return false if any mandatory video option was 305// found with an invalid value. 306bool ExtractVideoOptions(const MediaConstraintsInterface* all_constraints, 307 cricket::VideoOptions* options) { 308 bool all_valid = true; 309 310 all_valid &= ExtractOption(all_constraints, 311 MediaConstraintsInterface::kNoiseReduction, 312 &(options->video_noise_reduction)); 313 all_valid &= ExtractOption(all_constraints, 314 MediaConstraintsInterface::kLeakyBucket, 315 &(options->video_leaky_bucket)); 316 all_valid &= ExtractOption(all_constraints, 317 MediaConstraintsInterface::kTemporalLayeredScreencast, 318 &(options->video_temporal_layer_screencast)); 319 320 return all_valid; 321} 322 323} // anonymous namespace 324 325namespace webrtc { 326 327talk_base::scoped_refptr<LocalVideoSource> LocalVideoSource::Create( 328 cricket::ChannelManager* channel_manager, 329 cricket::VideoCapturer* capturer, 330 const webrtc::MediaConstraintsInterface* constraints) { 331 ASSERT(channel_manager != NULL); 332 ASSERT(capturer != NULL); 333 talk_base::scoped_refptr<LocalVideoSource> source( 334 new talk_base::RefCountedObject<LocalVideoSource>(channel_manager, 335 capturer)); 336 source->Initialize(constraints); 337 return source; 338} 339 340LocalVideoSource::LocalVideoSource(cricket::ChannelManager* channel_manager, 341 cricket::VideoCapturer* capturer) 342 : channel_manager_(channel_manager), 343 video_capturer_(capturer), 344 state_(kInitializing) { 345 channel_manager_->SignalVideoCaptureStateChange.connect( 346 this, &LocalVideoSource::OnStateChange); 347} 348 349LocalVideoSource::~LocalVideoSource() { 350 channel_manager_->StopVideoCapture(video_capturer_.get(), format_); 351 channel_manager_->SignalVideoCaptureStateChange.disconnect(this); 352} 353 354void LocalVideoSource::Initialize( 355 const webrtc::MediaConstraintsInterface* constraints) { 356 357 std::vector<cricket::VideoFormat> formats; 358 if (video_capturer_->GetSupportedFormats() && 359 video_capturer_->GetSupportedFormats()->size() > 0) { 360 formats = *video_capturer_->GetSupportedFormats(); 361 } else if (video_capturer_->IsScreencast()) { 362 // The screen capturer can accept any resolution and we will derive the 363 // format from the constraints if any. 364 // Note that this only affects tab capturing, not desktop capturing, 365 // since desktop capturer does not respect the VideoFormat passed in. 366 formats.push_back(cricket::VideoFormat(kDefaultResolution)); 367 } else { 368 // The VideoCapturer implementation doesn't support capability enumeration. 369 // We need to guess what the camera support. 370 for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) { 371 formats.push_back(cricket::VideoFormat(kVideoFormats[i])); 372 } 373 } 374 375 if (constraints) { 376 MediaConstraintsInterface::Constraints mandatory_constraints = 377 constraints->GetMandatory(); 378 MediaConstraintsInterface::Constraints optional_constraints; 379 optional_constraints = constraints->GetOptional(); 380 381 if (video_capturer_->IsScreencast()) { 382 // Use the maxWidth and maxHeight allowed by constraints for screencast. 383 FromConstraintsForScreencast(mandatory_constraints, &(formats[0])); 384 } 385 386 formats = FilterFormats(mandatory_constraints, optional_constraints, 387 formats); 388 } 389 390 if (formats.size() == 0) { 391 LOG(LS_WARNING) << "Failed to find a suitable video format."; 392 SetState(kEnded); 393 return; 394 } 395 396 cricket::VideoOptions options; 397 if (!ExtractVideoOptions(constraints, &options)) { 398 LOG(LS_WARNING) << "Could not satisfy mandatory options."; 399 SetState(kEnded); 400 return; 401 } 402 options_.SetAll(options); 403 404 format_ = GetBestCaptureFormat(formats); 405 // Start the camera with our best guess. 406 // TODO(perkj): Should we try again with another format it it turns out that 407 // the camera doesn't produce frames with the correct format? Or will 408 // cricket::VideCapturer be able to re-scale / crop to the requested 409 // resolution? 410 if (!channel_manager_->StartVideoCapture(video_capturer_.get(), format_)) { 411 SetState(kEnded); 412 return; 413 } 414 // Initialize hasn't succeeded until a successful state change has occurred. 415} 416 417void LocalVideoSource::AddSink(cricket::VideoRenderer* output) { 418 channel_manager_->AddVideoRenderer(video_capturer_.get(), output); 419} 420 421void LocalVideoSource::RemoveSink(cricket::VideoRenderer* output) { 422 channel_manager_->RemoveVideoRenderer(video_capturer_.get(), output); 423} 424 425// OnStateChange listens to the ChannelManager::SignalVideoCaptureStateChange. 426// This signal is triggered for all video capturers. Not only the one we are 427// interested in. 428void LocalVideoSource::OnStateChange(cricket::VideoCapturer* capturer, 429 cricket::CaptureState capture_state) { 430 if (capturer == video_capturer_.get()) { 431 SetState(GetReadyState(capture_state)); 432 } 433} 434 435void LocalVideoSource::SetState(SourceState new_state) { 436 if (VERIFY(state_ != new_state)) { 437 state_ = new_state; 438 FireOnChanged(); 439 } 440} 441 442} // namespace webrtc 443