videosource.cc revision d4e598d57aed714a599444a7eab5e8fdde52a950
1/* 2 * libjingle 3 * Copyright 2012, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include "talk/app/webrtc/videosource.h" 29 30#include <vector> 31 32#include "talk/app/webrtc/mediaconstraintsinterface.h" 33#include "talk/session/media/channelmanager.h" 34 35using cricket::CaptureState; 36using webrtc::MediaConstraintsInterface; 37using webrtc::MediaSourceInterface; 38 39namespace { 40 41const double kRoundingTruncation = 0.0005; 42 43enum { 44 MSG_VIDEOCAPTURESTATECONNECT, 45 MSG_VIDEOCAPTURESTATEDISCONNECT, 46 MSG_VIDEOCAPTURESTATECHANGE, 47}; 48 49// Default resolution. If no constraint is specified, this is the resolution we 50// will use. 51static const cricket::VideoFormatPod kDefaultFormat = 52 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}; 53 54// List of formats used if the camera doesn't support capability enumeration. 55static const cricket::VideoFormatPod kVideoFormats[] = { 56 {1920, 1080, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 57 {1280, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 58 {960, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 59 {640, 360, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 60 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 61 {320, 240, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 62 {320, 180, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY} 63}; 64 65MediaSourceInterface::SourceState 66GetReadyState(cricket::CaptureState state) { 67 switch (state) { 68 case cricket::CS_STARTING: 69 return MediaSourceInterface::kInitializing; 70 case cricket::CS_RUNNING: 71 return MediaSourceInterface::kLive; 72 case cricket::CS_FAILED: 73 case cricket::CS_NO_DEVICE: 74 case cricket::CS_STOPPED: 75 return MediaSourceInterface::kEnded; 76 case cricket::CS_PAUSED: 77 return MediaSourceInterface::kMuted; 78 default: 79 ASSERT(false && "GetReadyState unknown state"); 80 } 81 return MediaSourceInterface::kEnded; 82} 83 84void SetUpperLimit(int new_limit, int* original_limit) { 85 if (*original_limit < 0 || new_limit < *original_limit) 86 *original_limit = new_limit; 87} 88 89// Updates |format_upper_limit| from |constraint|. 90// If constraint.maxFoo is smaller than format_upper_limit.foo, 91// set format_upper_limit.foo to constraint.maxFoo. 92void SetUpperLimitFromConstraint( 93 const MediaConstraintsInterface::Constraint& constraint, 94 cricket::VideoFormat* format_upper_limit) { 95 if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 96 int value = rtc::FromString<int>(constraint.value); 97 SetUpperLimit(value, &(format_upper_limit->width)); 98 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 99 int value = rtc::FromString<int>(constraint.value); 100 SetUpperLimit(value, &(format_upper_limit->height)); 101 } 102} 103 104// Fills |format_out| with the max width and height allowed by |constraints|. 105void FromConstraintsForScreencast( 106 const MediaConstraintsInterface::Constraints& constraints, 107 cricket::VideoFormat* format_out) { 108 typedef MediaConstraintsInterface::Constraints::const_iterator 109 ConstraintsIterator; 110 111 cricket::VideoFormat upper_limit(-1, -1, 0, 0); 112 for (ConstraintsIterator constraints_it = constraints.begin(); 113 constraints_it != constraints.end(); ++constraints_it) 114 SetUpperLimitFromConstraint(*constraints_it, &upper_limit); 115 116 if (upper_limit.width >= 0) 117 format_out->width = upper_limit.width; 118 if (upper_limit.height >= 0) 119 format_out->height = upper_limit.height; 120} 121 122// Returns true if |constraint| is fulfilled. |format_out| can differ from 123// |format_in| if the format is changed by the constraint. Ie - the frame rate 124// can be changed by setting maxFrameRate. 125bool NewFormatWithConstraints( 126 const MediaConstraintsInterface::Constraint& constraint, 127 const cricket::VideoFormat& format_in, 128 bool mandatory, 129 cricket::VideoFormat* format_out) { 130 ASSERT(format_out != NULL); 131 *format_out = format_in; 132 133 if (constraint.key == MediaConstraintsInterface::kMinWidth) { 134 int value = rtc::FromString<int>(constraint.value); 135 return (value <= format_in.width); 136 } else if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 137 int value = rtc::FromString<int>(constraint.value); 138 return (value >= format_in.width); 139 } else if (constraint.key == MediaConstraintsInterface::kMinHeight) { 140 int value = rtc::FromString<int>(constraint.value); 141 return (value <= format_in.height); 142 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 143 int value = rtc::FromString<int>(constraint.value); 144 return (value >= format_in.height); 145 } else if (constraint.key == MediaConstraintsInterface::kMinFrameRate) { 146 int value = rtc::FromString<int>(constraint.value); 147 return (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)); 148 } else if (constraint.key == MediaConstraintsInterface::kMaxFrameRate) { 149 int value = rtc::FromString<int>(constraint.value); 150 if (value == 0) { 151 if (mandatory) { 152 // TODO(ronghuawu): Convert the constraint value to float when sub-1fps 153 // is supported by the capturer. 154 return false; 155 } else { 156 value = 1; 157 } 158 } 159 if (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)) { 160 format_out->interval = cricket::VideoFormat::FpsToInterval(value); 161 return true; 162 } else { 163 return false; 164 } 165 } else if (constraint.key == MediaConstraintsInterface::kMinAspectRatio) { 166 double value = rtc::FromString<double>(constraint.value); 167 // The aspect ratio in |constraint.value| has been converted to a string and 168 // back to a double, so it may have a rounding error. 169 // E.g if the value 1/3 is converted to a string, the string will not have 170 // infinite length. 171 // We add a margin of 0.0005 which is high enough to detect the same aspect 172 // ratio but small enough to avoid matching wrong aspect ratios. 173 double ratio = static_cast<double>(format_in.width) / format_in.height; 174 return (value <= ratio + kRoundingTruncation); 175 } else if (constraint.key == MediaConstraintsInterface::kMaxAspectRatio) { 176 double value = rtc::FromString<double>(constraint.value); 177 double ratio = static_cast<double>(format_in.width) / format_in.height; 178 // Subtract 0.0005 to avoid rounding problems. Same as above. 179 const double kRoundingTruncation = 0.0005; 180 return (value >= ratio - kRoundingTruncation); 181 } else if (constraint.key == MediaConstraintsInterface::kNoiseReduction || 182 constraint.key == MediaConstraintsInterface::kLeakyBucket || 183 constraint.key == 184 MediaConstraintsInterface::kTemporalLayeredScreencast) { 185 // These are actually options, not constraints, so they can be satisfied 186 // regardless of the format. 187 return true; 188 } 189 LOG(LS_WARNING) << "Found unknown MediaStream constraint. Name:" 190 << constraint.key << " Value:" << constraint.value; 191 return false; 192} 193 194// Removes cricket::VideoFormats from |formats| that don't meet |constraint|. 195void FilterFormatsByConstraint( 196 const MediaConstraintsInterface::Constraint& constraint, 197 bool mandatory, 198 std::vector<cricket::VideoFormat>* formats) { 199 std::vector<cricket::VideoFormat>::iterator format_it = 200 formats->begin(); 201 while (format_it != formats->end()) { 202 // Modify the format_it to fulfill the constraint if possible. 203 // Delete it otherwise. 204 if (!NewFormatWithConstraints(constraint, (*format_it), 205 mandatory, &(*format_it))) { 206 format_it = formats->erase(format_it); 207 } else { 208 ++format_it; 209 } 210 } 211} 212 213// Returns a vector of cricket::VideoFormat that best match |constraints|. 214std::vector<cricket::VideoFormat> FilterFormats( 215 const MediaConstraintsInterface::Constraints& mandatory, 216 const MediaConstraintsInterface::Constraints& optional, 217 const std::vector<cricket::VideoFormat>& supported_formats) { 218 typedef MediaConstraintsInterface::Constraints::const_iterator 219 ConstraintsIterator; 220 std::vector<cricket::VideoFormat> candidates = supported_formats; 221 222 for (ConstraintsIterator constraints_it = mandatory.begin(); 223 constraints_it != mandatory.end(); ++constraints_it) 224 FilterFormatsByConstraint(*constraints_it, true, &candidates); 225 226 if (candidates.size() == 0) 227 return candidates; 228 229 // Ok - all mandatory checked and we still have a candidate. 230 // Let's try filtering using the optional constraints. 231 for (ConstraintsIterator constraints_it = optional.begin(); 232 constraints_it != optional.end(); ++constraints_it) { 233 std::vector<cricket::VideoFormat> current_candidates = candidates; 234 FilterFormatsByConstraint(*constraints_it, false, ¤t_candidates); 235 if (current_candidates.size() > 0) { 236 candidates = current_candidates; 237 } 238 } 239 240 // We have done as good as we can to filter the supported resolutions. 241 return candidates; 242} 243 244// Find the format that best matches the default video size. 245// Constraints are optional and since the performance of a video call 246// might be bad due to bitrate limitations, CPU, and camera performance, 247// it is better to select a resolution that is as close as possible to our 248// default and still meets the contraints. 249const cricket::VideoFormat& GetBestCaptureFormat( 250 const std::vector<cricket::VideoFormat>& formats) { 251 ASSERT(formats.size() > 0); 252 253 int default_area = kDefaultFormat.width * kDefaultFormat.height; 254 255 std::vector<cricket::VideoFormat>::const_iterator it = formats.begin(); 256 std::vector<cricket::VideoFormat>::const_iterator best_it = formats.begin(); 257 int best_diff = abs(default_area - it->width* it->height); 258 for (; it != formats.end(); ++it) { 259 int diff = abs(default_area - it->width* it->height); 260 if (diff < best_diff) { 261 best_diff = diff; 262 best_it = it; 263 } 264 } 265 return *best_it; 266} 267 268// Set |option| to the highest-priority value of |key| in the constraints. 269// Return false if the key is mandatory, and the value is invalid. 270bool ExtractOption(const MediaConstraintsInterface* all_constraints, 271 const std::string& key, cricket::Settable<bool>* option) { 272 size_t mandatory = 0; 273 bool value; 274 if (FindConstraint(all_constraints, key, &value, &mandatory)) { 275 option->Set(value); 276 return true; 277 } 278 279 return mandatory == 0; 280} 281 282// Search |all_constraints| for known video options. Apply all options that are 283// found with valid values, and return false if any mandatory video option was 284// found with an invalid value. 285bool ExtractVideoOptions(const MediaConstraintsInterface* all_constraints, 286 cricket::VideoOptions* options) { 287 bool all_valid = true; 288 289 all_valid &= ExtractOption(all_constraints, 290 MediaConstraintsInterface::kNoiseReduction, 291 &(options->video_noise_reduction)); 292 all_valid &= ExtractOption(all_constraints, 293 MediaConstraintsInterface::kLeakyBucket, 294 &(options->video_leaky_bucket)); 295 all_valid &= ExtractOption(all_constraints, 296 MediaConstraintsInterface::kTemporalLayeredScreencast, 297 &(options->video_temporal_layer_screencast)); 298 299 return all_valid; 300} 301 302class FrameInputWrapper : public cricket::VideoRenderer { 303 public: 304 explicit FrameInputWrapper(cricket::VideoCapturer* capturer) 305 : capturer_(capturer) { 306 ASSERT(capturer_ != NULL); 307 } 308 309 virtual ~FrameInputWrapper() {} 310 311 // VideoRenderer implementation. 312 virtual bool SetSize(int width, int height, int reserved) OVERRIDE { 313 return true; 314 } 315 316 virtual bool RenderFrame(const cricket::VideoFrame* frame) OVERRIDE { 317 if (!capturer_->IsRunning()) { 318 return true; 319 } 320 321 // This signal will be made on media engine render thread. The clients 322 // of this signal should have no assumptions on what thread this signal 323 // come from. 324 capturer_->SignalVideoFrame(capturer_, frame); 325 return true; 326 } 327 328 private: 329 cricket::VideoCapturer* capturer_; 330 int width_; 331 int height_; 332 333 DISALLOW_COPY_AND_ASSIGN(FrameInputWrapper); 334}; 335 336} // anonymous namespace 337 338namespace webrtc { 339 340rtc::scoped_refptr<VideoSource> VideoSource::Create( 341 cricket::ChannelManager* channel_manager, 342 cricket::VideoCapturer* capturer, 343 const webrtc::MediaConstraintsInterface* constraints) { 344 ASSERT(channel_manager != NULL); 345 ASSERT(capturer != NULL); 346 rtc::scoped_refptr<VideoSource> source( 347 new rtc::RefCountedObject<VideoSource>(channel_manager, 348 capturer)); 349 source->Initialize(constraints); 350 return source; 351} 352 353VideoSource::VideoSource(cricket::ChannelManager* channel_manager, 354 cricket::VideoCapturer* capturer) 355 : channel_manager_(channel_manager), 356 video_capturer_(capturer), 357 state_(kInitializing) { 358 channel_manager_->SignalVideoCaptureStateChange.connect( 359 this, &VideoSource::OnStateChange); 360} 361 362VideoSource::~VideoSource() { 363 channel_manager_->StopVideoCapture(video_capturer_.get(), format_); 364 channel_manager_->SignalVideoCaptureStateChange.disconnect(this); 365} 366 367void VideoSource::Initialize( 368 const webrtc::MediaConstraintsInterface* constraints) { 369 370 std::vector<cricket::VideoFormat> formats; 371 if (video_capturer_->GetSupportedFormats() && 372 video_capturer_->GetSupportedFormats()->size() > 0) { 373 formats = *video_capturer_->GetSupportedFormats(); 374 } else if (video_capturer_->IsScreencast()) { 375 // The screen capturer can accept any resolution and we will derive the 376 // format from the constraints if any. 377 // Note that this only affects tab capturing, not desktop capturing, 378 // since desktop capturer does not respect the VideoFormat passed in. 379 formats.push_back(cricket::VideoFormat(kDefaultFormat)); 380 } else { 381 // The VideoCapturer implementation doesn't support capability enumeration. 382 // We need to guess what the camera support. 383 for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) { 384 formats.push_back(cricket::VideoFormat(kVideoFormats[i])); 385 } 386 } 387 388 if (constraints) { 389 MediaConstraintsInterface::Constraints mandatory_constraints = 390 constraints->GetMandatory(); 391 MediaConstraintsInterface::Constraints optional_constraints; 392 optional_constraints = constraints->GetOptional(); 393 394 if (video_capturer_->IsScreencast()) { 395 // Use the maxWidth and maxHeight allowed by constraints for screencast. 396 FromConstraintsForScreencast(mandatory_constraints, &(formats[0])); 397 } 398 399 formats = FilterFormats(mandatory_constraints, optional_constraints, 400 formats); 401 } 402 403 if (formats.size() == 0) { 404 LOG(LS_WARNING) << "Failed to find a suitable video format."; 405 SetState(kEnded); 406 return; 407 } 408 409 cricket::VideoOptions options; 410 if (!ExtractVideoOptions(constraints, &options)) { 411 LOG(LS_WARNING) << "Could not satisfy mandatory options."; 412 SetState(kEnded); 413 return; 414 } 415 options_.SetAll(options); 416 417 format_ = GetBestCaptureFormat(formats); 418 // Start the camera with our best guess. 419 // TODO(perkj): Should we try again with another format it it turns out that 420 // the camera doesn't produce frames with the correct format? Or will 421 // cricket::VideCapturer be able to re-scale / crop to the requested 422 // resolution? 423 if (!channel_manager_->StartVideoCapture(video_capturer_.get(), format_)) { 424 SetState(kEnded); 425 return; 426 } 427 // Initialize hasn't succeeded until a successful state change has occurred. 428} 429 430cricket::VideoRenderer* VideoSource::FrameInput() { 431 // Defer creation of frame_input_ until it's needed, e.g. the local video 432 // sources will never need it. 433 if (!frame_input_) { 434 frame_input_.reset(new FrameInputWrapper(video_capturer_.get())); 435 } 436 return frame_input_.get(); 437} 438 439void VideoSource::AddSink(cricket::VideoRenderer* output) { 440 channel_manager_->AddVideoRenderer(video_capturer_.get(), output); 441} 442 443void VideoSource::RemoveSink(cricket::VideoRenderer* output) { 444 channel_manager_->RemoveVideoRenderer(video_capturer_.get(), output); 445} 446 447// OnStateChange listens to the ChannelManager::SignalVideoCaptureStateChange. 448// This signal is triggered for all video capturers. Not only the one we are 449// interested in. 450void VideoSource::OnStateChange(cricket::VideoCapturer* capturer, 451 cricket::CaptureState capture_state) { 452 if (capturer == video_capturer_.get()) { 453 SetState(GetReadyState(capture_state)); 454 } 455} 456 457void VideoSource::SetState(SourceState new_state) { 458 if (VERIFY(state_ != new_state)) { 459 state_ = new_state; 460 FireOnChanged(); 461 } 462} 463 464} // namespace webrtc 465