1a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 3a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// found in the LICENSE file. 4a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 5a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)#include "content/browser/media/capture/video_capture_oracle.h" 6a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 7a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)#include "base/debug/trace_event.h" 8a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 9a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)namespace content { 10a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 11a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)namespace { 12a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 13a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// This value controls how many redundant, timer-base captures occur when the 14a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// content is static. Redundantly capturing the same frame allows iterative 15a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// quality enhancement, and also allows the buffer to fill in "buffered mode". 16a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// 17a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// TODO(nick): Controlling this here is a hack and a layering violation, since 18a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// it's a strategy specific to the WebRTC consumer, and probably just papers 19a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// over some frame dropping and quality bugs. It should either be controlled at 20a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// a higher level, or else redundant frame generation should be pushed down 21a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)// further into the WebRTC encoding stack. 22a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)const int kNumRedundantCapturesOfStaticContent = 200; 23a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 24a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} // anonymous namespace 25a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 26a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)VideoCaptureOracle::VideoCaptureOracle(base::TimeDelta capture_period, 27a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) bool events_are_reliable) 28a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) : capture_period_(capture_period), 29a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) frame_number_(0), 30a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) last_delivered_frame_number_(0), 31a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) sampler_(capture_period_, 32a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) events_are_reliable, 33a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) kNumRedundantCapturesOfStaticContent) {} 34a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 35a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)bool VideoCaptureOracle::ObserveEventAndDecideCapture( 365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Event event, 375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) base::TimeTicks event_time) { 38a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // Record |event| and decide whether it's a good time to capture. 39a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) const bool content_is_dirty = (event == kCompositorUpdate || 40a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) event == kSoftwarePaint); 41a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) bool should_sample; 42a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (content_is_dirty) { 43a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) frame_number_++; 44a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) should_sample = sampler_.AddEventAndConsiderSampling(event_time); 45a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } else { 46a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) should_sample = sampler_.IsOverdueForSamplingAt(event_time); 47a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 48a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return should_sample; 49a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 50a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 51a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)int VideoCaptureOracle::RecordCapture() { 52a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) sampler_.RecordSample(); 53a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return frame_number_; 54a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 55a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 56a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)bool VideoCaptureOracle::CompleteCapture(int frame_number, 575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) base::TimeTicks timestamp) { 58a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // Drop frame if previous frame number is higher or we're trying to deliver 59a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // a frame with the same timestamp. 60a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (last_delivered_frame_number_ > frame_number || 61a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) last_delivered_frame_timestamp_ == timestamp) { 62a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) LOG(ERROR) << "Frame with same timestamp or out of order delivery. " 63a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) << "Dropping frame."; 64a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return false; 65a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 66a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 67a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (last_delivered_frame_timestamp_ > timestamp) { 68a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // We should not get here unless time was adjusted backwards. 69a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) LOG(ERROR) << "Frame with past timestamp (" << timestamp.ToInternalValue() 70a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) << ") was delivered"; 71a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 72a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 73a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) last_delivered_frame_number_ = frame_number; 74a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) last_delivered_frame_timestamp_ = timestamp; 75a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 76a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return true; 77a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 78a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 79a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)SmoothEventSampler::SmoothEventSampler(base::TimeDelta capture_period, 80a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) bool events_are_reliable, 81a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) int redundant_capture_goal) 82a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) : events_are_reliable_(events_are_reliable), 83a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) capture_period_(capture_period), 84a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) redundant_capture_goal_(redundant_capture_goal), 85a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_capacity_(capture_period + capture_period / 2), 86a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) overdue_sample_count_(0), 87a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_(token_bucket_capacity_) { 88a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) DCHECK_GT(capture_period_.InMicroseconds(), 0); 89a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 90a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)bool SmoothEventSampler::AddEventAndConsiderSampling( 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) base::TimeTicks event_time) { 93a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) DCHECK(!event_time.is_null()); 94a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 95a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // Add tokens to the bucket based on advancement in time. Then, re-bound the 96a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // number of tokens in the bucket. Overflow occurs when there is too much 97a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // time between events (a common case), or when RecordSample() is not being 98a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // called often enough (a bug). On the other hand, if RecordSample() is being 99a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // called too often (e.g., as a reaction to IsOverdueForSamplingAt()), the 100a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // bucket will underflow. 101a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (!current_event_.is_null()) { 102a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (current_event_ < event_time) { 103a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_ += event_time - current_event_; 104a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (token_bucket_ > token_bucket_capacity_) 105a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_ = token_bucket_capacity_; 106a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 107a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // Side note: If the system clock is reset, causing |current_event_| to be 108a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // greater than |event_time|, everything here will simply gracefully adjust. 109a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (token_bucket_ < base::TimeDelta()) 110a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_ = base::TimeDelta(); 111a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) TRACE_COUNTER1("mirroring", 112a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) "MirroringTokenBucketUsec", 113a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) std::max<int64>(0, token_bucket_.InMicroseconds())); 114a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 115a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) current_event_ = event_time; 116a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 117a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // Return true if one capture period's worth of tokens are in the bucket. 118a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return token_bucket_ >= capture_period_; 119a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 120a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 121a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)void SmoothEventSampler::RecordSample() { 122a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) token_bucket_ -= capture_period_; 123a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) TRACE_COUNTER1("mirroring", 124a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) "MirroringTokenBucketUsec", 125a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) std::max<int64>(0, token_bucket_.InMicroseconds())); 126a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 127a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) bool was_paused = overdue_sample_count_ == redundant_capture_goal_; 128a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (HasUnrecordedEvent()) { 129a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) last_sample_ = current_event_; 130a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) overdue_sample_count_ = 0; 131a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } else { 132a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) ++overdue_sample_count_; 133a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 134a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) bool is_paused = overdue_sample_count_ == redundant_capture_goal_; 135a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 136f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) VLOG_IF(0, !was_paused && is_paused) 137f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) << "Tab content unchanged for " << redundant_capture_goal_ 138f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) << " frames; capture will halt until content changes."; 139f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) VLOG_IF(0, was_paused && !is_paused) 140f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) << "Content changed; capture will resume."; 141a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 142a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 1435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)bool SmoothEventSampler::IsOverdueForSamplingAt(base::TimeTicks event_time) 1445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const { 145a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) DCHECK(!event_time.is_null()); 146a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 147a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // If we don't get events on compositor updates on this platform, then we 148a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // don't reliably know whether we're dirty. 149a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (events_are_reliable_) { 150a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (!HasUnrecordedEvent() && 151a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) overdue_sample_count_ >= redundant_capture_goal_) { 152a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return false; // Not dirty. 153a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 154a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) } 155a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 1565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (last_sample_.is_null()) 1575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return true; 1585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 159a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // If we're dirty but not yet old, then we've recently gotten updates, so we 160a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) // won't request a sample just yet. 161a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) base::TimeDelta dirty_interval = event_time - last_sample_; 162a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) if (dirty_interval < capture_period_ * 4) 163a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return false; 164a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) else 165a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return true; 166a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 167a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 168a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)bool SmoothEventSampler::HasUnrecordedEvent() const { 169a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) return !current_event_.is_null() && current_event_ != last_sample_; 170a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} 171a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) 172a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles)} // namespace content 173