13e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 23e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * sched_clock for unstable cpu clocks 33e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 43e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 53e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 6c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Updates and enhancements: 7c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com> 8c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * 93e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Based on code by: 103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Ingo Molnar <mingo@redhat.com> 113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Guillaume Chazarain <guichaz@gmail.com> 123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 13c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 14c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * What: 15c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 16c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) provides a fast (execution time) high resolution 17c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clock with bounded drift between CPUs. The value of cpu_clock(i) 18c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * is monotonic for constant i. The timestamp returned is in nanoseconds. 19c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 20c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 21c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 22c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 23c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 24c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 25c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * There is no strict promise about the base, although it tends to start 26c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * at 0 on boot (but people really shouldn't rely on that). 27c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 28c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) -- can be used from any context, including NMI. 29c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * local_clock() -- is cpu_clock() on the current cpu. 30c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 31ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * sched_clock_cpu(i) 32ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * 33c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * How: 34c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 35c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * The implementation either uses sched_clock() when 36c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the 37c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * sched_clock() is assumed to provide these properties (mostly it means 38c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * the architecture provides a globally synchronized highres time source). 39c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 40c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Otherwise it tries to create a semi stable clock from a mixture of other 41c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clocks, including: 42c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 43c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * - GTOD (clock monotomic) 443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - sched_clock() 453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - explicit idle events 463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 47c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * We use GTOD as base and use sched_clock() deltas to improve resolution. The 48c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * deltas are filtered to provide monotonicity and keeping it within an 49c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * expected window. 503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Furthermore, explicit sleep and wakeup hooks allow us to account for time 523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * that is otherwise invisible (TSC gets stopped). 533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#include <linux/spinlock.h> 566409c4da289d6905f7ae2bd0630438368439bda2Ingo Molnar#include <linux/hardirq.h> 579984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 58b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/percpu.h> 59b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/ktime.h> 60b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/sched.h> 6135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra#include <linux/static_key.h> 626577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra#include <linux/workqueue.h> 6352f5684c8e1ec7463192aba8e2916df49807511aGideon Israel Dsouza#include <linux/compiler.h> 643e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 652c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins/* 662c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Scheduler clock - returns current time in nanosec units. 672c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * This is default implementation. 682c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Architectures and sub-architectures can override this. 692c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins */ 7052f5684c8e1ec7463192aba8e2916df49807511aGideon Israel Dsouzaunsigned long long __weak sched_clock(void) 712c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins{ 7292d23f703c608fcb2c8edd74a3fd0f4031e18606Ron return (unsigned long long)(jiffies - INITIAL_JIFFIES) 7392d23f703c608fcb2c8edd74a3fd0f4031e18606Ron * (NSEC_PER_SEC / HZ); 742c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins} 75b6ac23af2c66e114d3a87ef28d56f1ceec283007Divyesh ShahEXPORT_SYMBOL_GPL(sched_clock); 763e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 775bb6b1ea67a73f0665a41726dd7138977b992c6cPeter Zijlstra__read_mostly int sched_clock_running; 78c1955a3d4762e7a9bf84035eb3c4886a900f0d15Peter Zijlstra 793e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 8035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstrastatic struct static_key __sched_clock_stable = STATIC_KEY_INIT; 81d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstrastatic int __sched_clock_stable_early; 8235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 8335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstraint sched_clock_stable(void) 8435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 85d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return static_key_false(&__sched_clock_stable); 8635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 8735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 88d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstrastatic void __set_sched_clock_stable(void) 8935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 9035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (!sched_clock_stable()) 91d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra static_key_slow_inc(&__sched_clock_stable); 92d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra} 93d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 94d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstravoid set_sched_clock_stable(void) 95d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra{ 96d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __sched_clock_stable_early = 1; 97d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 98d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches sched_clock_init() */ 99d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 100d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_running) 101d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return; 102d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 103d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __set_sched_clock_stable(); 10435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 10535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 1066577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstrastatic void __clear_sched_clock_stable(struct work_struct *work) 10735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 10835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra /* XXX worry about clock continuity */ 10935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 110d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra static_key_slow_dec(&__sched_clock_stable); 11135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 1123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1136577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstrastatic DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable); 1146577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra 1156577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstravoid clear_sched_clock_stable(void) 1166577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra{ 117d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __sched_clock_stable_early = 0; 118d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 119d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches sched_clock_init() */ 120d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 121d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_running) 122d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return; 123d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 124d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra schedule_work(&sched_clock_work); 1256577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra} 1266577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra 1273e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastruct sched_clock_data { 1283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_raw; 1293e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_gtod; 1303e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 clock; 1313e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra}; 1323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 1343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1353e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *this_scd(void) 1363e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 13722127e93c587afa01e4f7225d2d1cf1d26ae7dfeChristoph Lameter return this_cpu_ptr(&sched_clock_data); 1383e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1393e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1403e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *cpu_sdc(int cpu) 1413e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1423e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &per_cpu(sched_clock_data, cpu); 1433e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_init(void) 1463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1473e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 ktime_now = ktime_to_ns(ktime_get()); 1483e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra int cpu; 1493e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra for_each_possible_cpu(cpu) { 1513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra struct sched_clock_data *scd = cpu_sdc(cpu); 1523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 153a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra scd->tick_raw = 0; 1543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = ktime_now; 1553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->clock = ktime_now; 1563e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 157a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 158a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra sched_clock_running = 1; 159d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 160d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra /* 161d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * Ensure that it is impossible to not do a static_key update. 162d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * 163d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * Either {set,clear}_sched_clock_stable() must see sched_clock_running 164d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * and do the update, or we must see their __sched_clock_stable_early 165d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * and do the update, or both. 166d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra */ 167d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches {set,clear}_sched_clock_stable() */ 168d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 169d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (__sched_clock_stable_early) 170d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __set_sched_clock_stable(); 171d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra else 172d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __clear_sched_clock_stable(NULL); 1733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1743e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1753e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 176b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * min, max except they take wrapping into account 177354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 178354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 179354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_min(u64 x, u64 y) 180354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 181354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) < 0 ? x : y; 182354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 183354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 184354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_max(u64 x, u64 y) 185354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 186354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) > 0 ? x : y; 187354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 188354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 189354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra/* 1903e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * update the percpu scd from the raw @now value 1913e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 1923e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - filter out backward motion 193354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * - use the GTOD tick value to create a window to filter crazy TSC values 1943e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 195def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_local(struct sched_clock_data *scd) 1963e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 197def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 now, clock, old_clock, min_clock, max_clock; 198def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra s64 delta; 1993e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 200def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 201def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra now = sched_clock(); 202def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra delta = now - scd->tick_raw; 203354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra if (unlikely(delta < 0)) 204354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra delta = 0; 2053e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 206def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_clock = scd->clock; 207def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 208354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra /* 209354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * scd->clock = clamp(scd->tick_gtod + delta, 210b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * max(scd->tick_gtod, scd->clock), 211b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * scd->tick_gtod + TICK_NSEC); 212354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 2133e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 214354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = scd->tick_gtod + delta; 215def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra min_clock = wrap_max(scd->tick_gtod, old_clock); 216def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC); 2173e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 218354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_max(clock, min_clock); 219354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_min(clock, max_clock); 2203e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 221152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock) 222def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 22356b906126d33904d4d67615d0d5b95dbdf1f27caIngo Molnar 224def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return clock; 2253e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2263e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 227def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_remote(struct sched_clock_data *scd) 2283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 229def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra struct sched_clock_data *my_scd = this_scd(); 230def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 this_clock, remote_clock; 231def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 *ptr, old_val, val; 232def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 233a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#if BITS_PER_LONG != 64 234a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixneragain: 235a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 236a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * Careful here: The local and the remote clock values need to 237a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * be read out atomic as we need to compare the values and 238a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * then update either the local or the remote side. So the 239a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * cmpxchg64 below only protects one readout. 240a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 241a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must reread via sched_clock_local() in the retry case on 242a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 32bit as an NMI could use sched_clock_local() via the 243a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * tracer and hit between the readout of 244a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 245a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 246a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner this_clock = sched_clock_local(my_scd); 247a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 248a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must enforce atomic readout on 32bit, otherwise the 249a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update on the remote cpu can hit inbetween the readout of 250a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 251a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 252a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner remote_clock = cmpxchg64(&scd->clock, 0, 0); 253a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#else 254a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 255a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * On 64bit the read of [my]scd->clock is atomic versus the 256a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update, so we can avoid the above 32bit dance. 257a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 258def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(my_scd); 259def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 260def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra this_clock = my_scd->clock; 261def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra remote_clock = scd->clock; 262a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#endif 263def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 264def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 265def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Use the opportunity that we have both locks 266def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * taken to couple the two clocks: we take the 267def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * larger time as the latest time for both 268def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * runqueues. (this creates monotonic movement) 269def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 270def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (likely((s64)(remote_clock - this_clock) < 0)) { 271def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &scd->clock; 272def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = remote_clock; 273def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = this_clock; 2743e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } else { 275def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 276def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Should be rare, but possible: 277def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 278def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &my_scd->clock; 279def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = this_clock; 280def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = remote_clock; 2813e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 282def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 283152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(ptr, old_val, val) != old_val) 284def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 285def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 286def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return val; 2873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2883e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 289c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 290c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock(), but requires local IRQs to be disabled. 291c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 292c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 293c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 2943e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrau64 sched_clock_cpu(int cpu) 2953e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 296b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar struct sched_clock_data *scd; 297def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 clock; 298def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 29935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 300b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar return sched_clock(); 301a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 302a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 303a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return 0ull; 304a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 30596b3d28bf4b00f62fc8386ff5d487d1830793a3dFernando Luis Vazquez Cao preempt_disable_notrace(); 306def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra scd = cpu_sdc(cpu); 3073e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 308def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (cpu != smp_processor_id()) 309def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_remote(scd); 310def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra else 311def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_local(scd); 31296b3d28bf4b00f62fc8386ff5d487d1830793a3dFernando Luis Vazquez Cao preempt_enable_notrace(); 313e4e4e534faa3c2be4e165ce414f44b76ada7208cIngo Molnar 3143e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return clock; 3153e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3163e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3173e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_tick(void) 3183e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3198325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra struct sched_clock_data *scd; 3203e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 now, now_gtod; 3213e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 32235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 3238325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return; 3248325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 325a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 326a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return; 327a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 3283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra WARN_ON_ONCE(!irqs_disabled()); 3293e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3308325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra scd = this_scd(); 3313e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra now_gtod = ktime_to_ns(ktime_get()); 332a83bc47c33ab182f1e48977fd5a04024d713c75eSteven Rostedt now = sched_clock(); 3333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_raw = now; 3353e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = now_gtod; 336def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(scd); 3373e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3383e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3393e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 3403e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We are going deep-idle (irqs are disabled): 3413e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3423e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_sleep_event(void) 3433e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra sched_clock_cpu(smp_processor_id()); 3453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); 3473e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3483e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 3493e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We just idled delta nanoseconds (called with irqs disabled): 3503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_wakeup_event(u64 delta_ns) 3523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3531c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner if (timekeeping_suspended) 3541c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner return; 3551c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner 356354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra sched_clock_tick(); 3573e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra touch_softlockup_watchdog(); 3583e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3593e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 3603e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 361c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 362c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * As outlined at the top, provides a fast, high resolution, nanosecond 363c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * time source that is monotonic per cpu argument and has bounded drift 364c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * between cpus. 365c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 366c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 367c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 368c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 369c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 370c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 371c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 372b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller{ 373d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_stable()) 37435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(cpu); 37535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 37635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 377b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 378b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller 379c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 380c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock() for the current cpu. Time will only be observed 381c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * to be monotonic if care is taken to only compare timestampt taken on the 382c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * same CPU. 383c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 384c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 385c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 386c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 387c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 388d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_stable()) 38935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(raw_smp_processor_id()); 39035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 39135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 392c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 393c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 3948325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 3958325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3968325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstravoid sched_clock_init(void) 3978325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3988325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra sched_clock_running = 1; 3998325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 4008325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 4018325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstrau64 sched_clock_cpu(int cpu) 4028325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 4038325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra if (unlikely(!sched_clock_running)) 4048325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return 0; 4058325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 4068325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return sched_clock(); 4078325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 4088325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 409c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 41076a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra{ 41135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 412b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 41376a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 414c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 415c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 41635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 417c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 418c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 419b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 42076a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 4214c9fe8ad813b257a2b9ddf0f752105a75a7dae63Ingo MolnarEXPORT_SYMBOL_GPL(cpu_clock); 422c676329abb2b8359d9a5d734dec0c81779823fd6Peter ZijlstraEXPORT_SYMBOL_GPL(local_clock); 423