clock.c revision 96b3d28bf4b00f62fc8386ff5d487d1830793a3d
13e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 23e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * sched_clock for unstable cpu clocks 33e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 43e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 53e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 6c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Updates and enhancements: 7c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com> 8c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * 93e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Based on code by: 103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Ingo Molnar <mingo@redhat.com> 113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Guillaume Chazarain <guichaz@gmail.com> 123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 13c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 14c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * What: 15c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 16c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) provides a fast (execution time) high resolution 17c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clock with bounded drift between CPUs. The value of cpu_clock(i) 18c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * is monotonic for constant i. The timestamp returned is in nanoseconds. 19c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 20c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 21c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 22c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 23c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 24c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 25c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * There is no strict promise about the base, although it tends to start 26c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * at 0 on boot (but people really shouldn't rely on that). 27c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 28c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) -- can be used from any context, including NMI. 29c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * local_clock() -- is cpu_clock() on the current cpu. 30c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 31ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * sched_clock_cpu(i) 32ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * 33c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * How: 34c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 35c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * The implementation either uses sched_clock() when 36c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the 37c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * sched_clock() is assumed to provide these properties (mostly it means 38c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * the architecture provides a globally synchronized highres time source). 39c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 40c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Otherwise it tries to create a semi stable clock from a mixture of other 41c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clocks, including: 42c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 43c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * - GTOD (clock monotomic) 443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - sched_clock() 453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - explicit idle events 463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 47c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * We use GTOD as base and use sched_clock() deltas to improve resolution. The 48c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * deltas are filtered to provide monotonicity and keeping it within an 49c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * expected window. 503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Furthermore, explicit sleep and wakeup hooks allow us to account for time 523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * that is otherwise invisible (TSC gets stopped). 533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#include <linux/spinlock.h> 566409c4da289d6905f7ae2bd0630438368439bda2Ingo Molnar#include <linux/hardirq.h> 579984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 58b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/percpu.h> 59b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/ktime.h> 60b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/sched.h> 6135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra#include <linux/static_key.h> 626577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra#include <linux/workqueue.h> 633e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 642c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins/* 652c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Scheduler clock - returns current time in nanosec units. 662c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * This is default implementation. 672c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Architectures and sub-architectures can override this. 682c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins */ 692c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickinsunsigned long long __attribute__((weak)) sched_clock(void) 702c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins{ 7192d23f703c608fcb2c8edd74a3fd0f4031e18606Ron return (unsigned long long)(jiffies - INITIAL_JIFFIES) 7292d23f703c608fcb2c8edd74a3fd0f4031e18606Ron * (NSEC_PER_SEC / HZ); 732c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins} 74b6ac23af2c66e114d3a87ef28d56f1ceec283007Divyesh ShahEXPORT_SYMBOL_GPL(sched_clock); 753e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 765bb6b1ea67a73f0665a41726dd7138977b992c6cPeter Zijlstra__read_mostly int sched_clock_running; 77c1955a3d4762e7a9bf84035eb3c4886a900f0d15Peter Zijlstra 783e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 7935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstrastatic struct static_key __sched_clock_stable = STATIC_KEY_INIT; 80d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstrastatic int __sched_clock_stable_early; 8135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 8235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstraint sched_clock_stable(void) 8335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 84d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return static_key_false(&__sched_clock_stable); 8535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 8635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 87d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstrastatic void __set_sched_clock_stable(void) 8835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 8935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (!sched_clock_stable()) 90d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra static_key_slow_inc(&__sched_clock_stable); 91d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra} 92d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 93d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstravoid set_sched_clock_stable(void) 94d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra{ 95d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __sched_clock_stable_early = 1; 96d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 97d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches sched_clock_init() */ 98d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 99d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_running) 100d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return; 101d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 102d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __set_sched_clock_stable(); 10335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 10435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 1056577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstrastatic void __clear_sched_clock_stable(struct work_struct *work) 10635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 10735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra /* XXX worry about clock continuity */ 10835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 109d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra static_key_slow_dec(&__sched_clock_stable); 11035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 1113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1126577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstrastatic DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable); 1136577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra 1146577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstravoid clear_sched_clock_stable(void) 1156577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra{ 116d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __sched_clock_stable_early = 0; 117d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 118d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches sched_clock_init() */ 119d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 120d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_running) 121d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra return; 122d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 123d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra schedule_work(&sched_clock_work); 1246577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra} 1256577e42a3e1633afe762f47da9e00061ee4b9a5ePeter Zijlstra 1263e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastruct sched_clock_data { 1273e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_raw; 1283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_gtod; 1293e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 clock; 1303e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra}; 1313e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 1333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *this_scd(void) 1353e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1363e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &__get_cpu_var(sched_clock_data); 1373e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1383e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1393e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *cpu_sdc(int cpu) 1403e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1413e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &per_cpu(sched_clock_data, cpu); 1423e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1433e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_init(void) 1453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 ktime_now = ktime_to_ns(ktime_get()); 1473e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra int cpu; 1483e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1493e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra for_each_possible_cpu(cpu) { 1503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra struct sched_clock_data *scd = cpu_sdc(cpu); 1513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 152a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra scd->tick_raw = 0; 1533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = ktime_now; 1543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->clock = ktime_now; 1553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 156a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 157a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra sched_clock_running = 1; 158d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 159d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra /* 160d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * Ensure that it is impossible to not do a static_key update. 161d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * 162d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * Either {set,clear}_sched_clock_stable() must see sched_clock_running 163d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * and do the update, or we must see their __sched_clock_stable_early 164d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra * and do the update, or both. 165d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra */ 166d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra smp_mb(); /* matches {set,clear}_sched_clock_stable() */ 167d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra 168d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (__sched_clock_stable_early) 169d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __set_sched_clock_stable(); 170d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra else 171d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra __clear_sched_clock_stable(NULL); 1723e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1743e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 175b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * min, max except they take wrapping into account 176354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 177354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 178354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_min(u64 x, u64 y) 179354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 180354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) < 0 ? x : y; 181354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 182354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 183354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_max(u64 x, u64 y) 184354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 185354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) > 0 ? x : y; 186354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 187354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 188354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra/* 1893e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * update the percpu scd from the raw @now value 1903e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 1913e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - filter out backward motion 192354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * - use the GTOD tick value to create a window to filter crazy TSC values 1933e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 194def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_local(struct sched_clock_data *scd) 1953e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 196def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 now, clock, old_clock, min_clock, max_clock; 197def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra s64 delta; 1983e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 199def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 200def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra now = sched_clock(); 201def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra delta = now - scd->tick_raw; 202354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra if (unlikely(delta < 0)) 203354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra delta = 0; 2043e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 205def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_clock = scd->clock; 206def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 207354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra /* 208354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * scd->clock = clamp(scd->tick_gtod + delta, 209b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * max(scd->tick_gtod, scd->clock), 210b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * scd->tick_gtod + TICK_NSEC); 211354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 2123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 213354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = scd->tick_gtod + delta; 214def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra min_clock = wrap_max(scd->tick_gtod, old_clock); 215def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC); 2163e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 217354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_max(clock, min_clock); 218354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_min(clock, max_clock); 2193e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 220152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock) 221def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 22256b906126d33904d4d67615d0d5b95dbdf1f27caIngo Molnar 223def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return clock; 2243e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2253e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 226def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_remote(struct sched_clock_data *scd) 2273e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 228def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra struct sched_clock_data *my_scd = this_scd(); 229def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 this_clock, remote_clock; 230def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 *ptr, old_val, val; 231def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 232a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#if BITS_PER_LONG != 64 233a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixneragain: 234a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 235a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * Careful here: The local and the remote clock values need to 236a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * be read out atomic as we need to compare the values and 237a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * then update either the local or the remote side. So the 238a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * cmpxchg64 below only protects one readout. 239a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 240a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must reread via sched_clock_local() in the retry case on 241a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 32bit as an NMI could use sched_clock_local() via the 242a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * tracer and hit between the readout of 243a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 244a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 245a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner this_clock = sched_clock_local(my_scd); 246a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 247a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must enforce atomic readout on 32bit, otherwise the 248a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update on the remote cpu can hit inbetween the readout of 249a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 250a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 251a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner remote_clock = cmpxchg64(&scd->clock, 0, 0); 252a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#else 253a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 254a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * On 64bit the read of [my]scd->clock is atomic versus the 255a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update, so we can avoid the above 32bit dance. 256a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 257def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(my_scd); 258def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 259def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra this_clock = my_scd->clock; 260def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra remote_clock = scd->clock; 261a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#endif 262def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 263def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 264def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Use the opportunity that we have both locks 265def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * taken to couple the two clocks: we take the 266def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * larger time as the latest time for both 267def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * runqueues. (this creates monotonic movement) 268def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 269def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (likely((s64)(remote_clock - this_clock) < 0)) { 270def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &scd->clock; 271def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = remote_clock; 272def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = this_clock; 2733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } else { 274def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 275def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Should be rare, but possible: 276def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 277def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &my_scd->clock; 278def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = this_clock; 279def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = remote_clock; 2803e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 281def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 282152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(ptr, old_val, val) != old_val) 283def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 284def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 285def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return val; 2863e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 288c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 289c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock(), but requires local IRQs to be disabled. 290c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 291c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 292c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 2933e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrau64 sched_clock_cpu(int cpu) 2943e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 295b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar struct sched_clock_data *scd; 296def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 clock; 297def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 29835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 299b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar return sched_clock(); 300a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 301a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 302a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return 0ull; 303a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 30496b3d28bf4b00f62fc8386ff5d487d1830793a3dFernando Luis Vazquez Cao preempt_disable_notrace(); 305def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra scd = cpu_sdc(cpu); 3063e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 307def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (cpu != smp_processor_id()) 308def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_remote(scd); 309def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra else 310def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_local(scd); 31196b3d28bf4b00f62fc8386ff5d487d1830793a3dFernando Luis Vazquez Cao preempt_enable_notrace(); 312e4e4e534faa3c2be4e165ce414f44b76ada7208cIngo Molnar 3133e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return clock; 3143e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3153e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3163e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_tick(void) 3173e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3188325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra struct sched_clock_data *scd; 3193e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 now, now_gtod; 3203e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 32135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 3228325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return; 3238325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 324a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 325a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return; 326a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 3273e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra WARN_ON_ONCE(!irqs_disabled()); 3283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3298325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra scd = this_scd(); 3303e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra now_gtod = ktime_to_ns(ktime_get()); 331a83bc47c33ab182f1e48977fd5a04024d713c75eSteven Rostedt now = sched_clock(); 3323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_raw = now; 3343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = now_gtod; 335def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(scd); 3363e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3373e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3383e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 3393e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We are going deep-idle (irqs are disabled): 3403e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3413e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_sleep_event(void) 3423e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3433e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra sched_clock_cpu(smp_processor_id()); 3443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); 3463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3473e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 3483e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We just idled delta nanoseconds (called with irqs disabled): 3493e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_wakeup_event(u64 delta_ns) 3513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3521c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner if (timekeeping_suspended) 3531c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner return; 3541c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner 355354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra sched_clock_tick(); 3563e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra touch_softlockup_watchdog(); 3573e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3583e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 3593e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 360c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 361c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * As outlined at the top, provides a fast, high resolution, nanosecond 362c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * time source that is monotonic per cpu argument and has bounded drift 363c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * between cpus. 364c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 365c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 366c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 367c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 368c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 369c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 370c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 371b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller{ 372d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_stable()) 37335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(cpu); 37435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 37535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 376b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 377b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller 378c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 379c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock() for the current cpu. Time will only be observed 380c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * to be monotonic if care is taken to only compare timestampt taken on the 381c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * same CPU. 382c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 383c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 384c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 385c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 386c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 387d375b4e0fa3771343b370be0d876a1963c02e0a0Peter Zijlstra if (!sched_clock_stable()) 38835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(raw_smp_processor_id()); 38935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 39035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 391c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 392c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 3938325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 3948325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3958325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstravoid sched_clock_init(void) 3968325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3978325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra sched_clock_running = 1; 3988325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 3998325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 4008325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstrau64 sched_clock_cpu(int cpu) 4018325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 4028325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra if (unlikely(!sched_clock_running)) 4038325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return 0; 4048325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 4058325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return sched_clock(); 4068325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 4078325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 408c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 40976a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra{ 41035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 411b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 41276a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 413c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 414c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 41535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 416c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 417c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 418b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 41976a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 4204c9fe8ad813b257a2b9ddf0f752105a75a7dae63Ingo MolnarEXPORT_SYMBOL_GPL(cpu_clock); 421c676329abb2b8359d9a5d734dec0c81779823fd6Peter ZijlstraEXPORT_SYMBOL_GPL(local_clock); 422