clock.c revision 35af99e646c7f7ea46dc2977601e9e71a51dadd5
13e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 23e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * sched_clock for unstable cpu clocks 33e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 43e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 53e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 6c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Updates and enhancements: 7c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com> 8c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * 93e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Based on code by: 103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Ingo Molnar <mingo@redhat.com> 113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Guillaume Chazarain <guichaz@gmail.com> 123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 13c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 14c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * What: 15c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 16c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) provides a fast (execution time) high resolution 17c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clock with bounded drift between CPUs. The value of cpu_clock(i) 18c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * is monotonic for constant i. The timestamp returned is in nanoseconds. 19c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 20c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 21c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 22c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 23c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 24c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 25c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * There is no strict promise about the base, although it tends to start 26c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * at 0 on boot (but people really shouldn't rely on that). 27c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 28c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) -- can be used from any context, including NMI. 29c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * local_clock() -- is cpu_clock() on the current cpu. 30c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 31ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * sched_clock_cpu(i) 32ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * 33c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * How: 34c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 35c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * The implementation either uses sched_clock() when 36c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the 37c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * sched_clock() is assumed to provide these properties (mostly it means 38c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * the architecture provides a globally synchronized highres time source). 39c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 40c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Otherwise it tries to create a semi stable clock from a mixture of other 41c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clocks, including: 42c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 43c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * - GTOD (clock monotomic) 443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - sched_clock() 453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - explicit idle events 463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 47c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * We use GTOD as base and use sched_clock() deltas to improve resolution. The 48c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * deltas are filtered to provide monotonicity and keeping it within an 49c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * expected window. 503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Furthermore, explicit sleep and wakeup hooks allow us to account for time 523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * that is otherwise invisible (TSC gets stopped). 533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#include <linux/spinlock.h> 566409c4da289d6905f7ae2bd0630438368439bda2Ingo Molnar#include <linux/hardirq.h> 579984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 58b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/percpu.h> 59b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/ktime.h> 60b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/sched.h> 6135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra#include <linux/static_key.h> 623e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 632c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins/* 642c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Scheduler clock - returns current time in nanosec units. 652c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * This is default implementation. 662c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Architectures and sub-architectures can override this. 672c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins */ 682c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickinsunsigned long long __attribute__((weak)) sched_clock(void) 692c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins{ 7092d23f703c608fcb2c8edd74a3fd0f4031e18606Ron return (unsigned long long)(jiffies - INITIAL_JIFFIES) 7192d23f703c608fcb2c8edd74a3fd0f4031e18606Ron * (NSEC_PER_SEC / HZ); 722c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins} 73b6ac23af2c66e114d3a87ef28d56f1ceec283007Divyesh ShahEXPORT_SYMBOL_GPL(sched_clock); 743e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 755bb6b1ea67a73f0665a41726dd7138977b992c6cPeter Zijlstra__read_mostly int sched_clock_running; 76c1955a3d4762e7a9bf84035eb3c4886a900f0d15Peter Zijlstra 773e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 7835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstrastatic struct static_key __sched_clock_stable = STATIC_KEY_INIT; 7935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 8035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstraint sched_clock_stable(void) 8135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 8235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (static_key_false(&__sched_clock_stable)) 8335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return false; 8435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return true; 8535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 8635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 8735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstravoid set_sched_clock_stable(void) 8835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 8935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (!sched_clock_stable()) 9035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra static_key_slow_dec(&__sched_clock_stable); 9135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 9235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 9335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstravoid clear_sched_clock_stable(void) 9435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra{ 9535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra /* XXX worry about clock continuity */ 9635af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 9735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra static_key_slow_inc(&__sched_clock_stable); 9835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra} 993e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1003e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastruct sched_clock_data { 1013e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_raw; 1023e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_gtod; 1033e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 clock; 1043e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra}; 1053e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1063e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 1073e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1083e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *this_scd(void) 1093e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &__get_cpu_var(sched_clock_data); 1113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1133e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *cpu_sdc(int cpu) 1143e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1153e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &per_cpu(sched_clock_data, cpu); 1163e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1173e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1183e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_init(void) 1193e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 1203e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 ktime_now = ktime_to_ns(ktime_get()); 1213e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra int cpu; 1223e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1233e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra for_each_possible_cpu(cpu) { 1243e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra struct sched_clock_data *scd = cpu_sdc(cpu); 1253e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 126a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra scd->tick_raw = 0; 1273e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = ktime_now; 1283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->clock = ktime_now; 1293e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 130a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 131a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra sched_clock_running = 1; 1323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 135b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * min, max except they take wrapping into account 136354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 137354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 138354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_min(u64 x, u64 y) 139354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 140354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) < 0 ? x : y; 141354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 142354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 143354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_max(u64 x, u64 y) 144354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 145354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) > 0 ? x : y; 146354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 147354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 148354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra/* 1493e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * update the percpu scd from the raw @now value 1503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 1513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - filter out backward motion 152354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * - use the GTOD tick value to create a window to filter crazy TSC values 1533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 154def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_local(struct sched_clock_data *scd) 1553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 156def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 now, clock, old_clock, min_clock, max_clock; 157def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra s64 delta; 1583e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 159def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 160def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra now = sched_clock(); 161def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra delta = now - scd->tick_raw; 162354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra if (unlikely(delta < 0)) 163354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra delta = 0; 1643e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 165def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_clock = scd->clock; 166def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 167354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra /* 168354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * scd->clock = clamp(scd->tick_gtod + delta, 169b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * max(scd->tick_gtod, scd->clock), 170b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * scd->tick_gtod + TICK_NSEC); 171354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 1723e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 173354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = scd->tick_gtod + delta; 174def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra min_clock = wrap_max(scd->tick_gtod, old_clock); 175def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC); 1763e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 177354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_max(clock, min_clock); 178354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_min(clock, max_clock); 1793e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 180152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock) 181def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 18256b906126d33904d4d67615d0d5b95dbdf1f27caIngo Molnar 183def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return clock; 1843e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1853e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 186def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_remote(struct sched_clock_data *scd) 1873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 188def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra struct sched_clock_data *my_scd = this_scd(); 189def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 this_clock, remote_clock; 190def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 *ptr, old_val, val; 191def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 192a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#if BITS_PER_LONG != 64 193a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixneragain: 194a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 195a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * Careful here: The local and the remote clock values need to 196a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * be read out atomic as we need to compare the values and 197a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * then update either the local or the remote side. So the 198a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * cmpxchg64 below only protects one readout. 199a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 200a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must reread via sched_clock_local() in the retry case on 201a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 32bit as an NMI could use sched_clock_local() via the 202a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * tracer and hit between the readout of 203a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 204a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 205a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner this_clock = sched_clock_local(my_scd); 206a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 207a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must enforce atomic readout on 32bit, otherwise the 208a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update on the remote cpu can hit inbetween the readout of 209a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 210a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 211a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner remote_clock = cmpxchg64(&scd->clock, 0, 0); 212a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#else 213a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 214a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * On 64bit the read of [my]scd->clock is atomic versus the 215a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update, so we can avoid the above 32bit dance. 216a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 217def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(my_scd); 218def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 219def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra this_clock = my_scd->clock; 220def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra remote_clock = scd->clock; 221a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#endif 222def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 223def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 224def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Use the opportunity that we have both locks 225def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * taken to couple the two clocks: we take the 226def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * larger time as the latest time for both 227def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * runqueues. (this creates monotonic movement) 228def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 229def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (likely((s64)(remote_clock - this_clock) < 0)) { 230def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &scd->clock; 231def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = remote_clock; 232def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = this_clock; 2333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } else { 234def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 235def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Should be rare, but possible: 236def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 237def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &my_scd->clock; 238def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = this_clock; 239def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = remote_clock; 2403e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 241def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 242152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(ptr, old_val, val) != old_val) 243def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 244def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 245def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return val; 2463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2473e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 248c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 249c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock(), but requires local IRQs to be disabled. 250c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 251c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 252c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 2533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrau64 sched_clock_cpu(int cpu) 2543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 255b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar struct sched_clock_data *scd; 256def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 clock; 257def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 25835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 259b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar return sched_clock(); 260a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 261a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 262a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return 0ull; 263a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 264ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra preempt_disable(); 265def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra scd = cpu_sdc(cpu); 2663e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 267def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (cpu != smp_processor_id()) 268def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_remote(scd); 269def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra else 270def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_local(scd); 271ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra preempt_enable(); 272e4e4e534faa3c2be4e165ce414f44b76ada7208cIngo Molnar 2733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return clock; 2743e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2753e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2763e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_tick(void) 2773e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 2788325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra struct sched_clock_data *scd; 2793e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 now, now_gtod; 2803e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 28135af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (sched_clock_stable()) 2828325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return; 2838325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 284a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 285a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return; 286a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 2873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra WARN_ON_ONCE(!irqs_disabled()); 2883e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2898325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra scd = this_scd(); 2903e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra now_gtod = ktime_to_ns(ktime_get()); 291a83bc47c33ab182f1e48977fd5a04024d713c75eSteven Rostedt now = sched_clock(); 2923e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2933e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_raw = now; 2943e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = now_gtod; 295def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(scd); 2963e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2973e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2983e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 2993e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We are going deep-idle (irqs are disabled): 3003e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3013e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_sleep_event(void) 3023e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3033e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra sched_clock_cpu(smp_processor_id()); 3043e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3053e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); 3063e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 3073e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 3083e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We just idled delta nanoseconds (called with irqs disabled): 3093e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 3103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_wakeup_event(u64 delta_ns) 3113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 3121c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner if (timekeeping_suspended) 3131c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner return; 3141c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner 315354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra sched_clock_tick(); 3163e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra touch_softlockup_watchdog(); 3173e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 3183e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 3193e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 320c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 321c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * As outlined at the top, provides a fast, high resolution, nanosecond 322c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * time source that is monotonic per cpu argument and has bounded drift 323c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * between cpus. 324c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 325c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 326c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 327c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 328c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 329c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 330c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 331b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller{ 33235af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (static_key_false(&__sched_clock_stable)) 33335af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(cpu); 33435af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 33535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 336b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 337b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller 338c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 339c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock() for the current cpu. Time will only be observed 340c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * to be monotonic if care is taken to only compare timestampt taken on the 341c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * same CPU. 342c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 343c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 344c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 345c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 346c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 34735af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra if (static_key_false(&__sched_clock_stable)) 34835af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock_cpu(raw_smp_processor_id()); 34935af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra 35035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 351c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 352c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 3538325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 3548325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3558325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstravoid sched_clock_init(void) 3568325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3578325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra sched_clock_running = 1; 3588325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 3598325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3608325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstrau64 sched_clock_cpu(int cpu) 3618325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3628325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra if (unlikely(!sched_clock_running)) 3638325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return 0; 3648325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3658325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return sched_clock(); 3668325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 3678325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 368c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 36976a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra{ 37035af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 371b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 37276a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 373c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 374c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 37535af99e646c7f7ea46dc2977601e9e71a51dadd5Peter Zijlstra return sched_clock(); 376c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 377c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 378b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 37976a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 3804c9fe8ad813b257a2b9ddf0f752105a75a7dae63Ingo MolnarEXPORT_SYMBOL_GPL(cpu_clock); 381c676329abb2b8359d9a5d734dec0c81779823fd6Peter ZijlstraEXPORT_SYMBOL_GPL(local_clock); 382