clock.c revision ef08f0fff87630d4f67ceb09514d8b444df833f8
13e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 23e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * sched_clock for unstable cpu clocks 33e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 43e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 53e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 6c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Updates and enhancements: 7c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com> 8c300ba252829e9325e08f0af60687add94445b25Steven Rostedt * 93e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Based on code by: 103e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Ingo Molnar <mingo@redhat.com> 113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Guillaume Chazarain <guichaz@gmail.com> 123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 13c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 14c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * What: 15c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 16c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) provides a fast (execution time) high resolution 17c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clock with bounded drift between CPUs. The value of cpu_clock(i) 18c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * is monotonic for constant i. The timestamp returned is in nanoseconds. 19c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 20c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 21c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 22c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 23c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 24c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 25c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * There is no strict promise about the base, although it tends to start 26c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * at 0 on boot (but people really shouldn't rely on that). 27c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 28c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * cpu_clock(i) -- can be used from any context, including NMI. 29c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * local_clock() -- is cpu_clock() on the current cpu. 30c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 31ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * sched_clock_cpu(i) 32ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra * 33c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * How: 34c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 35c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * The implementation either uses sched_clock() when 36c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the 37c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * sched_clock() is assumed to provide these properties (mostly it means 38c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * the architecture provides a globally synchronized highres time source). 39c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 40c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Otherwise it tries to create a semi stable clock from a mixture of other 41c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * clocks, including: 42c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 43c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * - GTOD (clock monotomic) 443e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - sched_clock() 453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - explicit idle events 463e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 47c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * We use GTOD as base and use sched_clock() deltas to improve resolution. The 48c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * deltas are filtered to provide monotonicity and keeping it within an 49c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * expected window. 503e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * Furthermore, explicit sleep and wakeup hooks allow us to account for time 523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * that is otherwise invisible (TSC gets stopped). 533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#include <linux/spinlock.h> 566409c4da289d6905f7ae2bd0630438368439bda2Ingo Molnar#include <linux/hardirq.h> 579984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 58b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/percpu.h> 59b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/ktime.h> 60b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar#include <linux/sched.h> 613e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 622c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins/* 632c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Scheduler clock - returns current time in nanosec units. 642c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * This is default implementation. 652c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins * Architectures and sub-architectures can override this. 662c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins */ 672c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickinsunsigned long long __attribute__((weak)) sched_clock(void) 682c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins{ 6992d23f703c608fcb2c8edd74a3fd0f4031e18606Ron return (unsigned long long)(jiffies - INITIAL_JIFFIES) 7092d23f703c608fcb2c8edd74a3fd0f4031e18606Ron * (NSEC_PER_SEC / HZ); 712c3d103ba90827cfb478bf10464d9b5b9cea369cHugh Dickins} 72b6ac23af2c66e114d3a87ef28d56f1ceec283007Divyesh ShahEXPORT_SYMBOL_GPL(sched_clock); 733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 745bb6b1ea67a73f0665a41726dd7138977b992c6cPeter Zijlstra__read_mostly int sched_clock_running; 75c1955a3d4762e7a9bf84035eb3c4886a900f0d15Peter Zijlstra 763e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 77b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar__read_mostly int sched_clock_stable; 783e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 793e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastruct sched_clock_data { 803e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_raw; 813e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 tick_gtod; 823e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 clock; 833e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra}; 843e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 853e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 863e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *this_scd(void) 883e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 893e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &__get_cpu_var(sched_clock_data); 903e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 913e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 923e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrastatic inline struct sched_clock_data *cpu_sdc(int cpu) 933e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 943e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return &per_cpu(sched_clock_data, cpu); 953e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 963e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 973e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_init(void) 983e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 993e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 ktime_now = ktime_to_ns(ktime_get()); 1003e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra int cpu; 1013e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1023e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra for_each_possible_cpu(cpu) { 1033e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra struct sched_clock_data *scd = cpu_sdc(cpu); 1043e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 105a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra scd->tick_raw = 0; 1063e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = ktime_now; 1073e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->clock = ktime_now; 1083e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 109a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 110a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra sched_clock_running = 1; 1113e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 1133e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 114b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * min, max except they take wrapping into account 115354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 116354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 117354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_min(u64 x, u64 y) 118354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 119354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) < 0 ? x : y; 120354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 121354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 122354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstrastatic inline u64 wrap_max(u64 x, u64 y) 123354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra{ 124354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra return (s64)(x - y) > 0 ? x : y; 125354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra} 126354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra 127354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra/* 1283e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * update the percpu scd from the raw @now value 1293e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * 1303e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * - filter out backward motion 131354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * - use the GTOD tick value to create a window to filter crazy TSC values 1323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 133def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_local(struct sched_clock_data *scd) 1343e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 135def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 now, clock, old_clock, min_clock, max_clock; 136def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra s64 delta; 1373e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 138def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 139def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra now = sched_clock(); 140def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra delta = now - scd->tick_raw; 141354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra if (unlikely(delta < 0)) 142354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra delta = 0; 1433e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 144def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_clock = scd->clock; 145def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 146354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra /* 147354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra * scd->clock = clamp(scd->tick_gtod + delta, 148b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * max(scd->tick_gtod, scd->clock), 149b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar * scd->tick_gtod + TICK_NSEC); 150354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra */ 1513e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 152354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = scd->tick_gtod + delta; 153def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra min_clock = wrap_max(scd->tick_gtod, old_clock); 154def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC); 1553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 156354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_max(clock, min_clock); 157354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra clock = wrap_min(clock, max_clock); 1583e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 159152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock) 160def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 16156b906126d33904d4d67615d0d5b95dbdf1f27caIngo Molnar 162def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return clock; 1633e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 1643e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 165def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstrastatic u64 sched_clock_remote(struct sched_clock_data *scd) 1663e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 167def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra struct sched_clock_data *my_scd = this_scd(); 168def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 this_clock, remote_clock; 169def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 *ptr, old_val, val; 170def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 171a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#if BITS_PER_LONG != 64 172a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixneragain: 173a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 174a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * Careful here: The local and the remote clock values need to 175a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * be read out atomic as we need to compare the values and 176a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * then update either the local or the remote side. So the 177a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * cmpxchg64 below only protects one readout. 178a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 179a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must reread via sched_clock_local() in the retry case on 180a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * 32bit as an NMI could use sched_clock_local() via the 181a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * tracer and hit between the readout of 182a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 183a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 184a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner this_clock = sched_clock_local(my_scd); 185a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 186a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * We must enforce atomic readout on 32bit, otherwise the 187a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update on the remote cpu can hit inbetween the readout of 188a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * the low32bit and the high 32bit portion. 189a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 190a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner remote_clock = cmpxchg64(&scd->clock, 0, 0); 191a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#else 192a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner /* 193a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * On 64bit the read of [my]scd->clock is atomic versus the 194a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner * update, so we can avoid the above 32bit dance. 195a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner */ 196def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(my_scd); 197def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstraagain: 198def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra this_clock = my_scd->clock; 199def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra remote_clock = scd->clock; 200a1cbcaa9ea87b87a96b9fc465951dcf36e459ca2Thomas Gleixner#endif 201def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 202def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 203def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Use the opportunity that we have both locks 204def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * taken to couple the two clocks: we take the 205def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * larger time as the latest time for both 206def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * runqueues. (this creates monotonic movement) 207def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 208def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (likely((s64)(remote_clock - this_clock) < 0)) { 209def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &scd->clock; 210def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = remote_clock; 211def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = this_clock; 2123e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } else { 213def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra /* 214def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra * Should be rare, but possible: 215def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra */ 216def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra ptr = &my_scd->clock; 217def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra old_val = this_clock; 218def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra val = remote_clock; 2193e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra } 220def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 221152f9d0710a62708710161bce1b29fa8292c8c11Eric Dumazet if (cmpxchg64(ptr, old_val, val) != old_val) 222def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra goto again; 223def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 224def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra return val; 2253e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2263e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 227c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 228c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock(), but requires local IRQs to be disabled. 229c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 230c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 231c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 2323e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstrau64 sched_clock_cpu(int cpu) 2333e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 234b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar struct sched_clock_data *scd; 235def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra u64 clock; 236def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra 237b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar if (sched_clock_stable) 238b342501cd31e5546d0c9ca8ceff5ded1832f9e5bIngo Molnar return sched_clock(); 239a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 240a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 241a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return 0ull; 242a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 243ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra preempt_disable(); 244def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra scd = cpu_sdc(cpu); 2453e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 246def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra if (cpu != smp_processor_id()) 247def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_remote(scd); 248def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra else 249def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra clock = sched_clock_local(scd); 250ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra preempt_enable(); 251e4e4e534faa3c2be4e165ce414f44b76ada7208cIngo Molnar 2523e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra return clock; 2533e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2543e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2553e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_tick(void) 2563e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 2578325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra struct sched_clock_data *scd; 2583e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra u64 now, now_gtod; 2593e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2608325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra if (sched_clock_stable) 2618325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return; 2628325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 263a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra if (unlikely(!sched_clock_running)) 264a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra return; 265a381759d6ad5c5dea5a981918e0b4493e9b66ac7Peter Zijlstra 2663e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra WARN_ON_ONCE(!irqs_disabled()); 2673e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2688325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra scd = this_scd(); 2693e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra now_gtod = ktime_to_ns(ktime_get()); 270a83bc47c33ab182f1e48977fd5a04024d713c75eSteven Rostedt now = sched_clock(); 2713e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2723e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_raw = now; 2733e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra scd->tick_gtod = now_gtod; 274def0a9b2573e00ab0b486cb5382625203ab4c4a6Peter Zijlstra sched_clock_local(scd); 2753e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2763e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2773e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 2783e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We are going deep-idle (irqs are disabled): 2793e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 2803e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_sleep_event(void) 2813e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 2823e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra sched_clock_cpu(smp_processor_id()); 2833e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2843e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); 2853e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 2863e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra/* 2873e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra * We just idled delta nanoseconds (called with irqs disabled): 2883e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra */ 2893e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstravoid sched_clock_idle_wakeup_event(u64 delta_ns) 2903e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra{ 2911c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner if (timekeeping_suspended) 2921c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner return; 2931c5745aa380efb6417b5681104b007c8612fb496Thomas Gleixner 294354879bb977e06695993435745f06a0f6d39ce2bPeter Zijlstra sched_clock_tick(); 2953e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra touch_softlockup_watchdog(); 2963e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra} 2973e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter ZijlstraEXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 2983e51f33fcc7f55e6df25d15b55ed10c8b4da84cdPeter Zijlstra 299c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 300c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * As outlined at the top, provides a fast, high resolution, nanosecond 301c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * time source that is monotonic per cpu argument and has bounded drift 302c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * between cpus. 303c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 304c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * ######################### BIG FAT WARNING ########################## 305c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 306c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * # go backwards !! # 307c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * #################################################################### 308c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 309c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 310b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller{ 311ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra return sched_clock_cpu(cpu); 312b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 313b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller 314c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra/* 315c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * Similar to cpu_clock() for the current cpu. Time will only be observed 316c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * to be monotonic if care is taken to only compare timestampt taken on the 317c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * same CPU. 318c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * 319c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra * See cpu_clock(). 320c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra */ 321c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 322c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 323ef08f0fff87630d4f67ceb09514d8b444df833f8Peter Zijlstra return sched_clock_cpu(raw_smp_processor_id()); 324c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 325c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 3268325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 3278325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3288325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstravoid sched_clock_init(void) 3298325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3308325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra sched_clock_running = 1; 3318325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 3328325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3338325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstrau64 sched_clock_cpu(int cpu) 3348325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra{ 3358325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra if (unlikely(!sched_clock_running)) 3368325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return 0; 3378325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 3388325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra return sched_clock(); 3398325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra} 3408325d9c09dedf45476f4d6261d1b6a72e4a7453fPeter Zijlstra 341c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 cpu_clock(int cpu) 34276a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra{ 343b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller return sched_clock_cpu(cpu); 344b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller} 34576a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 346c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstrau64 local_clock(void) 347c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra{ 348c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra return sched_clock_cpu(0); 349c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra} 350c676329abb2b8359d9a5d734dec0c81779823fd6Peter Zijlstra 351b9f8fcd55bbdb037e5332dbdb7b494f0b70861acDavid Miller#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 35276a2a6ee8a0660a29127f05989ac59ae1ce865faPeter Zijlstra 3534c9fe8ad813b257a2b9ddf0f752105a75a7dae63Ingo MolnarEXPORT_SYMBOL_GPL(cpu_clock); 354c676329abb2b8359d9a5d734dec0c81779823fd6Peter ZijlstraEXPORT_SYMBOL_GPL(local_clock); 355