1/* Copyright (c) 2006, Google Inc.
2 * All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 *     * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 *     * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 *     * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * ---
31 * Author: Sanjay Ghemawat
32 */
33
34#include <config.h>
35#include "base/spinlock.h"
36#include "base/synchronization_profiling.h"
37#include "base/spinlock_internal.h"
38#include "base/cycleclock.h"
39#include "base/sysinfo.h"   /* for NumCPUs() */
40
41// NOTE on the Lock-state values:
42//
43//   kSpinLockFree represents the unlocked state
44//   kSpinLockHeld represents the locked state with no waiters
45//
46// Values greater than kSpinLockHeld represent the locked state with waiters,
47// where the value is the time the current lock holder had to
48// wait before obtaining the lock.  The kSpinLockSleeper state is a special
49// "locked with waiters" state that indicates that a sleeper needs to
50// be woken, but the thread that just released the lock didn't wait.
51
52static int adaptive_spin_count = 0;
53
54const base::LinkerInitialized SpinLock::LINKER_INITIALIZED =
55    base::LINKER_INITIALIZED;
56
57namespace {
58struct SpinLock_InitHelper {
59  SpinLock_InitHelper() {
60    // On multi-cpu machines, spin for longer before yielding
61    // the processor or sleeping.  Reduces idle time significantly.
62    if (NumCPUs() > 1) {
63      adaptive_spin_count = 1000;
64    }
65  }
66};
67
68// Hook into global constructor execution:
69// We do not do adaptive spinning before that,
70// but nothing lock-intensive should be going on at that time.
71static SpinLock_InitHelper init_helper;
72
73}  // unnamed namespace
74
75// Monitor the lock to see if its value changes within some time period
76// (adaptive_spin_count loop iterations).  A timestamp indicating
77// when the thread initially started waiting for the lock is passed in via
78// the initial_wait_timestamp value.  The total wait time in cycles for the
79// lock is returned in the wait_cycles parameter.  The last value read
80// from the lock is returned from the method.
81Atomic32 SpinLock::SpinLoop(int64 initial_wait_timestamp,
82                            Atomic32* wait_cycles) {
83  int c = adaptive_spin_count;
84  while (base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree && --c > 0) {
85  }
86  Atomic32 spin_loop_wait_cycles = CalculateWaitCycles(initial_wait_timestamp);
87  Atomic32 lock_value =
88      base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree,
89                                           spin_loop_wait_cycles);
90  *wait_cycles = spin_loop_wait_cycles;
91  return lock_value;
92}
93
94void SpinLock::SlowLock() {
95  // The lock was not obtained initially, so this thread needs to wait for
96  // it.  Record the current timestamp in the local variable wait_start_time
97  // so the total wait time can be stored in the lockword once this thread
98  // obtains the lock.
99  int64 wait_start_time = CycleClock::Now();
100  Atomic32 wait_cycles;
101  Atomic32 lock_value = SpinLoop(wait_start_time, &wait_cycles);
102
103  int lock_wait_call_count = 0;
104  while (lock_value != kSpinLockFree) {
105    // If the lock is currently held, but not marked as having a sleeper, mark
106    // it as having a sleeper.
107    if (lock_value == kSpinLockHeld) {
108      // Here, just "mark" that the thread is going to sleep.  Don't store the
109      // lock wait time in the lock as that will cause the current lock
110      // owner to think it experienced contention.
111      lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_,
112                                                        kSpinLockHeld,
113                                                        kSpinLockSleeper);
114      if (lock_value == kSpinLockHeld) {
115        // Successfully transitioned to kSpinLockSleeper.  Pass
116        // kSpinLockSleeper to the SpinLockWait routine to properly indicate
117        // the last lock_value observed.
118        lock_value = kSpinLockSleeper;
119      } else if (lock_value == kSpinLockFree) {
120        // Lock is free again, so try and aquire it before sleeping.  The
121        // new lock state will be the number of cycles this thread waited if
122        // this thread obtains the lock.
123        lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_,
124                                                          kSpinLockFree,
125                                                          wait_cycles);
126        continue;  // skip the delay at the end of the loop
127      }
128    }
129
130    // Wait for an OS specific delay.
131    base::internal::SpinLockDelay(&lockword_, lock_value,
132                                  ++lock_wait_call_count);
133    // Spin again after returning from the wait routine to give this thread
134    // some chance of obtaining the lock.
135    lock_value = SpinLoop(wait_start_time, &wait_cycles);
136  }
137}
138
139// The wait time for contentionz lock profiling must fit into 32 bits.
140// However, the lower 32-bits of the cycle counter wrap around too quickly
141// with high frequency processors, so a right-shift by 7 is performed to
142// quickly divide the cycles by 128.  Using these 32 bits, reduces the
143// granularity of time measurement to 128 cycles, and loses track
144// of wait time for waits greater than 109 seconds on a 5 GHz machine
145// [(2^32 cycles/5 Ghz)*128 = 109.95 seconds]. Waits this long should be
146// very rare and the reduced granularity should not be an issue given
147// processors in the Google fleet operate at a minimum of one billion
148// cycles/sec.
149enum { PROFILE_TIMESTAMP_SHIFT = 7 };
150
151void SpinLock::SlowUnlock(uint64 wait_cycles) {
152  base::internal::SpinLockWake(&lockword_, false);  // wake waiter if necessary
153
154  // Collect contentionz profile info, expanding the wait_cycles back out to
155  // the full value.  If wait_cycles is <= kSpinLockSleeper, then no wait
156  // was actually performed, so don't record the wait time.  Note, that the
157  // CalculateWaitCycles method adds in kSpinLockSleeper cycles
158  // unconditionally to guarantee the wait time is not kSpinLockFree or
159  // kSpinLockHeld.  The adding in of these small number of cycles may
160  // overestimate the contention by a slight amount 50% of the time.  However,
161  // if this code tried to correct for that addition by subtracting out the
162  // kSpinLockSleeper amount that would underestimate the contention slightly
163  // 50% of the time.  Both ways get the wrong answer, so the code
164  // overestimates to be more conservative. Overestimating also makes the code
165  // a little simpler.
166  //
167  if (wait_cycles > kSpinLockSleeper) {
168    base::SubmitSpinLockProfileData(this,
169                                    wait_cycles << PROFILE_TIMESTAMP_SHIFT);
170  }
171}
172
173inline int32 SpinLock::CalculateWaitCycles(int64 wait_start_time) {
174  int32 wait_cycles = ((CycleClock::Now() - wait_start_time) >>
175                       PROFILE_TIMESTAMP_SHIFT);
176  // The number of cycles waiting for the lock is used as both the
177  // wait_cycles and lock value, so it can't be kSpinLockFree or
178  // kSpinLockHeld.  Make sure the value returned is at least
179  // kSpinLockSleeper.
180  wait_cycles |= kSpinLockSleeper;
181  return wait_cycles;
182}
183