11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifndef __LINUX_SEQLOCK_H 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define __LINUX_SEQLOCK_H 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Reader/writer consistent mechanism without starving writers. This type of 5d08df601a30df9e36c29f3214315f4f0c8784c68Robert P. J. Day * lock for data where the reader wants a consistent set of information 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and is willing to retry if the information changes. Readers never 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * block but they may have to retry if a writer is in 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * progress. Writers do not wait for readers. 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is not as cache friendly as brlock. Also, this will not work 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * for data that contains pointers, because any writer could 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * invalidate a pointer that a reader was following. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Expected reader usage: 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * do { 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * seq = read_seqbegin(&foo); 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ... 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * } while (read_seqretry(&foo, seq)); 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * On non-SMP the spin locks disappear but the writer still needs 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * to increment the sequence variables because an interrupt routine could 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * change the state of the data. 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Based on x86_64 vsyscall gettimeofday 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * by Keith Owens and Andrea Arcangeli 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/spinlock.h> 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/preempt.h> 3156a210526adb2854d5b7c398a40260720390ee05David Howells#include <asm/processor.h> 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldstypedef struct { 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned sequence; 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spinlock_t lock; 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} seqlock_t; 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * These macros triggered gcc-3.x compile-time problems. We think these are 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * OK now. Be cautious. 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 42e4d919188554a77c798a267e098059bc9aa39726Ingo Molnar#define __SEQLOCK_UNLOCKED(lockname) \ 43e4d919188554a77c798a267e098059bc9aa39726Ingo Molnar { 0, __SPIN_LOCK_UNLOCKED(lockname) } 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4599a3eb3845f034eb55640a3da73e5e28349678c6Ingo Molnar#define seqlock_init(x) \ 4699a3eb3845f034eb55640a3da73e5e28349678c6Ingo Molnar do { \ 4799a3eb3845f034eb55640a3da73e5e28349678c6Ingo Molnar (x)->sequence = 0; \ 4899a3eb3845f034eb55640a3da73e5e28349678c6Ingo Molnar spin_lock_init(&(x)->lock); \ 4999a3eb3845f034eb55640a3da73e5e28349678c6Ingo Molnar } while (0) 50e4d919188554a77c798a267e098059bc9aa39726Ingo Molnar 51e4d919188554a77c798a267e098059bc9aa39726Ingo Molnar#define DEFINE_SEQLOCK(x) \ 52e4d919188554a77c798a267e098059bc9aa39726Ingo Molnar seqlock_t x = __SEQLOCK_UNLOCKED(x) 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Lock out other writers and update the count. 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Acts like a normal spin_lock/unlock. 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Don't need preempt_disable() because that is in the spin_lock already. 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void write_seqlock(seqlock_t *sl) 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock(&sl->lock); 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++sl->sequence; 6220f09390b2da2432309afe8aaa0bd64ec64c4584Daniel Walker smp_wmb(); 6320f09390b2da2432309afe8aaa0bd64ec64c4584Daniel Walker} 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6520f09390b2da2432309afe8aaa0bd64ec64c4584Daniel Walkerstatic inline void write_sequnlock(seqlock_t *sl) 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_wmb(); 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sl->sequence++; 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock(&sl->lock); 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline int write_tryseqlock(seqlock_t *sl) 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = spin_trylock(&sl->lock); 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret) { 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++sl->sequence; 7820f09390b2da2432309afe8aaa0bd64ec64c4584Daniel Walker smp_wmb(); 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Start of read calculation -- fetch last complete writer token */ 84cde227afe6b997dce08bcfc2aa6e373fb56857b0mao, bibostatic __always_inline unsigned read_seqbegin(const seqlock_t *sl) 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 8688a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar unsigned ret; 8788a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar 8888a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnarrepeat: 895db1256a5131d3b133946fa02ac9770a784e6eb2Milton Miller ret = ACCESS_ONCE(sl->sequence); 9088a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar if (unlikely(ret & 1)) { 9188a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar cpu_relax(); 9288a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar goto repeat; 9388a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar } 945db1256a5131d3b133946fa02ac9770a784e6eb2Milton Miller smp_rmb(); 9588a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9988a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar/* 10088a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar * Test if reader processed invalid data. 10188a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar * 10288a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar * If sequence value changed then writer changed data while in section. 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 10488a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnarstatic __always_inline int read_seqretry(const seqlock_t *sl, unsigned start) 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_rmb(); 10788a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar 1083c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin return unlikely(sl->sequence != start); 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Version using sequence counter only. 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This can be used when code has its own mutex protecting the 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updating starting before the write_seqcountbeqin() and ending 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after the write_seqcount_end(). 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldstypedef struct seqcount { 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned sequence; 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} seqcount_t; 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define SEQCNT_ZERO { 0 } 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0) 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1263c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin/** 1273c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * __read_seqcount_begin - begin a seq-read critical section (without barrier) 1283c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @s: pointer to seqcount_t 1293c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Returns: count to be passed to read_seqcount_retry 1303c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 1313c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() 1323c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * barrier. Callers should ensure that smp_rmb() or equivalent ordering is 1333c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * provided before actually loading any of the variables that are to be 1343c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * protected in this critical section. 1353c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 1363c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Use carefully, only in critical code, and comment how the barrier is 1373c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * provided. 1383c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin */ 1393c22cd5709e8143444a6d08682a87f4c57902df3Nick Pigginstatic inline unsigned __read_seqcount_begin(const seqcount_t *s) 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 14188a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar unsigned ret; 14288a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar 14388a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnarrepeat: 1442f624278626677bfaf73fef97f86b37981621f5cLinus Torvalds ret = ACCESS_ONCE(s->sequence); 14588a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar if (unlikely(ret & 1)) { 14688a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar cpu_relax(); 14788a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar goto repeat; 14888a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar } 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1523c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin/** 1533c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * read_seqcount_begin - begin a seq-read critical section 1543c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @s: pointer to seqcount_t 1553c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Returns: count to be passed to read_seqcount_retry 1563c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 1573c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * read_seqcount_begin opens a read critical section of the given seqcount. 1583c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Validity of the critical section is tested by checking read_seqcount_retry 1593c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * function. 1603c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin */ 1613c22cd5709e8143444a6d08682a87f4c57902df3Nick Pigginstatic inline unsigned read_seqcount_begin(const seqcount_t *s) 1623c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin{ 1633c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin unsigned ret = __read_seqcount_begin(s); 1643c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin smp_rmb(); 1653c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin return ret; 1663c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin} 1673c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin 1683c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin/** 1694f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * raw_seqcount_begin - begin a seq-read critical section 1704f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * @s: pointer to seqcount_t 1714f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * Returns: count to be passed to read_seqcount_retry 1724f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * 1734f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * raw_seqcount_begin opens a read critical section of the given seqcount. 1744f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * Validity of the critical section is tested by checking read_seqcount_retry 1754f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * function. 1764f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * 1774f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * Unlike read_seqcount_begin(), this function will not wait for the count 1784f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * to stabilize. If a writer is active when we begin, we will fail the 1794f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * read_seqcount_retry() instead of stabilizing at the beginning of the 1804f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds * critical section. 1814f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds */ 1824f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvaldsstatic inline unsigned raw_seqcount_begin(const seqcount_t *s) 1834f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds{ 1844f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds unsigned ret = ACCESS_ONCE(s->sequence); 1854f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds smp_rmb(); 1864f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds return ret & ~1; 1874f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds} 1884f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds 1894f988f152ee087831ea5c1c77cda4454cacc052cLinus Torvalds/** 1903c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * __read_seqcount_retry - end a seq-read critical section (without barrier) 1913c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @s: pointer to seqcount_t 1923c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @start: count, from read_seqcount_begin 1933c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Returns: 1 if retry is required, else 0 1943c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 1953c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() 1963c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * barrier. Callers should ensure that smp_rmb() or equivalent ordering is 1973c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * provided before actually loading any of the variables that are to be 1983c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * protected in this critical section. 1993c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 2003c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Use carefully, only in critical code, and comment how the barrier is 2013c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * provided. 2023c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin */ 2033c22cd5709e8143444a6d08682a87f4c57902df3Nick Pigginstatic inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) 2043c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin{ 2053c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin return unlikely(s->sequence != start); 2063c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin} 2073c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin 2083c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin/** 2093c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * read_seqcount_retry - end a seq-read critical section 2103c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @s: pointer to seqcount_t 2113c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @start: count, from read_seqcount_begin 2123c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * Returns: 1 if retry is required, else 0 2133c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 2143c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * read_seqcount_retry closes a read critical section of the given seqcount. 2153c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * If the critical section was invalid, it must be ignored (and typically 2163c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * retried). 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 21888a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnarstatic inline int read_seqcount_retry(const seqcount_t *s, unsigned start) 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_rmb(); 22188a411c07b6fedcfc97b8dc51ae18540bd2beda0Ingo Molnar 2223c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin return __read_seqcount_retry(s, start); 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Sequence counter only version assumes that callers are using their 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * own mutexing. 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void write_seqcount_begin(seqcount_t *s) 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds s->sequence++; 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_wmb(); 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void write_seqcount_end(seqcount_t *s) 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_wmb(); 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds s->sequence++; 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2423c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin/** 2433c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * write_seqcount_barrier - invalidate in-progress read-side seq operations 2443c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * @s: pointer to seqcount_t 2453c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * 2463c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * After write_seqcount_barrier, no read-side seq operations will complete 2473c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin * successfully and see data older than this. 2483c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin */ 2493c22cd5709e8143444a6d08682a87f4c57902df3Nick Pigginstatic inline void write_seqcount_barrier(seqcount_t *s) 2503c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin{ 2513c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin smp_wmb(); 2523c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin s->sequence+=2; 2533c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin} 2543c22cd5709e8143444a6d08682a87f4c57902df3Nick Piggin 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Possible sw/hw IRQ protected versions of the interfaces. 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_seqlock_irqsave(lock, flags) \ 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { local_irq_save(flags); write_seqlock(lock); } while (0) 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_seqlock_irq(lock) \ 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { local_irq_disable(); write_seqlock(lock); } while (0) 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_seqlock_bh(lock) \ 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { local_bh_disable(); write_seqlock(lock); } while (0) 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_sequnlock_irqrestore(lock, flags) \ 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { write_sequnlock(lock); local_irq_restore(flags); } while(0) 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_sequnlock_irq(lock) \ 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { write_sequnlock(lock); local_irq_enable(); } while(0) 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define write_sequnlock_bh(lock) \ 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { write_sequnlock(lock); local_bh_enable(); } while(0) 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define read_seqbegin_irqsave(lock, flags) \ 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ({ local_irq_save(flags); read_seqbegin(lock); }) 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define read_seqretry_irqrestore(lock, iv, flags) \ 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ({ \ 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = read_seqretry(lock, iv); \ 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_irq_restore(flags); \ 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret; \ 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds }) 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif /* __LINUX_SEQLOCK_H */ 283