164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Read-Copy Update mechanism for mutual exclusion 364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * This program is free software; you can redistribute it and/or modify 564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * it under the terms of the GNU General Public License as published by 664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * the Free Software Foundation; either version 2 of the License, or 764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * (at your option) any later version. 864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * This program is distributed in the hope that it will be useful, 1064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * but WITHOUT ANY WARRANTY; without even the implied warranty of 1164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * GNU General Public License for more details. 1364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 1464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * You should have received a copy of the GNU General Public License 1587de1cfdc55b16b794e245b07322340725149d62Paul E. McKenney * along with this program; if not, you can access it online at 1687de1cfdc55b16b794e245b07322340725149d62Paul E. McKenney * http://www.gnu.org/licenses/gpl-2.0.html. 1764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 1864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Copyright IBM Corporation, 2008 1964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 2064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Authors: Dipankar Sarma <dipankar@in.ibm.com> 2164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Manfred Spraul <manfred@colorfullife.com> 2264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version 2364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 2464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Based on the original work by Paul McKenney <paulmck@us.ibm.com> 2564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. 2664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 2764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * For detailed explanation of Read-Copy Update mechanism see - 28a71fca58b7f4abca551ae2256ac08dd9123a03f9Paul E. McKenney * Documentation/RCU 2964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 3064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/types.h> 3164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/kernel.h> 3264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/init.h> 3364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/spinlock.h> 3464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/smp.h> 3564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/rcupdate.h> 3664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/interrupt.h> 3764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/sched.h> 38c1dc0b9c0c8979ce4d411caadff5c0d79dee58bcIngo Molnar#include <linux/nmi.h> 398826f3b0397562eee6f8785d548be9dfdb169100Paul E. McKenney#include <linux/atomic.h> 4064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/bitops.h> 419984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 4264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/completion.h> 4364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/moduleparam.h> 444102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#include <linux/module.h> 4564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/percpu.h> 4664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/notifier.h> 4764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/cpu.h> 4864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/mutex.h> 4964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#include <linux/time.h> 50bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney#include <linux/kernel_stat.h> 51a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney#include <linux/wait.h> 52a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney#include <linux/kthread.h> 53268bb0ce3e87872cb9290c322b0d35bce230d88fLinus Torvalds#include <linux/prefetch.h> 543d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney#include <linux/delay.h> 553d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney#include <linux/stop_machine.h> 56661a85dc0d2ec0404e3b80909e413a9d5e42a239Paul E. McKenney#include <linux/random.h> 57f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat)#include <linux/ftrace_event.h> 58d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov#include <linux/suspend.h> 5964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 604102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#include "tree.h" 6129c00b4a1d9e277786120032aa8364631820d863Paul E. McKenney#include "rcu.h" 629f77da9f40045253e91f55c12d4481254b513d2dPaul E. McKenney 634102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenneyMODULE_ALIAS("rcutree"); 644102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#ifdef MODULE_PARAM_PREFIX 654102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#undef MODULE_PARAM_PREFIX 664102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#endif 674102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#define MODULE_PARAM_PREFIX "rcutree." 684102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney 6964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* Data structures. */ 7064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 71f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneystatic struct lock_class_key rcu_node_class[RCU_NUM_LVLS]; 72394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenneystatic struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; 7388b91c7ca49bc8600cf1106eb891d08c1965b9cePeter Zijlstra 74f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat)/* 75f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * In order to export the rcu_state name to the tracing tools, it 76f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * needs to be added in the __tracepoint_string section. 77f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * This requires defining a separate variable tp_<sname>_varname 78f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * that points to the string being used, and this will allow 79f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * the tracing userspace tools to be able to decipher the string 80f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) * address to the matching string. 81f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) */ 82a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel#ifdef CONFIG_TRACING 83a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel# define DEFINE_RCU_TPS(sname) \ 84f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat)static char sname##_varname[] = #sname; \ 85a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvelstatic const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname; 86a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel# define RCU_STATE_NAME(sname) sname##_varname 87a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel#else 88a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel# define DEFINE_RCU_TPS(sname) 89a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel# define RCU_STATE_NAME(sname) __stringify(sname) 90a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel#endif 91a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel 92a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \ 93a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard BiesheuvelDEFINE_RCU_TPS(sname) \ 94a41bfeb2f8ed59410be7ca0f8fbc6138a758b746Steven Rostedt (Red Hat)struct rcu_state sname##_state = { \ 956c90cc7bf077f28144013e949ee0c122012d194aPaul E. McKenney .level = { &sname##_state.node[0] }, \ 96037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenney .call = cr, \ 97af446b702c58b700cc5fa99f6edc78b99e55b995Paul E. McKenney .fqs_state = RCU_GP_IDLE, \ 9842c3533eee88e012e1aa3c4d6d2cc53354130e24Paul E. McKenney .gpnum = 0UL - 300UL, \ 9942c3533eee88e012e1aa3c4d6d2cc53354130e24Paul E. McKenney .completed = 0UL - 300UL, \ 1007b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \ 1016c90cc7bf077f28144013e949ee0c122012d194aPaul E. McKenney .orphan_nxttail = &sname##_state.orphan_nxtlist, \ 1026c90cc7bf077f28144013e949ee0c122012d194aPaul E. McKenney .orphan_donetail = &sname##_state.orphan_donelist, \ 1037be7f0be907224445acc62b3884c892f38b7ff40Paul E. McKenney .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ 104a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \ 105a8a29b3b7b18251c4e3ffce501f25ae868302a75Ard Biesheuvel .name = RCU_STATE_NAME(sname), \ 106a488985851cf2facd2227bd982cc2c251df56268Paul E. McKenney .abbr = sabbr, \ 107a41bfeb2f8ed59410be7ca0f8fbc6138a758b746Steven Rostedt (Red Hat)}; \ 108a41bfeb2f8ed59410be7ca0f8fbc6138a758b746Steven Rostedt (Red Hat)DEFINE_PER_CPU(struct rcu_data, sname##_data) 10964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 110a41bfeb2f8ed59410be7ca0f8fbc6138a758b746Steven Rostedt (Red Hat)RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); 111a41bfeb2f8ed59410be7ca0f8fbc6138a758b746Steven Rostedt (Red Hat)RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); 112b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar 113e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharmastatic struct rcu_state *rcu_state_p; 1146ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenneyLIST_HEAD(rcu_struct_flavors); 11527f4d28057adf98750cf863c40baefb12f5b6d21Paul E. McKenney 116f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney/* Increase (but not decrease) the CONFIG_RCU_FANOUT_LEAF at boot time. */ 117f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneystatic int rcu_fanout_leaf = CONFIG_RCU_FANOUT_LEAF; 1187e5c2dfb4de15e21f62c956ec32cda9372ca993bPaul E. McKenneymodule_param(rcu_fanout_leaf, int, 0444); 119f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneyint rcu_num_lvls __read_mostly = RCU_NUM_LVLS; 120f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneystatic int num_rcu_lvl[] = { /* Number of rcu_nodes at specified level. */ 121f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney NUM_RCU_LVL_0, 122f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney NUM_RCU_LVL_1, 123f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney NUM_RCU_LVL_2, 124f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney NUM_RCU_LVL_3, 125f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney NUM_RCU_LVL_4, 126f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney}; 127f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneyint rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ 128f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 129b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney/* 130b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * The rcu_scheduler_active variable transitions from zero to one just 131b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * before the first task is spawned. So when this variable is zero, RCU 132b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * can assume that there is but one task, allowing RCU to (for example) 133b44f665623d15ff391c7e9fe7600c5e7bc2fc9c2Cody P Schafer * optimize synchronize_sched() to a simple barrier(). When this variable 134b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * is one, RCU must actually do all the hard work required to detect real 135b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * grace periods. This variable is also used to suppress boot-time false 136b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * positives from lockdep-RCU error checking. 137b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney */ 138bbad937983147c017c25406860287cb94da9af7cPaul E. McKenneyint rcu_scheduler_active __read_mostly; 139bbad937983147c017c25406860287cb94da9af7cPaul E. McKenneyEXPORT_SYMBOL_GPL(rcu_scheduler_active); 140bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney 141b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney/* 142b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * The rcu_scheduler_fully_active variable transitions from zero to one 143b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * during the early_initcall() processing, which is after the scheduler 144b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * is capable of creating new tasks. So RCU processing (for example, 145b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * creating tasks for RCU priority boosting) must be delayed until after 146b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * rcu_scheduler_fully_active transitions from zero to one. We also 147b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * currently delay invocation of any RCU callbacks until after this point. 148b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * 149b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * It might later prove better for people registering RCU callbacks during 150b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * early boot to take responsibility for these callbacks, but one step at 151b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney * a time. 152b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney */ 153b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenneystatic int rcu_scheduler_fully_active __read_mostly; 154b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney 155a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney#ifdef CONFIG_RCU_BOOST 156a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney 157b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar/* 158a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney * Control variables for per-CPU and per-rcu_node kthreads. These 159a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney * handle all flavors of RCU. 160a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney */ 161a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenneystatic DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); 162d71df90eadfc35aa549ff9a850842673febca71fPaul E. McKenneyDEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); 1635ece5bab3ed8594ce2c85c6c6e6b82109db36ca7Paul E. McKenneyDEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); 164d71df90eadfc35aa549ff9a850842673febca71fPaul E. McKenneyDEFINE_PER_CPU(char, rcu_cpu_has_work); 165a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney 166a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney#endif /* #ifdef CONFIG_RCU_BOOST */ 167a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney 1685d01bbd111d6ff9ea9d9847774f66dff39633776Thomas Gleixnerstatic void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); 169a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenneystatic void invoke_rcu_core(void); 170a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenneystatic void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); 171a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney 172a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney/* 1734a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * Track the rcutorture test sequence number and the update version 1744a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * number within a given test. The rcutorture_testseq is incremented 1754a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * on every rcutorture module load and unload, so has an odd value 1764a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * when a test is running. The rcutorture_vernum is set to zero 1774a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * when rcutorture starts and is incremented on each rcutorture update. 1784a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * These variables enable correlating rcutorture output with the 1794a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * RCU tracing information. 1804a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney */ 1814a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyunsigned long rcutorture_testseq; 1824a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyunsigned long rcutorture_vernum; 1834a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney 1844a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney/* 185fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s 186fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney * permit this function to be invoked without holding the root rcu_node 187fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney * structure's ->lock, but of course results can be subject to change. 188fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney */ 189fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenneystatic int rcu_gp_in_progress(struct rcu_state *rsp) 190fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney{ 191fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); 192fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney} 193fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney 194fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney/* 195d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * Note a quiescent state. Because we do not need to know 196b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar * how many quiescent states passed, just if there was at least 197d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * one since the start of the grace period, this just sets a flag. 198e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * The caller must have disabled preemption. 199b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar */ 200284a8c93af47306beed967a303d84730b32bab39Paul E. McKenneyvoid rcu_sched_qs(void) 201b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar{ 202284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) { 203284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney trace_rcu_grace_period(TPS("rcu_sched"), 204284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney __this_cpu_read(rcu_sched_data.gpnum), 205284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney TPS("cpuqs")); 206284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney __this_cpu_write(rcu_sched_data.passed_quiesce, 1); 207284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney } 208b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar} 209b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar 210284a8c93af47306beed967a303d84730b32bab39Paul E. McKenneyvoid rcu_bh_qs(void) 211b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar{ 212284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { 213284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney trace_rcu_grace_period(TPS("rcu_bh"), 214284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney __this_cpu_read(rcu_bh_data.gpnum), 215284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney TPS("cpuqs")); 216284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney __this_cpu_write(rcu_bh_data.passed_quiesce, 1); 217284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney } 218b1f77b0581b8fd837acb4a973f7d5496cae6efeeIngo Molnar} 21964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2204a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenneystatic DEFINE_PER_CPU(int, rcu_sched_qs_mask); 2214a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2224a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenneystatic DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { 2234a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE, 2244a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney .dynticks = ATOMIC_INIT(1), 2254a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney#ifdef CONFIG_NO_HZ_FULL_SYSIDLE 2264a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE, 2274a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney .dynticks_idle = ATOMIC_INIT(1), 2284a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ 2294a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney}; 2304a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2314a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney/* 2324a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * Let the RCU core know that this CPU has gone through the scheduler, 2334a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * which is a quiescent state. This is called when the need for a 2344a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * quiescent state is urgent, so we burn an atomic operation and full 2354a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * memory barriers to let the RCU core know about it, regardless of what 2364a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * this CPU might (or might not) do in the near future. 2374a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * 2384a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * We inform the RCU core by emulating a zero-duration dyntick-idle 2394a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * period, which we in turn do by incrementing the ->dynticks counter 2404a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * by two. 2414a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney */ 2424a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenneystatic void rcu_momentary_dyntick_idle(void) 2434a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney{ 2444a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney unsigned long flags; 2454a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney struct rcu_data *rdp; 2464a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney struct rcu_dynticks *rdtp; 2474a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney int resched_mask; 2484a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney struct rcu_state *rsp; 2494a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2504a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney local_irq_save(flags); 2514a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2524a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney /* 2534a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * Yes, we can lose flag-setting operations. This is OK, because 2544a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * the flag will be set again after some delay. 2554a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney */ 2564a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney resched_mask = raw_cpu_read(rcu_sched_qs_mask); 2574a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney raw_cpu_write(rcu_sched_qs_mask, 0); 2584a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2594a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney /* Find the flavor that needs a quiescent state. */ 2604a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney for_each_rcu_flavor(rsp) { 2614a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rdp = raw_cpu_ptr(rsp->rda); 2624a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney if (!(resched_mask & rsp->flavor_mask)) 2634a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney continue; 2644a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */ 2654a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney if (ACCESS_ONCE(rdp->mynode->completed) != 2664a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney ACCESS_ONCE(rdp->cond_resched_completed)) 2674a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney continue; 2684a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 2694a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney /* 2704a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * Pretend to be momentarily idle for the quiescent state. 2714a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * This allows the grace-period kthread to record the 2724a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * quiescent state, with no need for this CPU to do anything 2734a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * further. 2744a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney */ 2754a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rdtp = this_cpu_ptr(&rcu_dynticks); 2764a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney smp_mb__before_atomic(); /* Earlier stuff before QS. */ 2774a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney atomic_add(2, &rdtp->dynticks); /* QS. */ 2784a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney smp_mb__after_atomic(); /* Later stuff after QS. */ 2794a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney break; 2804a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney } 2814a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney local_irq_restore(flags); 2824a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney} 2834a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 28425502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney/* 28525502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney * Note a context switch. This is a quiescent state for RCU-sched, 28625502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney * and requires special handling for preemptible RCU. 287e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * The caller must have disabled preemption. 28825502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney */ 28925502a6c13745f4650cc59322bd198194f55e796Paul E. McKenneyvoid rcu_note_context_switch(int cpu) 29025502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney{ 291f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("Start context switch")); 292284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney rcu_sched_qs(); 293cba6d0d64ee53772b285d0c0c288deefbeaf7775Paul E. McKenney rcu_preempt_note_context_switch(cpu); 2944a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) 2954a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rcu_momentary_dyntick_idle(); 296f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("End context switch")); 29725502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney} 29829ce831000081dd757d3116bf774aafffc4b6b20Gleb NatapovEXPORT_SYMBOL_GPL(rcu_note_context_switch); 29925502a6c13745f4650cc59322bd198194f55e796Paul E. McKenney 300878d7439d0f45a95869e417576774673d1fa243fEric Dumazetstatic long blimit = 10; /* Maximum callbacks per rcu_do_batch. */ 301878d7439d0f45a95869e417576774673d1fa243fEric Dumazetstatic long qhimark = 10000; /* If this many pending, ignore blimit. */ 302878d7439d0f45a95869e417576774673d1fa243fEric Dumazetstatic long qlowmark = 100; /* Once only this many pending, use blimit. */ 30364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 304878d7439d0f45a95869e417576774673d1fa243fEric Dumazetmodule_param(blimit, long, 0444); 305878d7439d0f45a95869e417576774673d1fa243fEric Dumazetmodule_param(qhimark, long, 0444); 306878d7439d0f45a95869e417576774673d1fa243fEric Dumazetmodule_param(qlowmark, long, 0444); 3073d76c082907e8f83c5d5c4572f38d53ad8f00c4bPaul E. McKenney 308026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenneystatic ulong jiffies_till_first_fqs = ULONG_MAX; 309026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenneystatic ulong jiffies_till_next_fqs = ULONG_MAX; 310d40011f601b450396104de42c631981502946cf0Paul E. McKenney 311d40011f601b450396104de42c631981502946cf0Paul E. McKenneymodule_param(jiffies_till_first_fqs, ulong, 0644); 312d40011f601b450396104de42c631981502946cf0Paul E. McKenneymodule_param(jiffies_till_next_fqs, ulong, 0644); 313d40011f601b450396104de42c631981502946cf0Paul E. McKenney 3144a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney/* 3154a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * How long the grace period must be before we start recruiting 3164a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * quiescent-state help from rcu_note_context_switch(). 3174a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney */ 3184a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenneystatic ulong jiffies_till_sched_qs = HZ / 20; 3194a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenneymodule_param(jiffies_till_sched_qs, ulong, 0644); 3204a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney 32148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, 322910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney struct rcu_data *rdp); 323217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenneystatic void force_qs_rnp(struct rcu_state *rsp, 324217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney int (*f)(struct rcu_data *rsp, bool *isidle, 325217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney unsigned long *maxj), 326217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney bool *isidle, unsigned long *maxj); 3274cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenneystatic void force_quiescent_state(struct rcu_state *rsp); 328a157229cabd6dd8cfa82525fc9bf730c94cc9ac2Paul E. McKenneystatic int rcu_pending(int cpu); 32964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 33064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 331d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * Return the number of RCU-sched batches processed thus far for debug & stats. 33264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 333d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenneylong rcu_batches_completed_sched(void) 33464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 335d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney return rcu_sched_state.completed; 33664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 337d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_batches_completed_sched); 33864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 33964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 34064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Return the number of RCU BH batches processed thus far for debug & stats. 34164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 34264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneylong rcu_batches_completed_bh(void) 34364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 34464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return rcu_bh_state.completed; 34564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 34664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_batches_completed_bh); 34764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 34864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 349a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat * Force a quiescent state. 350a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat */ 351a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernatvoid rcu_force_quiescent_state(void) 352a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat{ 353e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma force_quiescent_state(rcu_state_p); 354a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat} 355a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina BernatEXPORT_SYMBOL_GPL(rcu_force_quiescent_state); 356a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat 357a381d757d93f6e43063f74888676edd6216d0affAndreea-Cristina Bernat/* 358bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney * Force a quiescent state for RCU BH. 359bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney */ 360bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenneyvoid rcu_bh_force_quiescent_state(void) 361bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney{ 3624cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney force_quiescent_state(&rcu_bh_state); 363bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney} 364bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenneyEXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); 365bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney 366bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney/* 367afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney * Show the state of the grace-period kthreads. 368afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney */ 369afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenneyvoid show_rcu_gp_kthreads(void) 370afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney{ 371afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney struct rcu_state *rsp; 372afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney 373afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney for_each_rcu_flavor(rsp) { 374afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney pr_info("%s: wait state: %d ->state: %#lx\n", 375afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney rsp->name, rsp->gp_state, rsp->gp_kthread->state); 376afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney /* sched_show_task(rsp->gp_kthread); */ 377afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney } 378afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney} 379afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenneyEXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); 380afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney 381afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney/* 3824a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * Record the number of times rcutorture tests have been initiated and 3834a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * terminated. This information allows the debugfs tracing stats to be 3844a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * correlated to the rcutorture messages, even when the rcutorture module 3854a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * is being repeatedly loaded and unloaded. In other words, we cannot 3864a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * store this state in rcutorture itself. 3874a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney */ 3884a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyvoid rcutorture_record_test_transition(void) 3894a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney{ 3904a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney rcutorture_testseq++; 3914a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney rcutorture_vernum = 0; 3924a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney} 3934a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyEXPORT_SYMBOL_GPL(rcutorture_record_test_transition); 3944a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney 3954a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney/* 396ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney * Send along grace-period-related data for rcutorture diagnostics. 397ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney */ 398ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenneyvoid rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, 399ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney unsigned long *gpnum, unsigned long *completed) 400ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney{ 401ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney struct rcu_state *rsp = NULL; 402ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney 403ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney switch (test_type) { 404ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney case RCU_FLAVOR: 405e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma rsp = rcu_state_p; 406ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney break; 407ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney case RCU_BH_FLAVOR: 408ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney rsp = &rcu_bh_state; 409ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney break; 410ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney case RCU_SCHED_FLAVOR: 411ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney rsp = &rcu_sched_state; 412ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney break; 413ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney default: 414ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney break; 415ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney } 416ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney if (rsp != NULL) { 417ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *flags = ACCESS_ONCE(rsp->gp_flags); 418ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *gpnum = ACCESS_ONCE(rsp->gpnum); 419ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *completed = ACCESS_ONCE(rsp->completed); 420ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney return; 421ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney } 422ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *flags = 0; 423ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *gpnum = 0; 424ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney *completed = 0; 425ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney} 426ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenneyEXPORT_SYMBOL_GPL(rcutorture_get_gp_data); 427ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney 428ad0dc7f94dbf417b1c7d42e1f0b250f045b27f8fPaul E. McKenney/* 4294a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * Record the number of writer passes through the current rcutorture test. 4304a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * This is also used to correlate debugfs tracing stats with the rcutorture 4314a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney * messages. 4324a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney */ 4334a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyvoid rcutorture_record_progress(unsigned long vernum) 4344a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney{ 4354a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney rcutorture_vernum++; 4364a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney} 4374a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenneyEXPORT_SYMBOL_GPL(rcutorture_record_progress); 4384a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney 4394a29865689dbb87a02e3b0fff4a4ae5041273173Paul E. McKenney/* 440bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney * Force a quiescent state for RCU-sched. 441bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney */ 442bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenneyvoid rcu_sched_force_quiescent_state(void) 443bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney{ 4444cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney force_quiescent_state(&rcu_sched_state); 445bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney} 446bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenneyEXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); 447bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney 448bf66f18e79e34c421bbd8f6511e2c556b779df2fPaul E. McKenney/* 44964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Does the CPU have callbacks ready to be invoked? 45064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 45164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic int 45264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneycpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) 45364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 4543fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] && 4553fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney rdp->nxttail[RCU_DONE_TAIL] != NULL; 45664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 45764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 45864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 459365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney * Return the root node of the specified rcu_state structure. 460365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney */ 461365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenneystatic struct rcu_node *rcu_get_root(struct rcu_state *rsp) 462365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney{ 463365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney return &rsp->node[0]; 464365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney} 465365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney 466365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney/* 467365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney * Is there any need for future grace periods? 468365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney * Interrupts must be disabled. If the caller does not hold the root 469365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney * rnp_node structure's ->lock, the results are advisory only. 470365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney */ 471365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenneystatic int rcu_future_needs_gp(struct rcu_state *rsp) 472365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney{ 473365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 474365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney int idx = (ACCESS_ONCE(rnp->completed) + 1) & 0x1; 475365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney int *fp = &rnp->need_future_gp[idx]; 476365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney 477365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney return ACCESS_ONCE(*fp); 478365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney} 479365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney 480365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney/* 481dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Does the current CPU require a not-yet-started grace period? 482dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * The caller must have disabled interrupts to prevent races with 483dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * normal callback registry. 48464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 48564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic int 48664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneycpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) 48764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 488dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney int i; 4893fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney 490dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rcu_gp_in_progress(rsp)) 491dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return 0; /* No, a grace period is already in progress. */ 492365187fbc04fd55766bf6a94e37e558505bf480aPaul E. McKenney if (rcu_future_needs_gp(rsp)) 49334ed62461ae4970695974afb9a60ac3df0086830Paul E. McKenney return 1; /* Yes, a no-CBs CPU needs one. */ 494dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (!rdp->nxttail[RCU_NEXT_TAIL]) 495dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return 0; /* No, this is a no-CBs (or offline) CPU. */ 496dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (*rdp->nxttail[RCU_NEXT_READY_TAIL]) 497dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return 1; /* Yes, this CPU has newly registered callbacks. */ 498dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) 499dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rdp->nxttail[i - 1] != rdp->nxttail[i] && 500dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney ULONG_CMP_LT(ACCESS_ONCE(rsp->completed), 501dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxtcompleted[i])) 502dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return 1; /* Yes, CBs for future grace period. */ 503dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return 0; /* No grace period needed. */ 50464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 50564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 50664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 507adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state 5089b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 5099b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * If the new value of the ->dynticks_nesting counter now is zero, 5109b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * we really have entered idle, and must do the appropriate accounting. 5119b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * The caller must have disabled interrupts. 5129b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney */ 513adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckerstatic void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, 514adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker bool user) 5159b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney{ 51696d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney struct rcu_state *rsp; 51796d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney struct rcu_data *rdp; 51896d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney 519f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting); 520cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney if (!user && !is_idle_task(current)) { 521289828e62de0334a0d01c0f65df91cd47d3a9e05Paul E. McKenney struct task_struct *idle __maybe_unused = 522289828e62de0334a0d01c0f65df91cd47d3a9e05Paul E. McKenney idle_task(smp_processor_id()); 5230989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney 524f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0); 525bf1304e9cd755be7814ac8365834b7a1d0f06b58Paul E. McKenney ftrace_dump(DUMP_ORIG); 5260989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", 5270989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney current->pid, current->comm, 5280989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney idle->pid, idle->comm); /* must be idle task! */ 5299b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney } 53096d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney for_each_rcu_flavor(rsp) { 53196d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney rdp = this_cpu_ptr(rsp->rda); 53296d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney do_nocb_deferred_wakeup(rdp); 53396d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney } 534aea1b35e29e658d42d7ba2237f3aa7f93e18509dPaul E. McKenney rcu_prepare_for_idle(smp_processor_id()); 5359b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 5364e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* See above. */ 5379b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney atomic_inc(&rdtp->dynticks); 5384e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); /* Force ordering with next sojourn. */ 5399b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 540176f8f7a52cc6d09d686f0d900abda6942a52fbbPaul E. McKenney rcu_dynticks_task_enter(); 541c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney 542c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney /* 543adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * It is illegal to enter an extended quiescent state while 544c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney * in an RCU read-side critical section. 545c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney */ 546c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney rcu_lockdep_assert(!lock_is_held(&rcu_lock_map), 547c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney "Illegal idle entry in RCU read-side critical section."); 548c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), 549c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney "Illegal idle entry in RCU-bh read-side critical section."); 550c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), 551c44e2cddacc2cf299186bad5697d738ea19668b7Paul E. McKenney "Illegal idle entry in RCU-sched read-side critical section."); 5529b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney} 55364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 554adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker/* 555adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Enter an RCU extended quiescent state, which can be either the 556adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * idle loop or adaptive-tickless usermode execution. 55764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 558adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckerstatic void rcu_eqs_enter(bool user) 55964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 5604145fa7fbee3ec1e61c52825b146192885d9759fPaul E. McKenney long long oldval; 56164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_dynticks *rdtp; 56264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 563c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rdtp = this_cpu_ptr(&rcu_dynticks); 5644145fa7fbee3ec1e61c52825b146192885d9759fPaul E. McKenney oldval = rdtp->dynticks_nesting; 56529e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); 5663a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) { 56729e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney rdtp->dynticks_nesting = 0; 5683a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney rcu_eqs_enter_common(rdtp, oldval, user); 5693a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney } else { 57029e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE; 5713a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney } 57264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 573adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker 574adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker/** 575adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_idle_enter - inform RCU that current CPU is entering idle 576adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 577adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Enter idle mode, in other words, -leave- the mode in which RCU 578adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * read-side critical sections can occur. (Though RCU read-side 579adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * critical sections can occur in irq handlers in idle, a possibility 580adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * handled by irq_enter() and irq_exit().) 581adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 582adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * We crowbar the ->dynticks_nesting field to zero to allow for 583adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * the possibility of usermode upcalls having messed up our count 584adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * of interrupt nesting level during the prior busy period. 585adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker */ 586adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckervoid rcu_idle_enter(void) 587adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker{ 588c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker unsigned long flags; 589c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker 590c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker local_irq_save(flags); 591cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney rcu_eqs_enter(false); 592c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rcu_sysidle_enter(this_cpu_ptr(&rcu_dynticks), 0); 593c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker local_irq_restore(flags); 594adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker} 5958a2ecf474d3ee8dd5d001490349e422cec52f39fPaul E. McKenneyEXPORT_SYMBOL_GPL(rcu_idle_enter); 59664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 5972b1d5024e17be459aa6385763ca3faa8f01c52d9Frederic Weisbecker#ifdef CONFIG_RCU_USER_QS 5989b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney/** 599adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_user_enter - inform RCU that we are resuming userspace. 600adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 601adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Enter RCU idle mode right before resuming userspace. No use of RCU 602adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * is permitted between this call and rcu_user_exit(). This way the 603adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * CPU doesn't need to maintain the tick for RCU maintenance purposes 604adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * when the CPU runs in userspace. 605adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker */ 606adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckervoid rcu_user_enter(void) 607adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker{ 60891d1aa43d30505b0b825db8898ffc80a8eca96c7Frederic Weisbecker rcu_eqs_enter(1); 609adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker} 6102b1d5024e17be459aa6385763ca3faa8f01c52d9Frederic Weisbecker#endif /* CONFIG_RCU_USER_QS */ 61119dd1591fc379f1d89f39cd99cbbe97433baa3c3Frederic Weisbecker 61219dd1591fc379f1d89f39cd99cbbe97433baa3c3Frederic Weisbecker/** 6139b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle 6149b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 6159b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * Exit from an interrupt handler, which might possibly result in entering 6169b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * idle mode, in other words, leaving the mode in which read-side critical 6179b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * sections can occur. 61864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 6199b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * This code assumes that the idle loop never does anything that might 6209b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * result in unbalanced calls to irq_enter() and irq_exit(). If your 6219b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * architecture violates this assumption, RCU will give you what you 6229b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * deserve, good and hard. But very infrequently and irreproducibly. 6239b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 6249b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * Use things like work queues to work around this limitation. 6259b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 6269b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * You have been warned. 62764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 6289b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenneyvoid rcu_irq_exit(void) 62964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 63064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 6314145fa7fbee3ec1e61c52825b146192885d9759fPaul E. McKenney long long oldval; 63264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_dynticks *rdtp; 63364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 63464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_save(flags); 635c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rdtp = this_cpu_ptr(&rcu_dynticks); 6364145fa7fbee3ec1e61c52825b146192885d9759fPaul E. McKenney oldval = rdtp->dynticks_nesting; 6379b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney rdtp->dynticks_nesting--; 6389b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney WARN_ON_ONCE(rdtp->dynticks_nesting < 0); 639b6fc6020140db437069d5bec447858fcfd64d31cFrederic Weisbecker if (rdtp->dynticks_nesting) 640f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting); 641b6fc6020140db437069d5bec447858fcfd64d31cFrederic Weisbecker else 642cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney rcu_eqs_enter_common(rdtp, oldval, true); 643eb348b898290da242e46df75ab0b9772003e08b8Paul E. McKenney rcu_sysidle_enter(rdtp, 1); 6449b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney local_irq_restore(flags); 6459b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney} 6469b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 6479b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney/* 648adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_eqs_exit_common - current CPU moving away from extended quiescent state 6499b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 6509b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * If the new value of the ->dynticks_nesting counter was previously zero, 6519b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * we really have exited idle, and must do the appropriate accounting. 6529b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * The caller must have disabled interrupts. 6539b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney */ 654adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckerstatic void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, 655adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker int user) 6569b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney{ 657176f8f7a52cc6d09d686f0d900abda6942a52fbbPaul E. McKenney rcu_dynticks_task_exit(); 6584e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */ 65923b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney atomic_inc(&rdtp->dynticks); 66023b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 6614e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); /* See above. */ 66223b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 6637cb92499000e3c86dae653077b1465458a039ef6Paul E. McKenney rcu_cleanup_after_idle(smp_processor_id()); 664f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); 665cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney if (!user && !is_idle_task(current)) { 666289828e62de0334a0d01c0f65df91cd47d3a9e05Paul E. McKenney struct task_struct *idle __maybe_unused = 667289828e62de0334a0d01c0f65df91cd47d3a9e05Paul E. McKenney idle_task(smp_processor_id()); 6680989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney 669f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("Error on exit: not idle task"), 6704145fa7fbee3ec1e61c52825b146192885d9759fPaul E. McKenney oldval, rdtp->dynticks_nesting); 671bf1304e9cd755be7814ac8365834b7a1d0f06b58Paul E. McKenney ftrace_dump(DUMP_ORIG); 6720989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", 6730989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney current->pid, current->comm, 6740989cb46783188ea7346ba6490be0046b9b7a725Paul E. McKenney idle->pid, idle->comm); /* must be idle task! */ 6759b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney } 6769b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney} 6779b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 678adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker/* 679adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Exit an RCU extended quiescent state, which can be either the 680adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * idle loop or adaptive-tickless usermode execution. 6819b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney */ 682adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckerstatic void rcu_eqs_exit(bool user) 6839b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney{ 6849b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney struct rcu_dynticks *rdtp; 6859b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney long long oldval; 6869b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 687c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rdtp = this_cpu_ptr(&rcu_dynticks); 6889b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney oldval = rdtp->dynticks_nesting; 68929e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney WARN_ON_ONCE(oldval < 0); 6903a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney if (oldval & DYNTICK_TASK_NEST_MASK) { 69129e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE; 6923a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney } else { 69329e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; 6943a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney rcu_eqs_exit_common(rdtp, oldval, user); 6953a5924052aec266f1035f2ff610b60b7d10dbe7fPaul E. McKenney } 6969b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney} 697adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker 698adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker/** 699adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_idle_exit - inform RCU that current CPU is leaving idle 700adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 701adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Exit idle mode, in other words, -enter- the mode in which RCU 702adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * read-side critical sections can occur. 703adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 704adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to 705adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * allow for the possibility of usermode upcalls messing up our count 706adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * of interrupt nesting level during the busy period that is just 707adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * now starting. 708adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker */ 709adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckervoid rcu_idle_exit(void) 710adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker{ 711c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker unsigned long flags; 712c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker 713c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker local_irq_save(flags); 714cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney rcu_eqs_exit(false); 715c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rcu_sysidle_exit(this_cpu_ptr(&rcu_dynticks), 0); 716c5d900bf676b1e2a61c44483932c8088651bbb4eFrederic Weisbecker local_irq_restore(flags); 717adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker} 7188a2ecf474d3ee8dd5d001490349e422cec52f39fPaul E. McKenneyEXPORT_SYMBOL_GPL(rcu_idle_exit); 7199b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 7202b1d5024e17be459aa6385763ca3faa8f01c52d9Frederic Weisbecker#ifdef CONFIG_RCU_USER_QS 7219b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney/** 722adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * rcu_user_exit - inform RCU that we are exiting userspace. 723adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * 724adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * Exit RCU idle mode while entering the kernel because it can 725adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker * run a RCU read side critical section anytime. 726adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker */ 727adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbeckervoid rcu_user_exit(void) 728adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker{ 72991d1aa43d30505b0b825db8898ffc80a8eca96c7Frederic Weisbecker rcu_eqs_exit(1); 730adf5091e6ccaa02905e7a28f9ff44f46c7f4c230Frederic Weisbecker} 7312b1d5024e17be459aa6385763ca3faa8f01c52d9Frederic Weisbecker#endif /* CONFIG_RCU_USER_QS */ 73219dd1591fc379f1d89f39cd99cbbe97433baa3c3Frederic Weisbecker 73319dd1591fc379f1d89f39cd99cbbe97433baa3c3Frederic Weisbecker/** 7349b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle 7359b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 7369b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * Enter an interrupt handler, which might possibly result in exiting 7379b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * idle mode, in other words, entering the mode in which read-side critical 7389b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * sections can occur. 7399b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 7409b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * Note that the Linux kernel is fully capable of entering an interrupt 7419b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * handler that it never exits, for example when doing upcalls to 7429b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * user mode! This code assumes that the idle loop never does upcalls to 7439b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * user mode. If your architecture does do upcalls from the idle loop (or 7449b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * does anything else that results in unbalanced calls to the irq_enter() 7459b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * and irq_exit() functions), RCU will give you what you deserve, good 7469b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * and hard. But very infrequently and irreproducibly. 7479b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 7489b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * Use things like work queues to work around this limitation. 7499b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * 7509b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * You have been warned. 7519b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney */ 7529b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenneyvoid rcu_irq_enter(void) 7539b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney{ 7549b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney unsigned long flags; 7559b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney struct rcu_dynticks *rdtp; 7569b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney long long oldval; 7579b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 7589b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney local_irq_save(flags); 759c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rdtp = this_cpu_ptr(&rcu_dynticks); 7609b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney oldval = rdtp->dynticks_nesting; 7619b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney rdtp->dynticks_nesting++; 7629b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney WARN_ON_ONCE(rdtp->dynticks_nesting == 0); 763b6fc6020140db437069d5bec447858fcfd64d31cFrederic Weisbecker if (oldval) 764f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting); 765b6fc6020140db437069d5bec447858fcfd64d31cFrederic Weisbecker else 766cb349ca95407cbc11424d5e9fc7c8e700709041bPaul E. McKenney rcu_eqs_exit_common(rdtp, oldval, true); 767eb348b898290da242e46df75ab0b9772003e08b8Paul E. McKenney rcu_sysidle_exit(rdtp, 1); 76864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_restore(flags); 76964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 77064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 77164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/** 77264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * rcu_nmi_enter - inform RCU of entry to NMI context 77364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 77464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * If the CPU was idle with dynamic ticks active, and there is no 77564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * irq handler running, this updates rdtp->dynticks_nmi to let the 77664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * RCU grace-period handling know that the CPU is active. 77764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 77864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyvoid rcu_nmi_enter(void) 77964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 780c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); 78164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 78223b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney if (rdtp->dynticks_nmi_nesting == 0 && 78323b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney (atomic_read(&rdtp->dynticks) & 0x1)) 78464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 78523b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney rdtp->dynticks_nmi_nesting++; 7864e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* Force delay from prior write. */ 78723b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney atomic_inc(&rdtp->dynticks); 78823b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 7894e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); /* See above. */ 79023b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 79164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 79264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 79364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/** 79464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * rcu_nmi_exit - inform RCU of exit from NMI context 79564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 79664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * If the CPU was idle with dynamic ticks active, and there is no 79764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * irq handler running, this updates rdtp->dynticks_nmi to let the 79864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * RCU grace-period handling know that the CPU is no longer active. 79964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 80064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyvoid rcu_nmi_exit(void) 80164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 802c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); 80364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 80423b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney if (rdtp->dynticks_nmi_nesting == 0 || 80523b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney --rdtp->dynticks_nmi_nesting != 0) 80664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 80723b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 8084e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* See above. */ 80923b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney atomic_inc(&rdtp->dynticks); 8104e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); /* Force delay to next write. */ 81123b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 81264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 81364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 81464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/** 8155c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * __rcu_is_watching - are RCU read-side critical sections safe? 8165c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * 8175c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * Return true if RCU is watching the running CPU, which means that 8185c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * this CPU can safely enter RCU read-side critical sections. Unlike 8195c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * rcu_is_watching(), the caller of __rcu_is_watching() must have at 8205c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * least disabled preemption. 8215c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney */ 8229418fb208059d6f916bdb116653e166c5db9456dSteven Rostedtbool notrace __rcu_is_watching(void) 8235c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney{ 8245c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1; 8255c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney} 8265c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney 8275c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney/** 8285c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney * rcu_is_watching - see if RCU thinks that the current CPU is idle 82964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 8309b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * If the current CPU is in its idle loop and is neither in an interrupt 83134240697d619c439c55f21989680024dcb604aabPaul E. McKenney * or NMI handler, return true. 83264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 8339418fb208059d6f916bdb116653e166c5db9456dSteven Rostedtbool notrace rcu_is_watching(void) 83464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 835f534ed1fd71cea885a59255d9b44c3b17df03eb1Pranith Kumar bool ret; 83634240697d619c439c55f21989680024dcb604aabPaul E. McKenney 83734240697d619c439c55f21989680024dcb604aabPaul E. McKenney preempt_disable(); 8385c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney ret = __rcu_is_watching(); 83934240697d619c439c55f21989680024dcb604aabPaul E. McKenney preempt_enable(); 84034240697d619c439c55f21989680024dcb604aabPaul E. McKenney return ret; 84164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 8425c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_is_watching); 84364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 84462fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) 845c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney 846c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney/* 847c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * Is the current CPU online? Disable preemption to avoid false positives 848c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * that could otherwise happen due to the current CPU number being sampled, 849c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * this task being preempted, its old CPU being taken offline, resuming 850c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * on some other CPU, then determining that its old CPU is now offline. 851c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * It is OK to use RCU on an offline processor during initial boot, hence 8522036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * the check for rcu_scheduler_fully_active. Note also that it is OK 8532036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * for a CPU coming online to use RCU for one jiffy prior to marking itself 8542036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * online in the cpu_online_mask. Similarly, it is OK for a CPU going 8552036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * offline to continue to use RCU for one jiffy after marking itself 8562036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * offline in the cpu_online_mask. This leniency is necessary given the 8572036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * non-atomic nature of the online and offline processing, for example, 8582036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * the fact that a CPU enters the scheduler after completing the CPU_DYING 8592036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * notifiers. 8602036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * 8612036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * This is also why RCU internally marks CPUs online during the 8622036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * CPU_UP_PREPARE phase and offline during the CPU_DEAD phase. 863c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * 864c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * Disable checking if in an NMI handler because we cannot safely report 865c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney * errors from NMI handlers anyway. 866c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney */ 867c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenneybool rcu_lockdep_current_cpu_online(void) 868c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney{ 8692036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney struct rcu_data *rdp; 8702036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney struct rcu_node *rnp; 871c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney bool ret; 872c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney 873c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney if (in_nmi()) 874f6f7ee9af7554e4d167ccd0ffe7cb8da0aa954f9Fengguang Wu return true; 875c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney preempt_disable(); 876c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter rdp = this_cpu_ptr(&rcu_sched_data); 8772036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rnp = rdp->mynode; 8782036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney ret = (rdp->grpmask & rnp->qsmaskinit) || 879c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney !rcu_scheduler_fully_active; 880c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney preempt_enable(); 881c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney return ret; 882c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney} 883c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online); 884c0d6d01bffdce19fa19baad6cb8cc3eed7bfd6f5Paul E. McKenney 88562fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */ 8869b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney 88764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/** 8889b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle 88964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 8909b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * If the current CPU is idle or running at a first-level (not nested) 8919b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * interrupt from idle, return true. The caller must have at least 8929b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * disabled preemption. 89364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 89462e3cb143fd78a2ee6f44ef0dfe50cdff2119d9aJosh Triplettstatic int rcu_is_cpu_rrupt_from_idle(void) 89564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 896c9d4b0af9e0609cc525c55de18229fde7c926d61Christoph Lameter return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 1; 89764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 89864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 89964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 90064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Snapshot the specified CPU's dynticks counter so that we can later 90164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * credit them with an implicit quiescent state. Return 1 if this CPU 9021eba8f84380bede3c602bd7758dea96925cead01Paul E. McKenney * is in dynticks idle mode, which is an extended quiescent state. 90364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 904217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenneystatic int dyntick_save_progress_counter(struct rcu_data *rdp, 905217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney bool *isidle, unsigned long *maxj) 90664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 90723b5c8fa01b723c70a20d6e4ef4ff54c7656d6e1Paul E. McKenney rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); 9080edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney rcu_sysidle_check_cpu(rdp, isidle, maxj); 9097941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat if ((rdp->dynticks_snap & 0x1) == 0) { 9107941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); 9117941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat return 1; 9127941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat } else { 9137941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat return 0; 9147941dbdebe2a1fda31aefa033794510f95720a5aAndreea-Cristina Bernat } 91564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 91664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 91764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 9186193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney * This function really isn't for public consumption, but RCU is special in 9196193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney * that context switches can allow the state machine to make progress. 9206193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney */ 9216193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenneyextern void resched_cpu(int cpu); 9226193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney 9236193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney/* 92464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Return true if the specified CPU has passed through a quiescent 92564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * state by virtue of being in or having passed through an dynticks 92664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * idle state since the last call to dyntick_save_progress_counter() 927a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * for this same CPU, or by virtue of having been offline. 92864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 929217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenneystatic int rcu_implicit_dynticks_qs(struct rcu_data *rdp, 930217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney bool *isidle, unsigned long *maxj) 93164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 9327eb4f4553ceaa6c64da83c8a71d5a991c0188655Paul E. McKenney unsigned int curr; 9334a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney int *rcrmp; 9347eb4f4553ceaa6c64da83c8a71d5a991c0188655Paul E. McKenney unsigned int snap; 93564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 9367eb4f4553ceaa6c64da83c8a71d5a991c0188655Paul E. McKenney curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks); 9377eb4f4553ceaa6c64da83c8a71d5a991c0188655Paul E. McKenney snap = (unsigned int)rdp->dynticks_snap; 93864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 93964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 94064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * If the CPU passed through or entered a dynticks idle phase with 94164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * no active irq/NMI handlers, then we can safely pretend that the CPU 94264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * already acknowledged the request to pass through a quiescent 94364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * state. Either way, that CPU cannot possibly be in an RCU 94464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * read-side critical section that started before the beginning 94564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * of the current RCU grace period. 94664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 9477eb4f4553ceaa6c64da83c8a71d5a991c0188655Paul E. McKenney if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) { 948f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); 94964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->dynticks_fqs++; 95064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 95164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 95264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 953a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney /* 954a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * Check for the CPU being offline, but only if the grace period 955a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * is old enough. We don't need to worry about the CPU changing 956a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * state: If we see it offline even once, it has been through a 957a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * quiescent state. 958a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * 959a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * The reason for insisting that the grace period be at least 960a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * one jiffy old is that CPUs that are not quite online and that 961a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * have just gone offline can still execute RCU read-side critical 962a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney * sections. 963a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney */ 964a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies)) 965a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney return 0; /* Grace period is not old enough. */ 966a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney barrier(); 967a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney if (cpu_is_offline(rdp->cpu)) { 968f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl")); 969a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney rdp->offline_fqs++; 970a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney return 1; 971a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney } 97265d798f0f9339ae2c4ebe9480e3260b33382a584Paul E. McKenney 97365d798f0f9339ae2c4ebe9480e3260b33382a584Paul E. McKenney /* 9744a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * A CPU running for an extended time within the kernel can 9754a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode, 9764a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * even context-switching back and forth between a pair of 9774a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * in-kernel CPU-bound tasks cannot advance grace periods. 9784a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * So if the grace period is old enough, make the CPU pay attention. 9794a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * Note that the unsynchronized assignments to the per-CPU 9804a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * rcu_sched_qs_mask variable are safe. Yes, setting of 9814a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * bits can be lost, but they will be set again on the next 9824a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * force-quiescent-state pass. So lost bit sets do not result 9834a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * in incorrect behavior, merely in a grace period lasting 9844a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * a few jiffies longer than it might otherwise. Because 9854a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * there are at most four threads involved, and because the 9864a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * updates are only once every few jiffies, the probability of 9874a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * lossage (and thus of slight grace-period extension) is 9884a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * quite low. 9894a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * 9904a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * Note that if the jiffies_till_sched_qs boot/sysfs parameter 9914a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * is set too high, we override with half of the RCU CPU stall 9924a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney * warning delay. 9936193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney */ 9944a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu); 9954a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney if (ULONG_CMP_GE(jiffies, 9964a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rdp->rsp->gp_start + jiffies_till_sched_qs) || 997cb1e78cfa267453bb19e7edafd214c03834b664cPaul E. McKenney ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { 9984a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney if (!(ACCESS_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { 9994a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney ACCESS_ONCE(rdp->cond_resched_completed) = 10004a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney ACCESS_ONCE(rdp->mynode->completed); 10014a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney smp_mb(); /* ->cond_resched_completed before *rcrmp. */ 10024a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney ACCESS_ONCE(*rcrmp) = 10034a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney ACCESS_ONCE(*rcrmp) + rdp->rsp->flavor_mask; 10044a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ 10054a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rdp->rsp->jiffies_resched += 5; /* Enable beating. */ 10064a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { 10074a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney /* Time to beat on that CPU again! */ 10084a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ 10094a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rdp->rsp->jiffies_resched += 5; /* Re-enable beating. */ 10104a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney } 10116193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney } 10126193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney 1013a82dcc76021e22c174ba85d90b7a8c750b7362d0Paul E. McKenney return 0; 101464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 101564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 101664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void record_gp_stall_check_time(struct rcu_state *rsp) 101764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 1018cb1e78cfa267453bb19e7edafd214c03834b664cPaul E. McKenney unsigned long j = jiffies; 10196193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney unsigned long j1; 102026cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney 102126cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney rsp->gp_start = j; 102226cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney smp_wmb(); /* Record start time before stall time. */ 10236193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney j1 = rcu_jiffies_till_stall_check(); 10244fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda ACCESS_ONCE(rsp->jiffies_stall) = j + j1; 10256193c76aba8ec3cc5f083c35efbab9ed924125f6Paul E. McKenney rsp->jiffies_resched = j + j1 / 2; 102664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 102764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 1028b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney/* 1029bc1dce514e9b29b64df28a533015885862f47814Paul E. McKenney * Dump stacks of all tasks running on stalled CPUs. 1030b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney */ 1031b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenneystatic void rcu_dump_cpu_stacks(struct rcu_state *rsp) 1032b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney{ 1033b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney int cpu; 1034b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney unsigned long flags; 1035b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney struct rcu_node *rnp; 1036b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney 1037b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney rcu_for_each_leaf_node(rsp, rnp) { 1038b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 1039b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney if (rnp->qsmask != 0) { 1040b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++) 1041b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney if (rnp->qsmask & (1UL << cpu)) 1042b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney dump_cpu_task(rnp->grplo + cpu); 1043b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney } 1044b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 1045b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney } 1046b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney} 1047b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney 104864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void print_other_cpu_stall(struct rcu_state *rsp) 104964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 105064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int cpu; 105164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney long delta; 105264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 1053285fe29481d865ae381ad3924c80894e6968c2d8Paul E. McKenney int ndetected = 0; 105464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 105553bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney long totqlen = 0; 105664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 105764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Only let one CPU complain about others per time interval. */ 105864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 10591304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 10604fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda delta = jiffies - ACCESS_ONCE(rsp->jiffies_stall); 1061fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { 10621304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 106364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 106464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 10654fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda ACCESS_ONCE(rsp->jiffies_stall) = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; 10661304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 106764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 10688cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney /* 10698cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * OK, time to rat on our buddy... 10708cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * See Documentation/RCU/stallwarn.txt for info on how to debug 10718cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * RCU CPU stall warnings. 10728cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney */ 1073d7f3e207397d7b4868e33d3f88396a06f4d5a8c7Paul E. McKenney pr_err("INFO: %s detected stalls on CPUs/tasks:", 10744300aa642cc9ecb35f2e0683dd294fb790ef028cPaul E. McKenney rsp->name); 1075a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney print_cpu_stall_info_begin(); 1076a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney rcu_for_each_leaf_node(rsp, rnp) { 10773acd9eb31c5f7eb97cb2009fa41472710fb4a10fPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 10789bc8b5586f94be6391458074ecbba8827ba8ba9dPaul E. McKenney ndetected += rcu_print_task_stall(rnp); 1079c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney if (rnp->qsmask != 0) { 1080c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++) 1081c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney if (rnp->qsmask & (1UL << cpu)) { 1082c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney print_cpu_stall_info(rsp, 1083c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney rnp->grplo + cpu); 1084c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney ndetected++; 1085c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney } 1086c8020a67e625c714c4dbedc8ae2944b461e204ecPaul E. McKenney } 10873acd9eb31c5f7eb97cb2009fa41472710fb4a10fPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 108864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 1089a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney 1090a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney /* 1091a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney * Now rat on any tasks that got kicked up to the root rcu_node 1092a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney * due to CPU offlining. 1093a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney */ 1094a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney rnp = rcu_get_root(rsp); 1095a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 1096285fe29481d865ae381ad3924c80894e6968c2d8Paul E. McKenney ndetected += rcu_print_task_stall(rnp); 1097a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 1098a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney 1099a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney print_cpu_stall_info_end(); 110053bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney for_each_possible_cpu(cpu) 110153bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen; 110283ebe63ead0fe60e4b548730800cb68293ce098bPaul E. McKenney pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n", 1103eee058826100e5a344f11601e0c47baeaf07c77bPaul E. McKenney smp_processor_id(), (long)(jiffies - rsp->gp_start), 110483ebe63ead0fe60e4b548730800cb68293ce098bPaul E. McKenney (long)rsp->gpnum, (long)rsp->completed, totqlen); 11059bc8b5586f94be6391458074ecbba8827ba8ba9dPaul E. McKenney if (ndetected == 0) 1106d7f3e207397d7b4868e33d3f88396a06f4d5a8c7Paul E. McKenney pr_err("INFO: Stall ended before state dump start\n"); 1107bc1dce514e9b29b64df28a533015885862f47814Paul E. McKenney else 1108b637a328bd4f43a0e146d1eef0142b650ba0d644Paul E. McKenney rcu_dump_cpu_stacks(rsp); 1109c1dc0b9c0c8979ce4d411caadff5c0d79dee58bcIngo Molnar 11104cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Complain about tasks blocking the grace period. */ 11111ed509a225008c9e8c0644fbd22168e09a7383a0Paul E. McKenney 11121ed509a225008c9e8c0644fbd22168e09a7383a0Paul E. McKenney rcu_print_detail_task_stall(rsp); 11131ed509a225008c9e8c0644fbd22168e09a7383a0Paul E. McKenney 11144cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney force_quiescent_state(rsp); /* Kick them all. */ 111564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 111664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 111764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void print_cpu_stall(struct rcu_state *rsp) 111864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 111953bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney int cpu; 112064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 112164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 112253bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney long totqlen = 0; 112364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 11248cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney /* 11258cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * OK, time to rat on ourselves... 11268cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * See Documentation/RCU/stallwarn.txt for info on how to debug 11278cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney * RCU CPU stall warnings. 11288cdd32a918350430483751feaae1c19cef816f69Paul E. McKenney */ 1129d7f3e207397d7b4868e33d3f88396a06f4d5a8c7Paul E. McKenney pr_err("INFO: %s self-detected stall on CPU", rsp->name); 1130a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney print_cpu_stall_info_begin(); 1131a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney print_cpu_stall_info(rsp, smp_processor_id()); 1132a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney print_cpu_stall_info_end(); 113353bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney for_each_possible_cpu(cpu) 113453bb857c373d6b7936f8a7b4451f0a99703c308ePaul E. McKenney totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen; 113583ebe63ead0fe60e4b548730800cb68293ce098bPaul E. McKenney pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n", 113683ebe63ead0fe60e4b548730800cb68293ce098bPaul E. McKenney jiffies - rsp->gp_start, 113783ebe63ead0fe60e4b548730800cb68293ce098bPaul E. McKenney (long)rsp->gpnum, (long)rsp->completed, totqlen); 1138bc1dce514e9b29b64df28a533015885862f47814Paul E. McKenney rcu_dump_cpu_stacks(rsp); 1139c1dc0b9c0c8979ce4d411caadff5c0d79dee58bcIngo Molnar 11401304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 11414fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda if (ULONG_CMP_GE(jiffies, ACCESS_ONCE(rsp->jiffies_stall))) 11424fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda ACCESS_ONCE(rsp->jiffies_stall) = jiffies + 11436bfc09e2327dfbffc312004c16188dbf8dfb0297Paul E. McKenney 3 * rcu_jiffies_till_stall_check() + 3; 11441304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 1145c1dc0b9c0c8979ce4d411caadff5c0d79dee58bcIngo Molnar 1146b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra /* 1147b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra * Attempt to revive the RCU machinery by forcing a context switch. 1148b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra * 1149b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra * A context switch would normally allow the RCU state machine to make 1150b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra * progress and it could be we're stuck in kernel space without context 1151b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra * switches for an entirely unreasonable amount of time. 1152b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra */ 1153b021fe3e25094fbec22d0eff846d2adeee1b9736Peter Zijlstra resched_cpu(smp_processor_id()); 115464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 115564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 115664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) 115764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 115826cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney unsigned long completed; 115926cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney unsigned long gpnum; 116026cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney unsigned long gps; 1161bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney unsigned long j; 1162bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney unsigned long js; 116364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp; 116464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 116526cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp)) 1166c68de2097a8799549a3c3bf27cbfeea24a604284Paul E. McKenney return; 1167cb1e78cfa267453bb19e7edafd214c03834b664cPaul E. McKenney j = jiffies; 116826cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney 116926cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney /* 117026cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * Lots of memory barriers to reject false positives. 117126cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * 117226cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * The idea is to pick up rsp->gpnum, then rsp->jiffies_stall, 117326cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * then rsp->gp_start, and finally rsp->completed. These values 117426cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * are updated in the opposite order with memory barriers (or 117526cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * equivalent) during grace-period initialization and cleanup. 117626cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * Now, a false positive can occur if we get an new value of 117726cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * rsp->gp_start and a old value of rsp->jiffies_stall. But given 117826cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * the memory barriers, the only way that this can happen is if one 117926cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * grace period ends and another starts between these two fetches. 118026cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * Detect this by comparing rsp->completed with the previous fetch 118126cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * from rsp->gpnum. 118226cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * 118326cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * Given this check, comparisons of jiffies, rsp->jiffies_stall, 118426cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney * and rsp->gp_start suffice to forestall false positives. 118526cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney */ 118626cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney gpnum = ACCESS_ONCE(rsp->gpnum); 118726cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney smp_rmb(); /* Pick up ->gpnum first... */ 1188bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney js = ACCESS_ONCE(rsp->jiffies_stall); 118926cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney smp_rmb(); /* ...then ->jiffies_stall before the rest... */ 119026cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney gps = ACCESS_ONCE(rsp->gp_start); 119126cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney smp_rmb(); /* ...and finally ->gp_start before ->completed. */ 119226cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney completed = ACCESS_ONCE(rsp->completed); 119326cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney if (ULONG_CMP_GE(completed, gpnum) || 119426cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney ULONG_CMP_LT(j, js) || 119526cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney ULONG_CMP_GE(gps, js)) 119626cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney return; /* No stall or GP completed since entering function. */ 119764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rdp->mynode; 1198c96ea7cfdd88d0a67c970502bc5313fede34b86bPaul E. McKenney if (rcu_gp_in_progress(rsp) && 119926cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney (ACCESS_ONCE(rnp->qsmask) & rdp->grpmask)) { 120064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 120164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* We haven't checked in, so go dump stack. */ 120264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney print_cpu_stall(rsp); 120364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 1204bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney } else if (rcu_gp_in_progress(rsp) && 1205bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) { 120664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 1207bad6e1393cb505fe17747344a23666464daa3fa7Paul E. McKenney /* They had a few time units to dump stack, so complain. */ 120864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney print_other_cpu_stall(rsp); 120964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 121064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 121164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 121253d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney/** 121353d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * rcu_cpu_stall_reset - prevent further stall warnings in current grace period 121453d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * 121553d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * Set the stall-warning timeout way off into the future, thus preventing 121653d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * any RCU CPU stall-warning messages from appearing in the current set of 121753d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * RCU grace periods. 121853d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * 121953d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney * The caller must disable hard irqs. 122053d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney */ 122153d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenneyvoid rcu_cpu_stall_reset(void) 122253d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney{ 12236ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 12246ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney 12256ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 12264fc5b75537d4f56577ad00355b4cd09627deb3c3Iulia Manda ACCESS_ONCE(rsp->jiffies_stall) = jiffies + ULONG_MAX / 2; 122753d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney} 122853d84e004d5e8c018be395c4330dc72fd60bd13ePaul E. McKenney 122964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 12303f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney * Initialize the specified rcu_data structure's callback list to empty. 12313f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney */ 12323f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenneystatic void init_callback_list(struct rcu_data *rdp) 12333f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney{ 12343f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney int i; 12353f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney 123634ed62461ae4970695974afb9a60ac3df0086830Paul E. McKenney if (init_nocb_callback_list(rdp)) 123734ed62461ae4970695974afb9a60ac3df0086830Paul E. McKenney return; 12383f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney rdp->nxtlist = NULL; 12393f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney for (i = 0; i < RCU_NEXT_SIZE; i++) 12403f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney rdp->nxttail[i] = &rdp->nxtlist; 12413f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney} 12423f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney 12433f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney/* 1244dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Determine the value that ->completed will have at the end of the 1245dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * next subsequent grace period. This is used to tag callbacks so that 1246dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * a CPU can invoke callbacks in a timely fashion even if that CPU has 1247dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * been dyntick-idle for an extended period with callbacks under the 1248dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * influence of RCU_FAST_NO_HZ. 1249dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * 1250dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * The caller must hold rnp->lock with interrupts disabled. 1251dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1252dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenneystatic unsigned long rcu_cbs_completed(struct rcu_state *rsp, 1253dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney struct rcu_node *rnp) 1254dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney{ 1255dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1256dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * If RCU is idle, we just wait for the next grace period. 1257dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * But we can only be sure that RCU is idle if we are looking 1258dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * at the root rcu_node structure -- otherwise, a new grace 1259dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * period might have started, but just not yet gotten around 1260dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * to initializing the current non-root rcu_node structure. 1261dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1262dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rcu_get_root(rsp) == rnp && rnp->gpnum == rnp->completed) 1263dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return rnp->completed + 1; 1264dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1265dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1266dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Otherwise, wait for a possible partial grace period and 1267dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * then the subsequent full grace period. 1268dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1269dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney return rnp->completed + 2; 1270dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney} 1271dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1272dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney/* 12730446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * Trace-event helper function for rcu_start_future_gp() and 12740446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * rcu_nocb_wait_gp(). 12750446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 12760446be489795d8bb994125a916ef03211f539e54Paul E. McKenneystatic void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, 1277e66c33d579ea566d10e8c8695a7168aae3e02992Steven Rostedt (Red Hat) unsigned long c, const char *s) 12780446be489795d8bb994125a916ef03211f539e54Paul E. McKenney{ 12790446be489795d8bb994125a916ef03211f539e54Paul E. McKenney trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, 12800446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rnp->completed, c, rnp->level, 12810446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rnp->grplo, rnp->grphi, s); 12820446be489795d8bb994125a916ef03211f539e54Paul E. McKenney} 12830446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 12840446be489795d8bb994125a916ef03211f539e54Paul E. McKenney/* 12850446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * Start some future grace period, as needed to handle newly arrived 12860446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * callbacks. The required future grace periods are recorded in each 128748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * rcu_node structure's ->need_future_gp field. Returns true if there 128848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * is reason to awaken the grace-period kthread. 12890446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * 12900446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * The caller must hold the specified rcu_node structure's ->lock. 12910446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 129248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool __maybe_unused 129348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneyrcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, 129448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney unsigned long *c_out) 12950446be489795d8bb994125a916ef03211f539e54Paul E. McKenney{ 12960446be489795d8bb994125a916ef03211f539e54Paul E. McKenney unsigned long c; 12970446be489795d8bb994125a916ef03211f539e54Paul E. McKenney int i; 129848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool ret = false; 12990446be489795d8bb994125a916ef03211f539e54Paul E. McKenney struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); 13000446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13010446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* 13020446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * Pick up grace-period number for new callbacks. If this 13030446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * grace period is already marked as needed, return to the caller. 13040446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13050446be489795d8bb994125a916ef03211f539e54Paul E. McKenney c = rcu_cbs_completed(rdp->rsp, rnp); 1306f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf")); 13070446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (rnp->need_future_gp[c & 0x1]) { 1308f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf")); 130948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney goto out; 13100446be489795d8bb994125a916ef03211f539e54Paul E. McKenney } 13110446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13120446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* 13130446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * If either this rcu_node structure or the root rcu_node structure 13140446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * believe that a grace period is in progress, then we must wait 13150446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * for the one following, which is in "c". Because our request 13160446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * will be noticed at the end of the current grace period, we don't 131748bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * need to explicitly start one. We only do the lockless check 131848bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * of rnp_root's fields if the current rcu_node structure thinks 131948bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * there is no grace period in flight, and because we hold rnp->lock, 132048bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * the only possible change is when rnp_root's two fields are 132148bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * equal, in which case rnp_root->gpnum might be concurrently 132248bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * incremented. But that is OK, as it will just result in our 132348bd8e9b82a750b983823f391c67e70553757afaPranith Kumar * doing some extra useless work. 13240446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13250446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (rnp->gpnum != rnp->completed || 132648bd8e9b82a750b983823f391c67e70553757afaPranith Kumar ACCESS_ONCE(rnp_root->gpnum) != ACCESS_ONCE(rnp_root->completed)) { 13270446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rnp->need_future_gp[c & 0x1]++; 1328f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf")); 132948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney goto out; 13300446be489795d8bb994125a916ef03211f539e54Paul E. McKenney } 13310446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13320446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* 13330446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * There might be no grace period in progress. If we don't already 13340446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * hold it, acquire the root rcu_node structure's lock in order to 13350446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * start one (if needed). 13360446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13376303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney if (rnp != rnp_root) { 13380446be489795d8bb994125a916ef03211f539e54Paul E. McKenney raw_spin_lock(&rnp_root->lock); 13396303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 13406303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney } 13410446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13420446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* 13430446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * Get a new grace-period number. If there really is no grace 13440446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * period in progress, it will be smaller than the one we obtained 13450446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * earlier. Adjust callbacks as needed. Note that even no-CBs 13460446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * CPUs have a ->nxtcompleted[] array, so no no-CBs checks needed. 13470446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13480446be489795d8bb994125a916ef03211f539e54Paul E. McKenney c = rcu_cbs_completed(rdp->rsp, rnp_root); 13490446be489795d8bb994125a916ef03211f539e54Paul E. McKenney for (i = RCU_DONE_TAIL; i < RCU_NEXT_TAIL; i++) 13500446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (ULONG_CMP_LT(c, rdp->nxtcompleted[i])) 13510446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rdp->nxtcompleted[i] = c; 13520446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13530446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* 13540446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * If the needed for the required grace period is already 13550446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * recorded, trace and leave. 13560446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13570446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (rnp_root->need_future_gp[c & 0x1]) { 1358f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot")); 13590446be489795d8bb994125a916ef03211f539e54Paul E. McKenney goto unlock_out; 13600446be489795d8bb994125a916ef03211f539e54Paul E. McKenney } 13610446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13620446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* Record the need for the future grace period. */ 13630446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rnp_root->need_future_gp[c & 0x1]++; 13640446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13650446be489795d8bb994125a916ef03211f539e54Paul E. McKenney /* If a grace period is not already in progress, start one. */ 13660446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (rnp_root->gpnum != rnp_root->completed) { 1367f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot")); 13680446be489795d8bb994125a916ef03211f539e54Paul E. McKenney } else { 1369f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot")); 137048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp); 13710446be489795d8bb994125a916ef03211f539e54Paul E. McKenney } 13720446be489795d8bb994125a916ef03211f539e54Paul E. McKenneyunlock_out: 13730446be489795d8bb994125a916ef03211f539e54Paul E. McKenney if (rnp != rnp_root) 13740446be489795d8bb994125a916ef03211f539e54Paul E. McKenney raw_spin_unlock(&rnp_root->lock); 137548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneyout: 137648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (c_out != NULL) 137748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney *c_out = c; 137848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return ret; 13790446be489795d8bb994125a916ef03211f539e54Paul E. McKenney} 13800446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13810446be489795d8bb994125a916ef03211f539e54Paul E. McKenney/* 13820446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * Clean up any old requests for the just-ended grace period. Also return 13830446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * whether any additional grace periods have been requested. Also invoke 13840446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * rcu_nocb_gp_cleanup() in order to wake up any no-callbacks kthreads 13850446be489795d8bb994125a916ef03211f539e54Paul E. McKenney * waiting for this grace period to complete. 13860446be489795d8bb994125a916ef03211f539e54Paul E. McKenney */ 13870446be489795d8bb994125a916ef03211f539e54Paul E. McKenneystatic int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) 13880446be489795d8bb994125a916ef03211f539e54Paul E. McKenney{ 13890446be489795d8bb994125a916ef03211f539e54Paul E. McKenney int c = rnp->completed; 13900446be489795d8bb994125a916ef03211f539e54Paul E. McKenney int needmore; 13910446be489795d8bb994125a916ef03211f539e54Paul E. McKenney struct rcu_data *rdp = this_cpu_ptr(rsp->rda); 13920446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 13930446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rcu_nocb_gp_cleanup(rsp, rnp); 13940446be489795d8bb994125a916ef03211f539e54Paul E. McKenney rnp->need_future_gp[c & 0x1] = 0; 13950446be489795d8bb994125a916ef03211f539e54Paul E. McKenney needmore = rnp->need_future_gp[(c + 1) & 0x1]; 1396f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_future_gp(rnp, rdp, c, 1397f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) needmore ? TPS("CleanupMore") : TPS("Cleanup")); 13980446be489795d8bb994125a916ef03211f539e54Paul E. McKenney return needmore; 13990446be489795d8bb994125a916ef03211f539e54Paul E. McKenney} 14000446be489795d8bb994125a916ef03211f539e54Paul E. McKenney 14010446be489795d8bb994125a916ef03211f539e54Paul E. McKenney/* 140248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Awaken the grace-period kthread for the specified flavor of RCU. 140348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Don't do a self-awaken, and don't bother awakening when there is 140448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * nothing for the grace-period kthread to do (as in several CPUs 140548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * raced to awaken, and we lost), and finally don't try to awaken 140648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * a kthread that has not yet been created. 140748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney */ 140848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic void rcu_gp_kthread_wake(struct rcu_state *rsp) 140948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney{ 141048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (current == rsp->gp_kthread || 141148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney !ACCESS_ONCE(rsp->gp_flags) || 141248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney !rsp->gp_kthread) 141348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return; 141448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney wake_up(&rsp->gp_wq); 141548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney} 141648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney 141748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney/* 1418dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * If there is room, assign a ->completed number to any callbacks on 1419dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * this CPU that have not already been assigned. Also accelerate any 1420dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * callbacks that were previously assigned a ->completed number that has 1421dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * since proven to be too conservative, which can happen if callbacks get 1422dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * assigned a ->completed number while RCU is idle, but with reference to 1423dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * a non-root rcu_node structure. This function is idempotent, so it does 142448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * not hurt to call it repeatedly. Returns an flag saying that we should 142548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * awaken the RCU grace-period kthread. 1426dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * 1427dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * The caller must hold rnp->lock with interrupts disabled. 1428dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 142948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, 1430dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney struct rcu_data *rdp) 1431dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney{ 1432dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney unsigned long c; 1433dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney int i; 143448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool ret; 1435dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1436dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* If the CPU has no callbacks, nothing to do. */ 1437dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) 143848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return false; 1439dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1440dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1441dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Starting from the sublist containing the callbacks most 1442dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * recently assigned a ->completed number and working down, find the 1443dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * first sublist that is not assignable to an upcoming grace period. 1444dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Such a sublist has something in it (first two tests) and has 1445dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * a ->completed number assigned that will complete sooner than 1446dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * the ->completed number for newly arrived callbacks (last test). 1447dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * 1448dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * The key point is that any later sublist can be assigned the 1449dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * same ->completed number as the newly arrived callbacks, which 1450dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * means that the callbacks in any of these later sublist can be 1451dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * grouped into a single sublist, whether or not they have already 1452dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * been assigned a ->completed number. 1453dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1454dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney c = rcu_cbs_completed(rsp, rnp); 1455dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (i = RCU_NEXT_TAIL - 1; i > RCU_DONE_TAIL; i--) 1456dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rdp->nxttail[i] != rdp->nxttail[i - 1] && 1457dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney !ULONG_CMP_GE(rdp->nxtcompleted[i], c)) 1458dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney break; 1459dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1460dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1461dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * If there are no sublist for unassigned callbacks, leave. 1462dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * At the same time, advance "i" one sublist, so that "i" will 1463dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * index into the sublist where all the remaining callbacks should 1464dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * be grouped into. 1465dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1466dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (++i >= RCU_NEXT_TAIL) 146748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return false; 1468dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1469dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1470dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Assign all subsequent callbacks' ->completed number to the next 1471dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * full grace period and group them all in the sublist initially 1472dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * indexed by "i". 1473dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1474dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (; i <= RCU_NEXT_TAIL; i++) { 1475dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxttail[i] = rdp->nxttail[RCU_NEXT_TAIL]; 1476dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxtcompleted[i] = c; 1477dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney } 1478910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney /* Record any needed additional grace periods. */ 147948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_start_future_gp(rnp, rdp, NULL); 14806d4b418c75a74eea1dd3701e106a9da8c335c451Paul E. McKenney 14816d4b418c75a74eea1dd3701e106a9da8c335c451Paul E. McKenney /* Trace depending on how much we were able to accelerate. */ 14826d4b418c75a74eea1dd3701e106a9da8c335c451Paul E. McKenney if (!*rdp->nxttail[RCU_WAIT_TAIL]) 1483f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB")); 14846d4b418c75a74eea1dd3701e106a9da8c335c451Paul E. McKenney else 1485f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB")); 148648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return ret; 1487dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney} 1488dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1489dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney/* 1490dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Move any callbacks whose grace period has completed to the 1491dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * RCU_DONE_TAIL sublist, then compact the remaining sublists and 1492dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * assign ->completed numbers to any callbacks in the RCU_NEXT_TAIL 1493dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * sublist. This function is idempotent, so it does not hurt to 1494dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * invoke it repeatedly. As long as it is not invoked -too- often... 149548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Returns true if the RCU grace-period kthread needs to be awakened. 1496dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * 1497dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * The caller must hold rnp->lock with interrupts disabled. 1498dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 149948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool rcu_advance_cbs(struct rcu_state *rsp, struct rcu_node *rnp, 1500dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney struct rcu_data *rdp) 1501dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney{ 1502dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney int i, j; 1503dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1504dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* If the CPU has no callbacks, nothing to do. */ 1505dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) 150648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return false; 1507dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1508dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* 1509dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * Find all callbacks whose ->completed numbers indicate that they 1510dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. 1511dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney */ 1512dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { 1513dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (ULONG_CMP_LT(rnp->completed, rdp->nxtcompleted[i])) 1514dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney break; 1515dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[i]; 1516dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney } 1517dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* Clean up any sublist tail pointers that were misordered above. */ 1518dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (j = RCU_WAIT_TAIL; j < i; j++) 1519dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxttail[j] = rdp->nxttail[RCU_DONE_TAIL]; 1520dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1521dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* Copy down callbacks to fill in empty sublists. */ 1522dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { 1523dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rdp->nxttail[j] == rdp->nxttail[RCU_NEXT_TAIL]) 1524dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney break; 1525dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxttail[j] = rdp->nxttail[i]; 1526dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney rdp->nxtcompleted[j] = rdp->nxtcompleted[i]; 1527dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney } 1528dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1529dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* Classify any remaining callbacks. */ 153048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return rcu_accelerate_cbs(rsp, rnp, rdp); 1531dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney} 1532dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1533dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney/* 1534ba9fbe955f026780e6b27c279dba7c86dfdcb7d5Paul E. McKenney * Update CPU-local rcu_data state to record the beginnings and ends of 1535ba9fbe955f026780e6b27c279dba7c86dfdcb7d5Paul E. McKenney * grace periods. The caller must hold the ->lock of the leaf rcu_node 1536ba9fbe955f026780e6b27c279dba7c86dfdcb7d5Paul E. McKenney * structure corresponding to the current CPU, and must have irqs disabled. 153748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Returns true if the grace-period kthread needs to be awakened. 1538d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney */ 153948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, 154048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney struct rcu_data *rdp) 1541d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney{ 154248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool ret; 154348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney 1544ba9fbe955f026780e6b27c279dba7c86dfdcb7d5Paul E. McKenney /* Handle the ends of any preceding grace periods first. */ 1545dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney if (rdp->completed == rnp->completed) { 1546d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney 1547ba9fbe955f026780e6b27c279dba7c86dfdcb7d5Paul E. McKenney /* No grace period end, so just accelerate recent callbacks. */ 154848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_accelerate_cbs(rsp, rnp, rdp); 1549d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney 1550dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney } else { 1551dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney 1552dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* Advance callbacks. */ 155348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_advance_cbs(rsp, rnp, rdp); 1554d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney 1555d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney /* Remember that we saw this grace-period completion. */ 1556d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney rdp->completed = rnp->completed; 1557f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); 1558d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney } 1559398ebe6000c16135d12ce2ff64318f306ffb20b0Paul E. McKenney 15606eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney if (rdp->gpnum != rnp->gpnum) { 15616eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney /* 15626eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney * If the current grace period is waiting for this CPU, 15636eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney * set up to detect a quiescent state, otherwise don't 15646eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney * go looking for one. 15656eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney */ 15666eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney rdp->gpnum = rnp->gpnum; 1567f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); 15686eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney rdp->passed_quiesce = 0; 15696eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask); 15706eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney zero_cpu_stall_ticks(rdp); 15716eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney } 157248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return ret; 15736eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney} 15746eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney 1575d34ea3221a0f34ed42eadabf054604bbcc7ecd27Paul E. McKenneystatic void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) 15766eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney{ 15776eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney unsigned long flags; 157848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool needwake; 15796eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney struct rcu_node *rnp; 15806eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney 15816eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney local_irq_save(flags); 15826eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney rnp = rdp->mynode; 1583d34ea3221a0f34ed42eadabf054604bbcc7ecd27Paul E. McKenney if ((rdp->gpnum == ACCESS_ONCE(rnp->gpnum) && 1584d34ea3221a0f34ed42eadabf054604bbcc7ecd27Paul E. McKenney rdp->completed == ACCESS_ONCE(rnp->completed)) || /* w/out lock. */ 15856eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */ 15866eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney local_irq_restore(flags); 15876eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney return; 15886eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney } 15896303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 159048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needwake = __note_gp_changes(rsp, rnp, rdp); 15916eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 159248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (needwake) 159348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney rcu_gp_kthread_wake(rsp); 15946eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney} 15956eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney 15966eaef633d77f50f031dd355ff5f91aaa1aaf9885Paul E. McKenney/* 1597f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney * Initialize a new grace period. Return 0 if no grace period required. 1598b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney */ 15997fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenneystatic int rcu_gp_init(struct rcu_state *rsp) 1600b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney{ 1601b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney struct rcu_data *rdp; 16027fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 1603b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 1604eb75767be0e514f97bf1b5cec763696cfc7f7e2aPaul E. McKenney rcu_bind_gp_kthread(); 16057fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_lock_irq(&rnp->lock); 16066303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 160791dc95427a0d30ac2c58d6e943c7f40a3f25d908Paul E. McKenney if (!ACCESS_ONCE(rsp->gp_flags)) { 1608f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney /* Spurious wakeup, tell caller to go back to sleep. */ 1609f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 1610f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney return 0; 1611f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney } 161291dc95427a0d30ac2c58d6e943c7f40a3f25d908Paul E. McKenney ACCESS_ONCE(rsp->gp_flags) = 0; /* Clear all flags: New grace period. */ 1613b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 1614f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) { 1615f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney /* 1616f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney * Grace period already in progress, don't start another. 1617f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney * Not supposed to be able to happen. 1618f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney */ 16197fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 16207fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney return 0; 16217fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney } 16227fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 16237fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney /* Advance to a new grace period and initialize state. */ 162426cdfedf6a902345f8604ea8e0b7dd2566b37a46Paul E. McKenney record_gp_stall_check_time(rsp); 1625765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney /* Record GP times before starting GP, hence smp_store_release(). */ 1626765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney smp_store_release(&rsp->gpnum, rsp->gpnum + 1); 1627f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start")); 16287fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 16297fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 16307fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney /* Exclude any concurrent CPU-hotplug operations. */ 1631a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_lock(&rsp->onoff_mutex); 1632765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney smp_mb__after_unlock_lock(); /* ->gpnum increment before GP! */ 16337fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 16347fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney /* 16357fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * Set the quiescent-state-needed bits in all the rcu_node 16367fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * structures for all currently online CPUs in breadth-first order, 16377fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * starting from the root rcu_node structure, relying on the layout 16387fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * of the tree within the rsp->node[] array. Note that other CPUs 16397fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * will access only the leaves of the hierarchy, thus seeing that no 16407fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * grace period is in progress, at least until the corresponding 16417fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * leaf node has been initialized. In addition, we have excluded 16427fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * CPU-hotplug operations. 16437fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * 16447fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * The grace period cannot complete until the initialization 16457fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * process finishes, because this kthread handles both. 16467fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney */ 16477fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rcu_for_each_node_breadth_first(rsp, rnp) { 1648b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney raw_spin_lock_irq(&rnp->lock); 16496303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 1650b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney rdp = this_cpu_ptr(rsp->rda); 16517fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rcu_preempt_check_blocked_tasks(rnp); 16527fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rnp->qsmask = rnp->qsmaskinit; 16530446be489795d8bb994125a916ef03211f539e54Paul E. McKenney ACCESS_ONCE(rnp->gpnum) = rsp->gpnum; 165425d30cf4250f74e5ceb35f8f39739782408db633Paul E. McKenney WARN_ON_ONCE(rnp->completed != rsp->completed); 16550446be489795d8bb994125a916ef03211f539e54Paul E. McKenney ACCESS_ONCE(rnp->completed) = rsp->completed; 16567fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney if (rnp == rdp->mynode) 165748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney (void)__note_gp_changes(rsp, rnp, rdp); 16587fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rcu_preempt_boost_start_gp(rnp); 16597fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney trace_rcu_grace_period_init(rsp->name, rnp->gpnum, 16607fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rnp->level, rnp->grplo, 16617fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rnp->grphi, rnp->qsmask); 16627fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 1663bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 16647fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney } 1665b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 1666a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_unlock(&rsp->onoff_mutex); 16677fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney return 1; 16687fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney} 1669b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 16707fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney/* 16714cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney * Do one round of quiescent-state forcing. 16724cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney */ 167301896f7e0a122e8f20082e24f6f9a340034b9c01Paul E. McKenneystatic int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in) 16744cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney{ 16754cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney int fqs_state = fqs_state_in; 1676217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney bool isidle = false; 1677217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney unsigned long maxj; 16784cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 16794cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney 16804cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney rsp->n_force_qs++; 16814cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney if (fqs_state == RCU_SAVE_DYNTICK) { 16824cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Collect dyntick-idle snapshots. */ 16830edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney if (is_sysidle_rcu_state(rsp)) { 1684e02b2edfa13878c6671d31d5c736f56f89d99bf1Pranith Kumar isidle = true; 16850edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney maxj = jiffies - ULONG_MAX / 4; 16860edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney } 1687217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney force_qs_rnp(rsp, dyntick_save_progress_counter, 1688217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney &isidle, &maxj); 16890edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney rcu_sysidle_report_gp(rsp, isidle, maxj); 16904cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney fqs_state = RCU_FORCE_QS; 16914cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } else { 16924cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Handle dyntick-idle and offline CPUs. */ 1693e02b2edfa13878c6671d31d5c736f56f89d99bf1Pranith Kumar isidle = false; 1694217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj); 16954cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } 16964cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Clear flag to prevent immediate re-entry. */ 16974cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { 16984cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney raw_spin_lock_irq(&rnp->lock); 16996303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 17004de376a1b14e32f550931274f06b571abc0f3d4bPranith Kumar ACCESS_ONCE(rsp->gp_flags) = 17014de376a1b14e32f550931274f06b571abc0f3d4bPranith Kumar ACCESS_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS; 17024cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 17034cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } 17044cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney return fqs_state; 17054cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney} 17064cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney 17074cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney/* 17087fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * Clean up after the old grace period. 17097fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney */ 17104cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenneystatic void rcu_gp_cleanup(struct rcu_state *rsp) 17117fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney{ 17127fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney unsigned long gp_duration; 171348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool needgp = false; 1714dae6e64d2bcfd4b06304ab864c7e3a4f6b5fedf4Paul E. McKenney int nocb = 0; 17157fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney struct rcu_data *rdp; 17167fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 1717b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 17187fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_lock_irq(&rnp->lock); 17196303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 17207fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney gp_duration = jiffies - rsp->gp_start; 17217fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney if (gp_duration > rsp->gp_max) 17227fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rsp->gp_max = gp_duration; 1723b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 17247fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney /* 17257fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * We know the grace period is complete, but to everyone else 17267fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * it appears to still be ongoing. But it is also the case 17277fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * that to everyone else it looks like there is nothing that 17287fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * they can do to advance the grace period. It is therefore 17297fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * safe for us to drop the lock in order to mark the grace 17307fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * period as completed in all of the rcu_node structures. 17317fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney */ 17325d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney raw_spin_unlock_irq(&rnp->lock); 1733b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 17345d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney /* 17355d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * Propagate new ->completed value to rcu_node structures so 17365d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * that other CPUs don't have to wait until the start of the next 17375d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * grace period to process their callbacks. This also avoids 17385d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * some nasty RCU grace-period initialization races by forcing 17395d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * the end of the current grace period to be completely recorded in 17405d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * all of the rcu_node structures before the beginning of the next 17415d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney * grace period is recorded in any of the rcu_node structures. 17425d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney */ 17435d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney rcu_for_each_node_breadth_first(rsp, rnp) { 1744755609a9087fa983f567dc5452b2fa7b089b591fPaul E. McKenney raw_spin_lock_irq(&rnp->lock); 17456303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 17460446be489795d8bb994125a916ef03211f539e54Paul E. McKenney ACCESS_ONCE(rnp->completed) = rsp->gpnum; 1747b11cc5760a9c48c870ad286e8a6d8fdb998fa58dPaul E. McKenney rdp = this_cpu_ptr(rsp->rda); 1748b11cc5760a9c48c870ad286e8a6d8fdb998fa58dPaul E. McKenney if (rnp == rdp->mynode) 174948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; 175078e4bc34e5d966cfd95f1238565afc399d56225cPaul E. McKenney /* smp_mb() provided by prior unlock-lock pair. */ 17510446be489795d8bb994125a916ef03211f539e54Paul E. McKenney nocb += rcu_future_gp_cleanup(rsp, rnp); 17525d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney raw_spin_unlock_irq(&rnp->lock); 1753bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 17547fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney } 17555d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney rnp = rcu_get_root(rsp); 17565d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney raw_spin_lock_irq(&rnp->lock); 1757765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney smp_mb__after_unlock_lock(); /* Order GP before ->completed update. */ 1758dae6e64d2bcfd4b06304ab864c7e3a4f6b5fedf4Paul E. McKenney rcu_nocb_gp_set(rnp, nocb); 17597fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 1760765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney /* Declare grace period done. */ 1761765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney ACCESS_ONCE(rsp->completed) = rsp->gpnum; 1762f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); 17637fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney rsp->fqs_state = RCU_GP_IDLE; 17645d4b86594984d8746b01487c768d8548463c173fPaul E. McKenney rdp = this_cpu_ptr(rsp->rda); 176548a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney /* Advance CBs to reduce false positives below. */ 176648a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; 176748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (needgp || cpu_needs_another_gp(rsp, rdp)) { 176891dc95427a0d30ac2c58d6e943c7f40a3f25d908Paul E. McKenney ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; 1769bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney trace_rcu_grace_period(rsp->name, 1770bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 1771bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney TPS("newreq")); 1772bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney } 17737fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney raw_spin_unlock_irq(&rnp->lock); 17747fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney} 17757fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 17767fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney/* 17777fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney * Body of kthread that handles grace periods. 17787fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney */ 17797fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenneystatic int __noreturn rcu_gp_kthread(void *arg) 17807fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney{ 17814cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney int fqs_state; 178288d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney int gf; 1783d40011f601b450396104de42c631981502946cf0Paul E. McKenney unsigned long j; 17844cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney int ret; 17857fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney struct rcu_state *rsp = arg; 17867fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 17877fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 17887fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney for (;;) { 17897fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney 17907fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney /* Handle grace-period start. */ 17917fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney for (;;) { 179263c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 179363c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 179463c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("reqwait")); 1795afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney rsp->gp_state = RCU_GP_WAIT_GPS; 17964cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney wait_event_interruptible(rsp->gp_wq, 1797591c6d1710cd73824057d08eda302cf2a7cfd18aPaul E. McKenney ACCESS_ONCE(rsp->gp_flags) & 17984cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney RCU_GP_FLAG_INIT); 179978e4bc34e5d966cfd95f1238565afc399d56225cPaul E. McKenney /* Locking provides needed memory barrier. */ 1800f7be82093952ee4a74ffc8c729b2811f908cd9a4Paul E. McKenney if (rcu_gp_init(rsp)) 18017fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney break; 1802bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 180373a860cd58a1eb258e889b615cebf738ab33aa23Paul E. McKenney WARN_ON(signal_pending(current)); 180463c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 180563c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 180663c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("reqwaitsig")); 18077fdefc10e1d730d4608cc59d386bc446f5b9ee99Paul E. McKenney } 1808cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney 18094cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Handle quiescent-state forcing. */ 18104cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney fqs_state = RCU_SAVE_DYNTICK; 1811d40011f601b450396104de42c631981502946cf0Paul E. McKenney j = jiffies_till_first_fqs; 1812d40011f601b450396104de42c631981502946cf0Paul E. McKenney if (j > HZ) { 1813d40011f601b450396104de42c631981502946cf0Paul E. McKenney j = HZ; 1814d40011f601b450396104de42c631981502946cf0Paul E. McKenney jiffies_till_first_fqs = HZ; 1815d40011f601b450396104de42c631981502946cf0Paul E. McKenney } 181688d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney ret = 0; 1817cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney for (;;) { 181888d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney if (!ret) 181988d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney rsp->jiffies_force_qs = jiffies + j; 182063c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 182163c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 182263c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("fqswait")); 1823afea227fd4acf4f097a9e77bbc2f07d4856ebd01Paul E. McKenney rsp->gp_state = RCU_GP_WAIT_FQS; 18244cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney ret = wait_event_interruptible_timeout(rsp->gp_wq, 182588d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney ((gf = ACCESS_ONCE(rsp->gp_flags)) & 182688d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney RCU_GP_FLAG_FQS) || 18274cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney (!ACCESS_ONCE(rnp->qsmask) && 18284cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney !rcu_preempt_blocked_readers_cgp(rnp)), 1829d40011f601b450396104de42c631981502946cf0Paul E. McKenney j); 183078e4bc34e5d966cfd95f1238565afc399d56225cPaul E. McKenney /* Locking provides needed memory barriers. */ 18314cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* If grace period done, leave loop. */ 1832cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney if (!ACCESS_ONCE(rnp->qsmask) && 18334cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney !rcu_preempt_blocked_readers_cgp(rnp)) 1834cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney break; 18354cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* If time for quiescent-state forcing, do it. */ 183688d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) || 183788d6df612cc3c99f56cc18461fcc531c3a145544Paul E. McKenney (gf & RCU_GP_FLAG_FQS)) { 183863c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 183963c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 184063c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("fqsstart")); 18414cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney fqs_state = rcu_gp_fqs(rsp, fqs_state); 184263c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 184363c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 184463c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("fqsend")); 1845bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 18464cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } else { 18474cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Deal with stray signal. */ 1848bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 184973a860cd58a1eb258e889b615cebf738ab33aa23Paul E. McKenney WARN_ON(signal_pending(current)); 185063c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney trace_rcu_grace_period(rsp->name, 185163c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney ACCESS_ONCE(rsp->gpnum), 185263c4db78e80407976e47bccaa2a4d8251b5a10bcPaul E. McKenney TPS("fqswaitsig")); 18534cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } 1854d40011f601b450396104de42c631981502946cf0Paul E. McKenney j = jiffies_till_next_fqs; 1855d40011f601b450396104de42c631981502946cf0Paul E. McKenney if (j > HZ) { 1856d40011f601b450396104de42c631981502946cf0Paul E. McKenney j = HZ; 1857d40011f601b450396104de42c631981502946cf0Paul E. McKenney jiffies_till_next_fqs = HZ; 1858d40011f601b450396104de42c631981502946cf0Paul E. McKenney } else if (j < 1) { 1859d40011f601b450396104de42c631981502946cf0Paul E. McKenney j = 1; 1860d40011f601b450396104de42c631981502946cf0Paul E. McKenney jiffies_till_next_fqs = 1; 1861d40011f601b450396104de42c631981502946cf0Paul E. McKenney } 1862cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney } 18634cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney 18644cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney /* Handle grace-period end. */ 18654cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney rcu_gp_cleanup(rsp); 1866b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney } 1867b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney} 1868b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 1869b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney/* 187064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Start a new RCU grace period if warranted, re-initializing the hierarchy 187164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * in preparation for detecting the next grace period. The caller must hold 1872b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney * the root node's ->lock and hard irqs must be disabled. 1873e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney * 1874e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney * Note that it is legal for a dying CPU (which is marked as offline) to 1875e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney * invoke this function. This can happen when the dying CPU reports its 1876e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney * quiescent state. 187748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * 187848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Returns true if the grace-period kthread must be awakened. 187964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 188048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool 1881910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenneyrcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, 1882910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney struct rcu_data *rdp) 188364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 1884b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) { 1885afe24b122eb6edb5f1cb942570ac8d766105c7fcPaul E. McKenney /* 1886b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney * Either we have not yet spawned the grace-period 188762da1921292ef789c23a7bf01d671d7572baf377Paul E. McKenney * task, this CPU does not need another grace period, 188862da1921292ef789c23a7bf01d671d7572baf377Paul E. McKenney * or a grace period is already in progress. 1889b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney * Either way, don't start a new grace period. 1890afe24b122eb6edb5f1cb942570ac8d766105c7fcPaul E. McKenney */ 189148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return false; 1892afe24b122eb6edb5f1cb942570ac8d766105c7fcPaul E. McKenney } 189391dc95427a0d30ac2c58d6e943c7f40a3f25d908Paul E. McKenney ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; 1894bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum), 1895bb311eccbdab974639263060b8452bf304af0b0cPaul E. McKenney TPS("newreq")); 189662da1921292ef789c23a7bf01d671d7572baf377Paul E. McKenney 1897016a8d5be6ddcc72ef0432d82d9f6fa34f61b907Steven Rostedt /* 1898016a8d5be6ddcc72ef0432d82d9f6fa34f61b907Steven Rostedt * We can't do wakeups while holding the rnp->lock, as that 18991eafd31c640d6799c63136246a59d608bed93c74Paul E. McKenney * could cause possible deadlocks with the rq->lock. Defer 190048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * the wakeup to our caller. 1901016a8d5be6ddcc72ef0432d82d9f6fa34f61b907Steven Rostedt */ 190248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return true; 190364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 190464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 190564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 1906910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's 1907910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * callbacks. Note that rcu_start_gp_advanced() cannot do this because it 1908910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * is invoked indirectly from rcu_advance_cbs(), which would result in 1909910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * endless recursion -- or would do so if it wasn't for the self-deadlock 1910910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * that is encountered beforehand. 191148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * 191248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney * Returns true if the grace-period kthread needs to be awakened. 1913910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney */ 191448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenneystatic bool rcu_start_gp(struct rcu_state *rsp) 1915910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney{ 1916910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney struct rcu_data *rdp = this_cpu_ptr(rsp->rda); 1917910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 191848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool ret = false; 1919910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney 1920910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney /* 1921910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * If there is no grace period in progress right now, any 1922910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * callbacks we have up to this point will be satisfied by the 1923910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * next grace period. Also, advancing the callbacks reduces the 1924910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * probability of false positives from cpu_needs_another_gp() 1925910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * resulting in pointless grace periods. So, advance callbacks 1926910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney * then start the grace period! 1927910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney */ 192848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_advance_cbs(rsp, rnp, rdp) || ret; 192948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret; 193048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney return ret; 1931910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney} 1932910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney 1933910ee45db2f4837c8440e770474758493ab94bf7Paul E. McKenney/* 1934d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * Report a full set of quiescent states to the specified rcu_state 1935d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * data structure. This involves cleaning up after the prior grace 1936d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * period and letting rcu_start_gp() start up the next grace period 1937b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney * if one is needed. Note that the caller must hold rnp->lock, which 1938b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney * is released before return. 1939f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney */ 1940d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenneystatic void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags) 1941fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney __releases(rcu_get_root(rsp)->lock) 1942f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney{ 1943fc2219d49ef1606e7fd2c88af2b423b01ff3d319Paul E. McKenney WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); 1944cabc49c1ff51baaf1958d501a7a616ce91245c93Paul E. McKenney raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags); 19452aa792e6faf1a00f5accf1f69e87e11a390ba2cdPranith Kumar rcu_gp_kthread_wake(rsp); 1946f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney} 1947f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney 1948f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney/* 1949d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * Similar to rcu_report_qs_rdp(), for which it is a helper function. 1950d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * Allows quiescent states for a group of CPUs to be reported at one go 1951d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * to the specified rcu_node structure, though all the CPUs in the group 1952d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * must be represented by the same rcu_node structure (which need not be 1953d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * a leaf rcu_node structure, though it often will be). That structure's 1954d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * lock must be held upon entry, and it is released before return. 195564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 195664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void 1957d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenneyrcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, 1958d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney struct rcu_node *rnp, unsigned long flags) 195964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney __releases(rnp->lock) 196064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 196128ecd58020409be8eb176c716f957fc3386fa2faPaul E. McKenney struct rcu_node *rnp_c; 196228ecd58020409be8eb176c716f957fc3386fa2faPaul E. McKenney 196364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Walk up the rcu_node hierarchy. */ 196464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney for (;;) { 196564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (!(rnp->qsmask & mask)) { 196664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 196764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Our bit has already been cleared, so done. */ 19681304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 196964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 197064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 197164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->qsmask &= ~mask; 1972d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum, 1973d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney mask, rnp->qsmask, rnp->level, 1974d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney rnp->grplo, rnp->grphi, 1975d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney !!rnp->gp_tasks); 197627f4d28057adf98750cf863c40baefb12f5b6d21Paul E. McKenney if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) { 197764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 197864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Other bits still set at this level, so done. */ 19791304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 198064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 198164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 198264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney mask = rnp->grpmask; 198364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (rnp->parent == NULL) { 198464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 198564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* No more levels. Exit loop holding root lock. */ 198664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 198764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 198864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 19891304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 199028ecd58020409be8eb176c716f957fc3386fa2faPaul E. McKenney rnp_c = rnp; 199164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rnp->parent; 19921304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 19936303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 199428ecd58020409be8eb176c716f957fc3386fa2faPaul E. McKenney WARN_ON_ONCE(rnp_c->qsmask); 199564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 199664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 199764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 199864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Get here if we are the last CPU to pass through a quiescent 1999d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * state for this grace period. Invoke rcu_report_qs_rsp() 2000f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney * to clean up and start the next grace period if one is needed. 200164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2002d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */ 200364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 200464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 200564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2006d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * Record a quiescent state for the specified CPU to that CPU's rcu_data 2007d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * structure. This must be either called from the specified CPU, or 2008d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * called when the specified CPU is known to be offline (and when it is 2009d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * also known that no other CPU is concurrently trying to help the offline 2010d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * CPU). The lastcomp argument is used to make sure we are still in the 2011d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * grace period of interest. We don't want to end the current grace period 2012d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * based on quiescent states detected in an earlier grace period! 201364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 201464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void 2015d7d6a11e8609f0319d4a2d8ede348f8b3374b652Paul E. McKenneyrcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) 201664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 201764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 201864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long mask; 201948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool needwake; 202064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp; 202164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 202264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rdp->mynode; 20231304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 20246303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 2025d7d6a11e8609f0319d4a2d8ede348f8b3374b652Paul E. McKenney if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum || 2026d7d6a11e8609f0319d4a2d8ede348f8b3374b652Paul E. McKenney rnp->completed == rnp->gpnum) { 202764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 202864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 2029e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * The grace period in which this quiescent state was 2030e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * recorded has ended, so don't report it upwards. 2031e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * We will instead need a new quiescent state that lies 2032e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney * within the current grace period. 203364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2034e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney rdp->passed_quiesce = 0; /* need qs for new gp. */ 20351304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 203664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 203764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 203864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney mask = rdp->grpmask; 203964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if ((rnp->qsmask & mask) == 0) { 20401304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 204164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } else { 204264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->qs_pending = 0; 204364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 204464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 204564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * This GP can't end until cpu checks in, so all of our 204664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * callbacks can be processed during the next GP. 204764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 204848a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needwake = rcu_accelerate_cbs(rsp, rnp, rdp); 204964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2050d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney rcu_report_qs_rnp(mask, rsp, rnp, flags); /* rlses rnp->lock */ 205148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (needwake) 205248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney rcu_gp_kthread_wake(rsp); 205364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 205464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 205564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 205664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 205764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Check to see if there is a new grace period of which this CPU 205864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * is not yet aware, and if so, set up local rcu_data state for it. 205964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Otherwise, see if this CPU has just passed through its first 206064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * quiescent state for this grace period, and record that fact if so. 206164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 206264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void 206364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyrcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 206464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 206505eb552bf5ed9e7277bdc9c273ed2f4e9b7dc3e5Paul E. McKenney /* Check for grace-period ends and beginnings. */ 206605eb552bf5ed9e7277bdc9c273ed2f4e9b7dc3e5Paul E. McKenney note_gp_changes(rsp, rdp); 206764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 206864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 206964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Does this CPU still need to do its part for current grace period? 207064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * If no, return and let the other CPUs do their part as well. 207164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 207264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (!rdp->qs_pending) 207364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 207464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 207564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 207664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Was there a quiescent state since the beginning of the grace 207764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * period? If no, then exit and wait for the next call. 207864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2079e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney if (!rdp->passed_quiesce) 208064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 208164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2082d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney /* 2083d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * Tell RCU we are done (but rcu_report_qs_rdp() will be the 2084d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney * judge of that). 2085d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney */ 2086d7d6a11e8609f0319d4a2d8ede348f8b3374b652Paul E. McKenney rcu_report_qs_rdp(rdp->cpu, rsp, rdp); 208764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 208864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 208964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#ifdef CONFIG_HOTPLUG_CPU 209064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 209164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2092b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Send the specified CPU's RCU callbacks to the orphanage. The 2093b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * specified CPU must be offline, and the caller must hold the 20947b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney * ->orphan_lock. 2095e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney */ 2096b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenneystatic void 2097b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenneyrcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp, 2098b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney struct rcu_node *rnp, struct rcu_data *rdp) 2099e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney{ 21003fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney /* No-CBs CPUs do not have orphanable callbacks. */ 2101d1e43fa5f8bb25f83a86a29f11fcfb57ed4d7566Frederic Weisbecker if (rcu_is_nocb_cpu(rdp->cpu)) 21023fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney return; 21033fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney 2104b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* 2105b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Orphan the callbacks. First adjust the counts. This is safe 2106abfd6e58aed4f89fd69b9b17bc4b4527efe3a645Paul E. McKenney * because _rcu_barrier() excludes CPU-hotplug operations, so it 2107abfd6e58aed4f89fd69b9b17bc4b4527efe3a645Paul E. McKenney * cannot be running now. Thus no memory barrier is required. 2108b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 2109a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney if (rdp->nxtlist != NULL) { 2110b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->qlen_lazy += rdp->qlen_lazy; 2111b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->qlen += rdp->qlen; 2112b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->n_cbs_orphaned += rdp->qlen; 2113a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney rdp->qlen_lazy = 0; 21141d1fb395f6dbc07b36285bbedcf01a73b57f7cb5Paul E. McKenney ACCESS_ONCE(rdp->qlen) = 0; 2115a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney } 2116a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney 2117a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney /* 2118b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Next, move those callbacks still needing a grace period to 2119b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * the orphanage, where some other CPU will pick them up. 2120b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Some of the callbacks might have gone partway through a grace 2121b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * period, but that is too bad. They get to start over because we 2122b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * cannot assume that grace periods are synchronized across CPUs. 2123b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * We don't bother updating the ->nxttail[] array yet, instead 2124b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * we just reset the whole thing later on. 2125a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney */ 2126b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney if (*rdp->nxttail[RCU_DONE_TAIL] != NULL) { 2127b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rsp->orphan_nxttail = *rdp->nxttail[RCU_DONE_TAIL]; 2128b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_nxttail = rdp->nxttail[RCU_NEXT_TAIL]; 2129b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rdp->nxttail[RCU_DONE_TAIL] = NULL; 2130a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney } 2131a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney 2132a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney /* 2133b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Then move the ready-to-invoke callbacks to the orphanage, 2134b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * where some other CPU will pick them up. These will not be 2135b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * required to pass though another grace period: They are done. 2136a50c3af910e06f35bc0c68f89d8fef98c0fec0eaPaul E. McKenney */ 2137e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney if (rdp->nxtlist != NULL) { 2138b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rsp->orphan_donetail = rdp->nxtlist; 2139b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL]; 2140e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney } 2141e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney 2142b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* Finally, initialize the rcu_data structure's list to empty. */ 21433f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney init_callback_list(rdp); 2144b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney} 2145b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2146b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney/* 2147b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Adopt the RCU callbacks from the specified rcu_state structure's 21487b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney * orphanage. The caller must hold the ->orphan_lock. 2149b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 215096d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenneystatic void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags) 2151b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney{ 2152b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney int i; 2153fa07a58f71ee23a82597ce337126982d0cc60b72Christoph Lameter struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); 2154b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 21553fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney /* No-CBs CPUs are handled specially. */ 215696d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney if (rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags)) 21573fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney return; 21583fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney 2159b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* Do the accounting first. */ 2160b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->qlen_lazy += rsp->qlen_lazy; 2161b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->qlen += rsp->qlen; 2162b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->n_cbs_adopted += rsp->qlen; 21638f5af6f1f2d09fe5eac86a5dc1731a5917c1503aPaul E. McKenney if (rsp->qlen_lazy != rsp->qlen) 21648f5af6f1f2d09fe5eac86a5dc1731a5917c1503aPaul E. McKenney rcu_idle_count_callbacks_posted(); 2165b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->qlen_lazy = 0; 2166b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->qlen = 0; 2167b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2168b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* 2169b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * We do not need a memory barrier here because the only way we 2170b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * can get here if there is an rcu_barrier() in flight is if 2171b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * we are the task doing the rcu_barrier(). 2172b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 2173b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2174b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* First adopt the ready-to-invoke callbacks. */ 2175b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney if (rsp->orphan_donelist != NULL) { 2176b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rsp->orphan_donetail = *rdp->nxttail[RCU_DONE_TAIL]; 2177b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rdp->nxttail[RCU_DONE_TAIL] = rsp->orphan_donelist; 2178b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney for (i = RCU_NEXT_SIZE - 1; i >= RCU_DONE_TAIL; i--) 2179b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) 2180b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->nxttail[i] = rsp->orphan_donetail; 2181b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_donelist = NULL; 2182b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_donetail = &rsp->orphan_donelist; 2183b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney } 2184b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2185b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* And then adopt the callbacks that still need a grace period. */ 2186b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney if (rsp->orphan_nxtlist != NULL) { 2187b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxtlist; 2188b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxttail; 2189b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_nxtlist = NULL; 2190b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rsp->orphan_nxttail = &rsp->orphan_nxtlist; 2191b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney } 2192b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney} 2193b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2194b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney/* 2195b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Trace the fact that this CPU is going offline. 2196b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 2197b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenneystatic void rcu_cleanup_dying_cpu(struct rcu_state *rsp) 2198b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney{ 2199b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney RCU_TRACE(unsigned long mask); 2200b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda)); 2201b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney RCU_TRACE(struct rcu_node *rnp = rdp->mynode); 2202b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 2203b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney RCU_TRACE(mask = rdp->grpmask); 2204e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney trace_rcu_grace_period(rsp->name, 2205e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney rnp->gpnum + 1 - !!(rnp->qsmask & mask), 2206f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) TPS("cpuofl")); 220764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 220864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 220964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2210e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney * The CPU has been completely removed, and some other CPU is reporting 2211b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * this fact from process context. Do the remainder of the cleanup, 2212b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * including orphaning the outgoing CPU's RCU callbacks, and also 22131331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * adopting them. There can only be one CPU hotplug operation at a time, 22141331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * so no other CPU can be attempting to update rcu_cpu_kthread_task. 221564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2216e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenneystatic void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) 221764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 22182036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney unsigned long flags; 22192036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney unsigned long mask; 22202036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney int need_report = 0; 2221e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); 2222b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ 2223e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenney 22242036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney /* Adjust any no-longer-needed kthreads. */ 22255d01bbd111d6ff9ea9d9847774f66dff39633776Thomas Gleixner rcu_boost_kthread_setaffinity(rnp, -1); 22262036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney 22272036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney /* Exclude any attempts to start a new grace period. */ 2228a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_lock(&rsp->onoff_mutex); 22297b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney raw_spin_lock_irqsave(&rsp->orphan_lock, flags); 22302036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney 2231b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */ 2232b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp); 223396d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney rcu_adopt_orphan_cbs(rsp, flags); 2234b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 22352036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */ 22362036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney mask = rdp->grpmask; /* rnp->grplo is constant. */ 22372036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney do { 22382036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney raw_spin_lock(&rnp->lock); /* irqs already disabled. */ 22396303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 22402036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rnp->qsmaskinit &= ~mask; 22412036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney if (rnp->qsmaskinit != 0) { 22422036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney if (rnp != rdp->mynode) 22432036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ 22442036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney break; 22452036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney } 22462036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney if (rnp == rdp->mynode) 22472036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp); 22482036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney else 22492036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ 22502036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney mask = rnp->grpmask; 22512036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rnp = rnp->parent; 22522036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney } while (rnp != NULL); 22532036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney 22542036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney /* 22552036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * We still hold the leaf rcu_node structure lock here, and 22562036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * irqs are still disabled. The reason for this subterfuge is 22577b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney * because invoking rcu_report_unblock_qs_rnp() with ->orphan_lock 22582036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney * held leads to deadlock. 22592036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney */ 22607b2e6011f150c42235c4a541d20cf6891afe878aPaul E. McKenney raw_spin_unlock(&rsp->orphan_lock); /* irqs remain disabled. */ 22612036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rnp = rdp->mynode; 22622036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney if (need_report & RCU_OFL_TASKS_NORM_GP) 22632036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rcu_report_unblock_qs_rnp(rnp, flags); 22642036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney else 22652036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 22662036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney if (need_report & RCU_OFL_TASKS_EXP_GP) 22672036d94a7b61ca5032ce90f2bda06afec0fe713ePaul E. McKenney rcu_report_exp_rnp(rsp, rnp, true); 2268cf01537ecf192d9ff15c32a355db5d5af22bea4dPaul E. McKenney WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL, 2269cf01537ecf192d9ff15c32a355db5d5af22bea4dPaul E. McKenney "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n", 2270cf01537ecf192d9ff15c32a355db5d5af22bea4dPaul E. McKenney cpu, rdp->qlen, rdp->nxtlist); 22710d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney init_callback_list(rdp); 22720d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney /* Disallow further callbacks on this CPU. */ 22730d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney rdp->nxttail[RCU_NEXT_TAIL] = NULL; 2274a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_unlock(&rsp->onoff_mutex); 227564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 227664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 227764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#else /* #ifdef CONFIG_HOTPLUG_CPU */ 227864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2279e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenneystatic void rcu_cleanup_dying_cpu(struct rcu_state *rsp) 2280e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney{ 2281e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney} 2282e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney 2283e5601400081651060a59bd1f45f2821bb8e97f95Paul E. McKenneystatic void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) 228464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 228564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 228664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 228764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 228864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 228964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 229064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Invoke any RCU callbacks that have made it to the end of their grace 229164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * period. Thottle as specified by rdp->blimit. 229264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 229337c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenneystatic void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) 229464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 229564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 229664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_head *next, *list, **tail; 2297878d7439d0f45a95869e417576774673d1fa243fEric Dumazet long bl, count, count_lazy; 2298878d7439d0f45a95869e417576774673d1fa243fEric Dumazet int i; 229964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2300dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney /* If no callbacks are ready, just return. */ 230129c00b4a1d9e277786120032aa8364631820d863Paul E. McKenney if (!cpu_has_callbacks_ready_to_invoke(rdp)) { 2302486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0); 23034968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist), 23044968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney need_resched(), is_idle_task(current), 23054968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney rcu_is_callbacks_kthread()); 230664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return; 230729c00b4a1d9e277786120032aa8364631820d863Paul E. McKenney } 230864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 230964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 231064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Extract the list of ready callbacks, disabling to prevent 231164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * races with call_rcu() from interrupt handlers. 231264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 231364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_save(flags); 23148146c4e2e2c1972216afece5c50e072e86120e42Paul E. McKenney WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 231529c00b4a1d9e277786120032aa8364631820d863Paul E. McKenney bl = rdp->blimit; 2316486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl); 231764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney list = rdp->nxtlist; 231864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL]; 231964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney *rdp->nxttail[RCU_DONE_TAIL] = NULL; 232064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney tail = rdp->nxttail[RCU_DONE_TAIL]; 2321b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney for (i = RCU_NEXT_SIZE - 1; i >= 0; i--) 2322b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) 2323b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney rdp->nxttail[i] = &rdp->nxtlist; 232464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_restore(flags); 232564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 232664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Invoke callbacks. */ 2327486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney count = count_lazy = 0; 232864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney while (list) { 232964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney next = list->next; 233064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney prefetch(next); 2331551d55a944b143ef26fbd482d1c463199d6f65cfMathieu Desnoyers debug_rcu_head_unqueue(list); 2332486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney if (__rcu_reclaim(rsp->name, list)) 2333486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney count_lazy++; 233464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney list = next; 2335dff1672d9199fffddb58fa7970ccf59005fc35f3Paul E. McKenney /* Stop only if limit reached and CPU has something to do. */ 2336dff1672d9199fffddb58fa7970ccf59005fc35f3Paul E. McKenney if (++count >= bl && 2337dff1672d9199fffddb58fa7970ccf59005fc35f3Paul E. McKenney (need_resched() || 2338dff1672d9199fffddb58fa7970ccf59005fc35f3Paul E. McKenney (!is_idle_task(current) && !rcu_is_callbacks_kthread()))) 233964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 234064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 234164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 234264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_save(flags); 23434968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney trace_rcu_batch_end(rsp->name, count, !!list, need_resched(), 23444968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney is_idle_task(current), 23454968c300e1fa5389fdf1f1ebd8b8e4aec9aa4a9ePaul E. McKenney rcu_is_callbacks_kthread()); 234664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 234764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Update count, and requeue any remaining callbacks. */ 234864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (list != NULL) { 234964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney *tail = rdp->nxtlist; 235064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->nxtlist = list; 2351b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney for (i = 0; i < RCU_NEXT_SIZE; i++) 2352b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney if (&rdp->nxtlist == rdp->nxttail[i]) 2353b41772abebc27c61dd578b76da99aa5240b4c99aPaul E. McKenney rdp->nxttail[i] = tail; 235464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney else 235564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 235664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 2357b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney smp_mb(); /* List handling before counting for rcu_barrier(). */ 2358b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->qlen_lazy -= count_lazy; 2359a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney ACCESS_ONCE(rdp->qlen) = rdp->qlen - count; 2360b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->n_cbs_invoked += count; 236164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 236264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Reinstate batch limit if we have worked down the excess. */ 236364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark) 236464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->blimit = blimit; 236564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 236637c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */ 236737c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney if (rdp->qlen == 0 && rdp->qlen_last_fqs_check != 0) { 236837c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney rdp->qlen_last_fqs_check = 0; 236937c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney rdp->n_force_qs_snap = rsp->n_force_qs; 237037c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark) 237137c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney rdp->qlen_last_fqs_check = rdp->qlen; 2372cfca927972e31a5b3da49bf641c525732ff3c357Paul E. McKenney WARN_ON_ONCE((rdp->nxtlist == NULL) != (rdp->qlen == 0)); 237337c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney 237464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_restore(flags); 237564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2376e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney /* Re-invoke RCU core processing if there are callbacks remaining. */ 237764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (cpu_has_callbacks_ready_to_invoke(rdp)) 2378a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney invoke_rcu_core(); 237964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 238064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 238164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 238264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Check to see if this CPU is in a non-context-switch quiescent state 238364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * (user mode or idle loop for rcu, non-softirq execution for rcu_bh). 2384e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * Also schedule RCU core processing. 238564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 23869b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney * This function must be called from hardirq context. It is normally 238764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * invoked from the scheduling-clock interrupt. If rcu_pending returns 238864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * false, there is no point in invoking rcu_check_callbacks(). 238964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 239064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyvoid rcu_check_callbacks(int cpu, int user) 239164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 2392f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("Start scheduler-tick")); 2393a858af2875fb291d0f4b0a4419fefbf03c2379c0Paul E. McKenney increment_cpu_stall_ticks(); 23949b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney if (user || rcu_is_cpu_rrupt_from_idle()) { 239564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 239664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 239764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Get here if this CPU took its interrupt from user 239864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * mode or from the idle loop, and if this is not a 239964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * nested interrupt. In this case, the CPU is in 2400d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * a quiescent state, so note it. 240164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * 240264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * No memory barrier is required here because both 2403d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local 2404d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * variables that other CPUs neither access nor modify, 2405d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * at least not while the corresponding CPU is online. 240664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 240764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2408284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney rcu_sched_qs(); 2409284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney rcu_bh_qs(); 241064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 241164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } else if (!in_softirq()) { 241264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 241364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 241464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Get here if this CPU did not take its interrupt from 241564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * softirq, in other words, if it is not interrupting 241664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * a rcu_bh read-side critical section. This is an _bh 2417d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * critical section, so note it. 241864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 241964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2420284a8c93af47306beed967a303d84730b32bab39Paul E. McKenney rcu_bh_qs(); 242164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 2422f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney rcu_preempt_check_callbacks(cpu); 2423d21670acab9fcb4bc74a40b68a6941059234c55cPaul E. McKenney if (rcu_pending(cpu)) 2424a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney invoke_rcu_core(); 24258315f42295d2667a7f942f154b73a86fd7cb2227Paul E. McKenney if (user) 24268315f42295d2667a7f942f154b73a86fd7cb2227Paul E. McKenney rcu_note_voluntary_context_switch(current); 2427f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("End scheduler-tick")); 242864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 242964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 243064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 243164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Scan the leaf rcu_node structures, processing dyntick state for any that 243264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * have not yet encountered a quiescent state, using the function specified. 243327f4d28057adf98750cf863c40baefb12f5b6d21Paul E. McKenney * Also initiate boosting for any threads blocked on the root rcu_node. 243427f4d28057adf98750cf863c40baefb12f5b6d21Paul E. McKenney * 2435ee47eb9f4da6f44af965d6d049e77ee8c8a4b822Paul E. McKenney * The caller must have suppressed start of new grace periods. 243664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2437217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenneystatic void force_qs_rnp(struct rcu_state *rsp, 2438217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney int (*f)(struct rcu_data *rsp, bool *isidle, 2439217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney unsigned long *maxj), 2440217af2a2ffbfc1498d1cf3a89fa478b5632df8f7Paul E. McKenney bool *isidle, unsigned long *maxj) 244164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 244264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long bit; 244364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int cpu; 244464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 244564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long mask; 2446a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney struct rcu_node *rnp; 244764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2448a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney rcu_for_each_leaf_node(rsp, rnp) { 2449bde6c3aa993066acb0d6ce32ecabe03b9d5df92dPaul E. McKenney cond_resched_rcu_qs(); 245064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney mask = 0; 24511304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 24526303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 2453ee47eb9f4da6f44af965d6d049e77ee8c8a4b822Paul E. McKenney if (!rcu_gp_in_progress(rsp)) { 24541304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 24550f10dc826646134dce3e5751512b87d30f3903e4Paul E. McKenney return; 245664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 2457a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney if (rnp->qsmask == 0) { 24581217ed1ba5c67393293dfb0f03c353b118dadeb4Paul E. McKenney rcu_initiate_boost(rnp, flags); /* releases rnp->lock */ 245964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney continue; 246064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 2461a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney cpu = rnp->grplo; 246264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney bit = 1; 2463a0b6c9a78c41dc36732d6e1e90f0f2f57b29816fPaul E. McKenney for (; cpu <= rnp->grphi; cpu++, bit <<= 1) { 24640edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney if ((rnp->qsmask & bit) != 0) { 24650edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney if ((rnp->qsmaskinit & bit) != 0) 2466e02b2edfa13878c6671d31d5c736f56f89d99bf1Pranith Kumar *isidle = false; 24670edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj)) 24680edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney mask |= bit; 24690edd1b1784cbdad55aca2c1293be018f53c0ab1dPaul E. McKenney } 247064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 247145f014c52eef022873b19d6a20eb0ec9668f2b09Paul E. McKenney if (mask != 0) { 247264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2473d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney /* rcu_report_qs_rnp() releases rnp->lock. */ 2474d3f6bad3911736e44ba11f3f3f6ac4e8c837fdfcPaul E. McKenney rcu_report_qs_rnp(mask, rsp, rnp, flags); 247564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney continue; 247664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 24771304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 247864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 247927f4d28057adf98750cf863c40baefb12f5b6d21Paul E. McKenney rnp = rcu_get_root(rsp); 24801217ed1ba5c67393293dfb0f03c353b118dadeb4Paul E. McKenney if (rnp->qsmask == 0) { 24811217ed1ba5c67393293dfb0f03c353b118dadeb4Paul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 24826303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 24831217ed1ba5c67393293dfb0f03c353b118dadeb4Paul E. McKenney rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ 24841217ed1ba5c67393293dfb0f03c353b118dadeb4Paul E. McKenney } 248564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 248664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 248764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 248864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Force quiescent states on reluctant CPUs, and also detect which 248964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * CPUs are in dyntick-idle mode. 249064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 24914cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenneystatic void force_quiescent_state(struct rcu_state *rsp) 249264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 249364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 2494394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney bool ret; 2495394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney struct rcu_node *rnp; 2496394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney struct rcu_node *rnp_old = NULL; 2497394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney 2498394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney /* Funnel through hierarchy to reduce memory contention. */ 2499d860d40327dde251d508a234fa00bd0d90fbb656Shan Wei rnp = __this_cpu_read(rsp->rda->mynode); 2500394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney for (; rnp != NULL; rnp = rnp->parent) { 2501394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney ret = (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || 2502394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney !raw_spin_trylock(&rnp->fqslock); 2503394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney if (rnp_old != NULL) 2504394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_unlock(&rnp_old->fqslock); 2505394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney if (ret) { 2506a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney rsp->n_force_qs_lh++; 2507394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney return; 2508394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney } 2509394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney rnp_old = rnp; 2510394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney } 2511394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney /* rnp_old == rcu_get_root(rsp), rnp == NULL. */ 251264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2513394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney /* Reached the root of the rcu_node tree, acquire lock. */ 2514394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_lock_irqsave(&rnp_old->lock, flags); 25156303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 2516394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_unlock(&rnp_old->fqslock); 2517394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { 2518a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney rsp->n_force_qs_lh++; 2519394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_unlock_irqrestore(&rnp_old->lock, flags); 25204cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney return; /* Someone beat us to it. */ 252146a1e34eda805501a8b32f26394faa435149f6d1Paul E. McKenney } 25224de376a1b14e32f550931274f06b571abc0f3d4bPranith Kumar ACCESS_ONCE(rsp->gp_flags) = 25234de376a1b14e32f550931274f06b571abc0f3d4bPranith Kumar ACCESS_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS; 2524394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_unlock_irqrestore(&rnp_old->lock, flags); 25252aa792e6faf1a00f5accf1f69e87e11a390ba2cdPranith Kumar rcu_gp_kthread_wake(rsp); 252664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 252764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 252864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2529e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * This does the RCU core processing work for the specified rcu_state 2530e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * and rcu_data structures. This may be called only from the CPU to 2531e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * whom the rdp belongs. 253264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 253364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void 25341bca8cf1a2c3c6683b12ad28a3e826ca7a834978Paul E. McKenney__rcu_process_callbacks(struct rcu_state *rsp) 253564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 253664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 253748a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool needwake; 2538fa07a58f71ee23a82597ce337126982d0cc60b72Christoph Lameter struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); 253964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 25402e597558086dec36d5c33521a36e0f6b1bc3f3a7Paul E. McKenney WARN_ON_ONCE(rdp->beenonline == 0); 25412e597558086dec36d5c33521a36e0f6b1bc3f3a7Paul E. McKenney 254264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Update RCU state based on any recent quiescent states. */ 254364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rcu_check_quiescent_state(rsp, rdp); 254464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 254564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Does this CPU require a not-yet-started grace period? */ 2546dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney local_irq_save(flags); 254764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (cpu_needs_another_gp(rsp, rdp)) { 2548dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney raw_spin_lock(&rcu_get_root(rsp)->lock); /* irqs disabled. */ 254948a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needwake = rcu_start_gp(rsp); 2550b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags); 255148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (needwake) 255248a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney rcu_gp_kthread_wake(rsp); 2553dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney } else { 2554dc35c8934eba959b690921615fcd987e8bc17e4aPaul E. McKenney local_irq_restore(flags); 255564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 255664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 255764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* If there are callbacks ready, invoke them. */ 255809223371deac67d08ca0b70bd18787920284c967Shaohua Li if (cpu_has_callbacks_ready_to_invoke(rdp)) 2559a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney invoke_rcu_callbacks(rsp, rdp); 256096d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney 256196d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney /* Do any needed deferred wakeups of rcuo kthreads. */ 256296d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney do_nocb_deferred_wakeup(rdp); 256309223371deac67d08ca0b70bd18787920284c967Shaohua Li} 256409223371deac67d08ca0b70bd18787920284c967Shaohua Li 256564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2566e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * Do RCU core processing for the current CPU. 256764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 256809223371deac67d08ca0b70bd18787920284c967Shaohua Listatic void rcu_process_callbacks(struct softirq_action *unused) 256964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 25706ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 25716ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney 2572bfa00b4c4028f39357d16279ff0fddf550241593Paul E. McKenney if (cpu_is_offline(smp_processor_id())) 2573bfa00b4c4028f39357d16279ff0fddf550241593Paul E. McKenney return; 2574f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("Start RCU core")); 25756ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 25766ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney __rcu_process_callbacks(rsp); 2577f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("End RCU core")); 257864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 257964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 2580a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney/* 2581e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * Schedule RCU callback invocation. If the specified type of RCU 2582e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * does not support RCU priority boosting, just do a direct call, 2583e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * otherwise wake up the per-CPU kernel kthread. Note that because we 2584e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * are running on the current CPU with interrupts disabled, the 2585e0f23060adfa3f27beaa7918eff70258b88471b6Paul E. McKenney * rcu_cpu_kthread_task cannot disappear out from under us. 2586a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney */ 2587a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenneystatic void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) 2588a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney{ 2589b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) 2590b0d304172f49061b4ff78f9e2b02719ac69c8a7ePaul E. McKenney return; 2591a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney if (likely(!rsp->boost)) { 2592a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney rcu_do_batch(rsp, rdp); 2593a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney return; 2594a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney } 2595a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenney invoke_rcu_callbacks_kthread(); 2596a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney} 2597a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney 2598a46e0899eec7a3069bcadd45dfba7bf67c6ed016Paul E. McKenneystatic void invoke_rcu_core(void) 259909223371deac67d08ca0b70bd18787920284c967Shaohua Li{ 2600b0f740360efec6e6471547c0548f250bc045a233Paul E. McKenney if (cpu_online(smp_processor_id())) 2601b0f740360efec6e6471547c0548f250bc045a233Paul E. McKenney raise_softirq(RCU_SOFTIRQ); 260209223371deac67d08ca0b70bd18787920284c967Shaohua Li} 260309223371deac67d08ca0b70bd18787920284c967Shaohua Li 260429154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney/* 260529154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney * Handle any core-RCU processing required by a call_rcu() invocation. 260629154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney */ 260729154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenneystatic void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, 260829154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney struct rcu_head *head, unsigned long flags) 260964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 261048a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney bool needwake; 261148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney 261262fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney /* 261362fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney * If called from an extended quiescent state, invoke the RCU 261462fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney * core in order to force a re-evaluation of RCU's idleness. 261562fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney */ 26165c173eb8bcb9c1aa888bd6d14a4cb746f3dd2420Paul E. McKenney if (!rcu_is_watching() && cpu_online(smp_processor_id())) 261762fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney invoke_rcu_core(); 261862fde6edf12b60fddb13a3f0a779c8be0bb7447ePaul E. McKenney 2619a16b7a693430406dc229ab0c6b154f669a2031c5Paul E. McKenney /* If interrupts were disabled or CPU offline, don't invoke RCU core. */ 262029154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id())) 26212655d57ef35aa327a2e58a1c5dc7b65c65003f4ePaul E. McKenney return; 262264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 262337c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney /* 262437c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney * Force the grace period if too many callbacks or too long waiting. 262537c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney * Enforce hysteresis, and don't invoke force_quiescent_state() 262637c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney * if some other CPU has recently done so. Also, don't bother 262737c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney * invoking force_quiescent_state() if the newly enqueued callback 262837c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney * is the only one waiting for a grace period to complete. 262937c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney */ 26302655d57ef35aa327a2e58a1c5dc7b65c65003f4ePaul E. McKenney if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) { 2631b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney 2632b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney /* Are we ignoring a completed grace period? */ 2633470716fc043aba2fea832334e58d5cd5d82288a3Paul E. McKenney note_gp_changes(rsp, rdp); 2634b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney 2635b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney /* Start a new grace period if one not already started. */ 2636b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney if (!rcu_gp_in_progress(rsp)) { 2637b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney struct rcu_node *rnp_root = rcu_get_root(rsp); 2638b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney 2639b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney raw_spin_lock(&rnp_root->lock); 26406303b9c87d52eaedc82968d3ff59c471e7682afcPaul E. McKenney smp_mb__after_unlock_lock(); 264148a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney needwake = rcu_start_gp(rsp); 2642b8462084a2a88a6a0489f9bb7d8b1bb95bc455abPaul E. McKenney raw_spin_unlock(&rnp_root->lock); 264348a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney if (needwake) 264448a7639ce80cf279834d0d44865e49ecd714f37dPaul E. McKenney rcu_gp_kthread_wake(rsp); 2645b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney } else { 2646b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney /* Give the grace period a kick. */ 2647b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney rdp->blimit = LONG_MAX; 2648b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney if (rsp->n_force_qs == rdp->n_force_qs_snap && 2649b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney *rdp->nxttail[RCU_DONE_TAIL] != head) 26504cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney force_quiescent_state(rsp); 2651b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney rdp->n_force_qs_snap = rsp->n_force_qs; 2652b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney rdp->qlen_last_fqs_check = rdp->qlen; 2653b52573d2796274f7f31cfeff7185c320adcd4f12Paul E. McKenney } 26544cdfc175c25c89eedc08460b5e6239c2ec67fcb6Paul E. McKenney } 265529154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney} 265629154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney 26573fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney/* 2658ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney * RCU callback function to leak a callback. 2659ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney */ 2660ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenneystatic void rcu_leak_callback(struct rcu_head *rhp) 2661ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney{ 2662ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney} 2663ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney 2664ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney/* 26653fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney * Helper function for call_rcu() and friends. The cpu argument will 26663fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney * normally be -1, indicating "currently running CPU". It may specify 26673fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier() 26683fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney * is expected to specify a CPU. 26693fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney */ 267064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void 267164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 26723fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney struct rcu_state *rsp, int cpu, bool lazy) 267364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 267464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 267564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_data *rdp; 267664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 26771146edcbef3789228454c4aa42c08ddc2c275990Paul E. McKenney WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */ 2678ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney if (debug_rcu_head_queue(head)) { 2679ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney /* Probable double call_rcu(), so leak the callback. */ 2680ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney ACCESS_ONCE(head->func) = rcu_leak_callback; 2681ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n"); 2682ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney return; 2683ae15018456c44b742d352af323e0b89eae4a6383Paul E. McKenney } 268464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney head->func = func; 268564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney head->next = NULL; 268664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 268764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* 268864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Opportunistically note grace-period endings and beginnings. 268964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Note that we might see a beginning right after we see an 269064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * end, but never vice versa, since this CPU has to pass through 269164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * a quiescent state betweentimes. 269264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 269364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_save(flags); 2694394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan rdp = this_cpu_ptr(rsp->rda); 269564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 269664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Add the callback to our list. */ 26973fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL) || cpu != -1) { 26983fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney int offline; 26993fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney 27003fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney if (cpu != -1) 27013fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney rdp = per_cpu_ptr(rsp->rda, cpu); 270296d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney offline = !__call_rcu_nocb(rdp, head, lazy, flags); 27033fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney WARN_ON_ONCE(offline); 27040d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney /* _call_rcu() is illegal on offline CPU; leak the callback. */ 27050d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney local_irq_restore(flags); 27060d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney return; 27070d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney } 2708a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1; 2709486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney if (lazy) 2710486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney rdp->qlen_lazy++; 2711c57afe80db4e169135eb675acc2d241e26cc064ePaul E. McKenney else 2712c57afe80db4e169135eb675acc2d241e26cc064ePaul E. McKenney rcu_idle_count_callbacks_posted(); 2713b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney smp_mb(); /* Count before adding callback for rcu_barrier(). */ 2714b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney *rdp->nxttail[RCU_NEXT_TAIL] = head; 2715b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp->nxttail[RCU_NEXT_TAIL] = &head->next; 27162655d57ef35aa327a2e58a1c5dc7b65c65003f4ePaul E. McKenney 2717d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney if (__is_kfree_rcu_offset((unsigned long)func)) 2718d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func, 2719486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney rdp->qlen_lazy, rdp->qlen); 2720d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney else 2721486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen); 2722d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney 272329154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney /* Go handle any RCU core processing required. */ 272429154c57e35a191c83b19c61b1935c9f21957662Paul E. McKenney __call_rcu_core(rsp, rdp, head, flags); 272564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney local_irq_restore(flags); 272664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 272764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 272864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2729d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenney * Queue an RCU-sched callback for invocation after a grace period. 273064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 2731d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenneyvoid call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 273264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 27333fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney __call_rcu(head, func, &rcu_sched_state, -1, 0); 273464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 2735d6714c22b43fbcbead7e7b706ff270e15f04a791Paul E. McKenneyEXPORT_SYMBOL_GPL(call_rcu_sched); 273664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 273764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 2738486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney * Queue an RCU callback for invocation after a quicker grace period. 273964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 274064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyvoid call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 274164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 27423fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney __call_rcu(head, func, &rcu_bh_state, -1, 0); 274364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 274464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneyEXPORT_SYMBOL_GPL(call_rcu_bh); 274564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 27466d8133919bac4270883b24328500875a49e71b36Paul E. McKenney/* 2747495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat * Queue an RCU callback for lazy invocation after a grace period. 2748495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat * This will likely be later named something like "call_rcu_lazy()", 2749495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat * but this change will require some way of tagging the lazy RCU 2750495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat * callbacks in the list of pending callbacks. Until then, this 2751495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat * function may only be called from __kfree_rcu(). 2752495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat */ 2753495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernatvoid kfree_call_rcu(struct rcu_head *head, 2754495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat void (*func)(struct rcu_head *rcu)) 2755495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat{ 2756e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma __call_rcu(head, func, rcu_state_p, -1, 1); 2757495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat} 2758495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina BernatEXPORT_SYMBOL_GPL(kfree_call_rcu); 2759495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat 2760495aa969dbaef2e3d28094a2b3c752d069932748Andreea-Cristina Bernat/* 27616d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * Because a context switch is a grace period for RCU-sched and RCU-bh, 27626d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * any blocking grace-period wait automatically implies a grace period 27636d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * if there is only one CPU online at any point time during execution 27646d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to 27656d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * occasionally incorrectly indicate that there are multiple CPUs online 27666d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * when there was in fact only one the whole time, as this just adds 27676d8133919bac4270883b24328500875a49e71b36Paul E. McKenney * some overhead: RCU still operates correctly. 27686d8133919bac4270883b24328500875a49e71b36Paul E. McKenney */ 27696d8133919bac4270883b24328500875a49e71b36Paul E. McKenneystatic inline int rcu_blocking_is_gp(void) 27706d8133919bac4270883b24328500875a49e71b36Paul E. McKenney{ 277195f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney int ret; 277295f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney 27736d8133919bac4270883b24328500875a49e71b36Paul E. McKenney might_sleep(); /* Check for RCU read-side critical section. */ 277495f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney preempt_disable(); 277595f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney ret = num_online_cpus() <= 1; 277695f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney preempt_enable(); 277795f0c1de3e6ed4383cc4b5f52ce4ecfb21026b49Paul E. McKenney return ret; 27786d8133919bac4270883b24328500875a49e71b36Paul E. McKenney} 27796d8133919bac4270883b24328500875a49e71b36Paul E. McKenney 27806ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney/** 27816ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * synchronize_sched - wait until an rcu-sched grace period has elapsed. 27826ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * 27836ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * Control will return to the caller some time after a full rcu-sched 27846ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * grace period has elapsed, in other words after all currently executing 27856ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * rcu-sched read-side critical sections have completed. These read-side 27866ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * critical sections are delimited by rcu_read_lock_sched() and 27876ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), 27886ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * local_irq_disable(), and so on may be used in place of 27896ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * rcu_read_lock_sched(). 27906ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * 27916ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * This means that all preempt_disable code sequences, including NMI and 2792f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * non-threaded hardware-interrupt handlers, in progress on entry will 2793f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * have completed before this primitive returns. However, this does not 2794f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * guarantee that softirq handlers will have completed, since in some 2795f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * kernels, these handlers can run in process context, and can block. 2796f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * 2797f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * Note that this guarantee implies further memory-ordering guarantees. 2798f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * On systems with more than one CPU, when synchronize_sched() returns, 2799f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * each CPU is guaranteed to have executed a full memory barrier since the 2800f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * end of its last RCU-sched read-side critical section whose beginning 2801f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * preceded the call to synchronize_sched(). In addition, each CPU having 2802f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * an RCU read-side critical section that extends beyond the return from 2803f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * synchronize_sched() is guaranteed to have executed a full memory barrier 2804f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * after the beginning of synchronize_sched() and before the beginning of 2805f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * that RCU read-side critical section. Note that these guarantees include 2806f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * CPUs that are offline, idle, or executing in user mode, as well as CPUs 2807f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * that are executing in the kernel. 2808f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * 2809f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * Furthermore, if CPU A invoked synchronize_sched(), which returned 2810f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * to its caller on CPU B, then both CPU A and CPU B are guaranteed 2811f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * to have executed a full memory barrier during the execution of 2812f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but 2813f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * again only if the system has more than one CPU). 28146ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * 28156ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * This primitive provides the guarantees made by the (now removed) 28166ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * synchronize_kernel() API. In contrast, synchronize_rcu() only 28176ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * guarantees that rcu_read_lock() sections will have completed. 28186ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * In "classic RCU", these two guarantees happen to be one and 28196ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * the same, but can differ in realtime RCU implementations. 28206ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney */ 28216ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenneyvoid synchronize_sched(void) 28226ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney{ 2823fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && 2824fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney !lock_is_held(&rcu_lock_map) && 2825fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney !lock_is_held(&rcu_sched_lock_map), 2826fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney "Illegal synchronize_sched() in RCU-sched read-side critical section"); 28276ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney if (rcu_blocking_is_gp()) 28286ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney return; 28293705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen if (rcu_expedited) 28303705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen synchronize_sched_expedited(); 28313705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen else 28323705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen wait_rcu_gp(call_rcu_sched); 28336ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney} 28346ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenneyEXPORT_SYMBOL_GPL(synchronize_sched); 28356ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney 28366ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney/** 28376ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. 28386ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * 28396ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * Control will return to the caller some time after a full rcu_bh grace 28406ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * period has elapsed, in other words after all currently executing rcu_bh 28416ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * read-side critical sections have completed. RCU read-side critical 28426ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), 28436ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney * and may be nested. 2844f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * 2845f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * See the description of synchronize_sched() for more detailed information 2846f0a0e6f282c72247e7c8ec17c68d528c1bb4d49ePaul E. McKenney * on memory ordering guarantees. 28476ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney */ 28486ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenneyvoid synchronize_rcu_bh(void) 28496ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney{ 2850fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && 2851fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney !lock_is_held(&rcu_lock_map) && 2852fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney !lock_is_held(&rcu_sched_lock_map), 2853fe15d706cfc1cb321dbe2329b04b5ca185edff60Paul E. McKenney "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section"); 28546ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney if (rcu_blocking_is_gp()) 28556ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney return; 28563705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen if (rcu_expedited) 28573705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen synchronize_rcu_bh_expedited(); 28583705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen else 28593705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen wait_rcu_gp(call_rcu_bh); 28606ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney} 28616ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenneyEXPORT_SYMBOL_GPL(synchronize_rcu_bh); 28626ebb237bece23275d1da149b61a342f0d4d06a08Paul E. McKenney 2863765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney/** 2864765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * get_state_synchronize_rcu - Snapshot current RCU state 2865765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * 2866765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * Returns a cookie that is used by a later call to cond_synchronize_rcu() 2867765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * to determine whether or not a full grace period has elapsed in the 2868765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * meantime. 2869765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney */ 2870765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenneyunsigned long get_state_synchronize_rcu(void) 2871765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney{ 2872765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney /* 2873765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * Any prior manipulation of RCU-protected data must happen 2874765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * before the load from ->gpnum. 2875765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney */ 2876765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney smp_mb(); /* ^^^ */ 2877765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney 2878765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney /* 2879765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * Make sure this load happens before the purportedly 2880765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * time-consuming work between get_state_synchronize_rcu() 2881765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * and cond_synchronize_rcu(). 2882765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney */ 2883e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma return smp_load_acquire(&rcu_state_p->gpnum); 2884765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney} 2885765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenneyEXPORT_SYMBOL_GPL(get_state_synchronize_rcu); 2886765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney 2887765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney/** 2888765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * cond_synchronize_rcu - Conditionally wait for an RCU grace period 2889765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * 2890765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * @oldstate: return value from earlier call to get_state_synchronize_rcu() 2891765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * 2892765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * If a full RCU grace period has elapsed since the earlier call to 2893765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * get_state_synchronize_rcu(), just return. Otherwise, invoke 2894765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * synchronize_rcu() to wait for a full grace period. 2895765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * 2896765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * Yes, this function does not take counter wrap into account. But 2897765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * counter wrap is harmless. If the counter wraps, we have waited for 2898765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * more than 2 billion grace periods (and way more on a 64-bit system!), 2899765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * so waiting for one additional grace period should be just fine. 2900765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney */ 2901765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenneyvoid cond_synchronize_rcu(unsigned long oldstate) 2902765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney{ 2903765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney unsigned long newstate; 2904765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney 2905765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney /* 2906765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * Ensure that this load happens before any RCU-destructive 2907765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney * actions the caller might carry out after we return. 2908765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney */ 2909e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma newstate = smp_load_acquire(&rcu_state_p->completed); 2910765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney if (ULONG_CMP_GE(oldstate, newstate)) 2911765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney synchronize_rcu(); 2912765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney} 2913765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenneyEXPORT_SYMBOL_GPL(cond_synchronize_rcu); 2914765a3f4fed708ae429ee095914a7897acb3a65bdPaul E. McKenney 29153d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenneystatic int synchronize_sched_expedited_cpu_stop(void *data) 29163d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney{ 29173d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney /* 29183d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * There must be a full memory barrier on each affected CPU 29193d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * between the time that try_stop_cpus() is called and the 29203d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * time that it returns. 29213d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * 29223d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * In the current initial implementation of cpu_stop, the 29233d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * above condition is already met when the control reaches 29243d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * this point and the following smp_mb() is not strictly 29253d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * necessary. Do smp_mb() anyway for documentation and 29263d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * robustness against future implementation changes. 29273d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney */ 29283d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney smp_mb(); /* See above comment block. */ 29293d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney return 0; 29303d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney} 29313d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 2932236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney/** 2933236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * synchronize_sched_expedited - Brute-force RCU-sched grace period 2934236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * 2935236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * Wait for an RCU-sched grace period to elapse, but use a "big hammer" 2936236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * approach to force the grace period to end quickly. This consumes 2937236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * significant time on all CPUs and is unfriendly to real-time workloads, 2938236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * so is thus not recommended for any sort of common-case code. In fact, 2939236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * if you are using synchronize_sched_expedited() in a loop, please 2940236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * restructure your code to batch your updates, and then use a single 2941236fefafe5d3d34b78ed2ccf5510909716112326Paul E. McKenney * synchronize_sched() instead. 29423d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * 29433d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * This implementation can be thought of as an application of ticket 29443d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * locking to RCU, with sync_sched_expedited_started and 29453d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * sync_sched_expedited_done taking on the roles of the halves 29463d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * of the ticket-lock word. Each task atomically increments 29473d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * sync_sched_expedited_started upon entry, snapshotting the old value, 29483d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * then attempts to stop all the CPUs. If this succeeds, then each 29493d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * CPU will have executed a context switch, resulting in an RCU-sched 29503d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * grace period. We are then done, so we use atomic_cmpxchg() to 29513d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * update sync_sched_expedited_done to match our snapshot -- but 29523d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * only if someone else has not already advanced past our snapshot. 29533d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * 29543d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * On the other hand, if try_stop_cpus() fails, we check the value 29553d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * of sync_sched_expedited_done. If it has advanced past our 29563d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * initial snapshot, then someone else must have forced a grace period 29573d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * some time after we took our snapshot. In this case, our work is 29583d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * done for us, and we can simply return. Otherwise, we try again, 29593d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * but keep our initial snapshot for purposes of checking for someone 29603d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * doing our work for us. 29613d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * 29623d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * If we fail too many times in a row, we fall back to synchronize_sched(). 29633d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney */ 29643d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenneyvoid synchronize_sched_expedited(void) 29653d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney{ 29661924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney long firstsnap, s, snap; 29671924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney int trycount = 0; 296840694d6644d5cca28531707559466122eb212d8bPaul E. McKenney struct rcu_state *rsp = &rcu_sched_state; 29693d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 29701924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney /* 29711924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * If we are in danger of counter wrap, just do synchronize_sched(). 29721924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * By allowing sync_sched_expedited_started to advance no more than 29731924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * ULONG_MAX/8 ahead of sync_sched_expedited_done, we are ensuring 29741924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * that more than 3.5 billion CPUs would be required to force a 29751924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * counter wrap on a 32-bit system. Quite a few more CPUs would of 29761924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * course be required on a 64-bit system. 29771924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney */ 297840694d6644d5cca28531707559466122eb212d8bPaul E. McKenney if (ULONG_CMP_GE((ulong)atomic_long_read(&rsp->expedited_start), 297940694d6644d5cca28531707559466122eb212d8bPaul E. McKenney (ulong)atomic_long_read(&rsp->expedited_done) + 29801924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney ULONG_MAX / 8)) { 29811924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney synchronize_sched(); 2982a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_wrap); 29831924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney return; 29841924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney } 29853d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 29861924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney /* 29871924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * Take a ticket. Note that atomic_inc_return() implies a 29881924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * full memory barrier. 29891924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney */ 299040694d6644d5cca28531707559466122eb212d8bPaul E. McKenney snap = atomic_long_inc_return(&rsp->expedited_start); 29911924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney firstsnap = snap; 2992dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney if (!try_get_online_cpus()) { 2993dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney /* CPU hotplug operation in flight, fall back to normal GP. */ 2994dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney wait_rcu_gp(call_rcu_sched); 2995dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney atomic_long_inc(&rsp->expedited_normal); 2996dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney return; 2997dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney } 29981cc85961e214773cb7d7f2ccbe3bc644dd466df0Hugh Dickins WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); 29993d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30003d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney /* 30013d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * Each pass through the following loop attempts to force a 30023d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * context switch on each CPU. 30033d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney */ 30043d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney while (try_stop_cpus(cpu_online_mask, 30053d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney synchronize_sched_expedited_cpu_stop, 30063d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney NULL) == -EAGAIN) { 30073d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney put_online_cpus(); 3008a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_tryfail); 30093d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30101924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney /* Check to see if someone else did our work for us. */ 301140694d6644d5cca28531707559466122eb212d8bPaul E. McKenney s = atomic_long_read(&rsp->expedited_done); 30121924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) { 3013a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney /* ensure test happens before caller kfree */ 30144e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* ^^^ */ 3015a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_workdone1); 30161924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney return; 30171924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney } 30183d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30193d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney /* No joy, try again later. Or just synchronize_sched(). */ 3020c701d5d9b384ff03ceb232ef21236364d784a411Paul E. McKenney if (trycount++ < 10) { 30213d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney udelay(trycount * num_online_cpus()); 3022c701d5d9b384ff03ceb232ef21236364d784a411Paul E. McKenney } else { 30233705b88db0d7cc4a097c32d9e554054103d3f807Antti P Miettinen wait_rcu_gp(call_rcu_sched); 3024a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_normal); 30253d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney return; 30263d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney } 30273d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30281924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney /* Recheck to see if someone else did our work for us. */ 302940694d6644d5cca28531707559466122eb212d8bPaul E. McKenney s = atomic_long_read(&rsp->expedited_done); 30301924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) { 3031a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney /* ensure test happens before caller kfree */ 30324e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* ^^^ */ 3033a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_workdone2); 30343d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney return; 30353d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney } 30363d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30373d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney /* 30383d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * Refetching sync_sched_expedited_started allows later 30391924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * callers to piggyback on our grace period. We retry 30401924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * after they started, so our grace period works for them, 30411924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * and they started after our first try, so their grace 30421924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * period works for us. 30433d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney */ 3044dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney if (!try_get_online_cpus()) { 3045dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney /* CPU hotplug operation in flight, use normal GP. */ 3046dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney wait_rcu_gp(call_rcu_sched); 3047dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney atomic_long_inc(&rsp->expedited_normal); 3048dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney return; 3049dd56af42bd829c6e770ed69812bd65a04eaeb1e4Paul E. McKenney } 305040694d6644d5cca28531707559466122eb212d8bPaul E. McKenney snap = atomic_long_read(&rsp->expedited_start); 30513d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney smp_mb(); /* ensure read is before try_stop_cpus(). */ 30523d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney } 3053a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_stoppedcpus); 30543d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30553d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney /* 30563d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * Everyone up to our most recent fetch is covered by our grace 30573d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * period. Update the counter, but only if our work is still 30583d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney * relevant -- which it won't be if someone who started later 30591924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney * than we did already did their update. 30603d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney */ 30613d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney do { 3062a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_done_tries); 306340694d6644d5cca28531707559466122eb212d8bPaul E. McKenney s = atomic_long_read(&rsp->expedited_done); 30641924bcb0259711eea98491a7942d1ffbf677e114Paul E. McKenney if (ULONG_CMP_GE((ulong)s, (ulong)snap)) { 3065a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney /* ensure test happens before caller kfree */ 30664e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); /* ^^^ */ 3067a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_done_lost); 30683d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney break; 30693d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney } 307040694d6644d5cca28531707559466122eb212d8bPaul E. McKenney } while (atomic_long_cmpxchg(&rsp->expedited_done, s, snap) != s); 3071a30489c5228fba6f16b4c740a0292879ef13371ePaul E. McKenney atomic_long_inc(&rsp->expedited_done_exit); 30723d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 30733d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney put_online_cpus(); 30743d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney} 30753d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenneyEXPORT_SYMBOL_GPL(synchronize_sched_expedited); 30763d3b7db0a22085cfc05c3318b9874f7fb8266d18Paul E. McKenney 307764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 307864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Check to see if there is any immediate RCU-related work to be done 307964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * by the current CPU, for the specified type of RCU, returning 1 if so. 308064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * The checks are in order of increasing expense: checks that can be 308164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * carried out against CPU-local state are performed first. However, 308264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * we must check for CPU stalls first, else we might not get a chance. 308364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 308464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) 308564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 30862f51f9884f6a36b0fe9636d5a1937e5cbd25723bPaul E. McKenney struct rcu_node *rnp = rdp->mynode; 30872f51f9884f6a36b0fe9636d5a1937e5cbd25723bPaul E. McKenney 308864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->n_rcu_pending++; 308964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 309064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Check for CPU stalls, if enabled. */ 309164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney check_cpu_stall(rsp, rdp); 309264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3093a096932f0c9c9dca9cce72f1c0fb2395df8f2dffPaul E. McKenney /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */ 3094a096932f0c9c9dca9cce72f1c0fb2395df8f2dffPaul E. McKenney if (rcu_nohz_full_cpu(rsp)) 3095a096932f0c9c9dca9cce72f1c0fb2395df8f2dffPaul E. McKenney return 0; 3096a096932f0c9c9dca9cce72f1c0fb2395df8f2dffPaul E. McKenney 309764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Is the RCU core waiting for a quiescent state from this CPU? */ 30985c51dd7349d4bb26f845f17f85daa168f5fa03f2Paul E. McKenney if (rcu_scheduler_fully_active && 30995c51dd7349d4bb26f845f17f85daa168f5fa03f2Paul E. McKenney rdp->qs_pending && !rdp->passed_quiesce) { 3100d21670acab9fcb4bc74a40b68a6941059234c55cPaul E. McKenney rdp->n_rp_qs_pending++; 3101e4cc1f22b2f4e9b0207a8cdb63e56dcf99e82d35Paul E. McKenney } else if (rdp->qs_pending && rdp->passed_quiesce) { 3102d21670acab9fcb4bc74a40b68a6941059234c55cPaul E. McKenney rdp->n_rp_report_qs++; 310364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 31047ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney } 310564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 310664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Does this CPU have callbacks ready to invoke? */ 31077ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney if (cpu_has_callbacks_ready_to_invoke(rdp)) { 31087ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney rdp->n_rp_cb_ready++; 310964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 31107ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney } 311164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 311264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Has RCU gone idle with this CPU needing another grace period? */ 31137ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney if (cpu_needs_another_gp(rsp, rdp)) { 31147ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney rdp->n_rp_cpu_needs_gp++; 311564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 31167ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney } 311764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 311864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Has another RCU grace period completed? */ 31192f51f9884f6a36b0fe9636d5a1937e5cbd25723bPaul E. McKenney if (ACCESS_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ 31207ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney rdp->n_rp_gp_completed++; 312164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 31227ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney } 312364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 312464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Has a new RCU grace period started? */ 31252f51f9884f6a36b0fe9636d5a1937e5cbd25723bPaul E. McKenney if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum) { /* outside lock */ 31267ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney rdp->n_rp_gp_started++; 312764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 1; 31287ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney } 312964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 313096d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney /* Does this CPU need a deferred NOCB wakeup? */ 313196d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney if (rcu_nocb_need_deferred_wakeup(rdp)) { 313296d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney rdp->n_rp_nocb_defer_wakeup++; 313396d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney return 1; 313496d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney } 313596d3fd0d315a949e30adc80f086031c5cdf070d1Paul E. McKenney 313664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* nothing to do */ 31377ba5c840e64d4a967379f1ae3eca73278180b11dPaul E. McKenney rdp->n_rp_need_nothing++; 313864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney return 0; 313964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 314064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 314164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 314264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Check to see if there is any immediate RCU-related work to be done 314364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * by the current CPU, returning 1 if so. This function is part of the 314464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * RCU implementation; it is -not- an exported member of the RCU API. 314564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 3146a157229cabd6dd8cfa82525fc9bf730c94cc9ac2Paul E. McKenneystatic int rcu_pending(int cpu) 314764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 31486ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 31496ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney 31506ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 31516ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney if (__rcu_pending(rsp, per_cpu_ptr(rsp->rda, cpu))) 31526ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney return 1; 31536ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney return 0; 315464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 315564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 315664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 3157c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney * Return true if the specified CPU has any callback. If all_lazy is 3158c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney * non-NULL, store an indication of whether all callbacks are lazy. 3159c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney * (If there are no callbacks, all of them are deemed to be lazy.) 316064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 3161ffa83fb565fbc397cbafb4b71fd1cce276d4c3b6Paul E. McKenneystatic int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy) 316264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 3163c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney bool al = true; 3164c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney bool hc = false; 3165c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney struct rcu_data *rdp; 31666ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 31676ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney 3168c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney for_each_rcu_flavor(rsp) { 3169c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney rdp = per_cpu_ptr(rsp->rda, cpu); 317069c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney if (!rdp->nxtlist) 317169c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney continue; 317269c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney hc = true; 317369c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney if (rdp->qlen != rdp->qlen_lazy || !all_lazy) { 3174c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney al = false; 317569c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney break; 317669c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney } 3177c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney } 3178c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney if (all_lazy) 3179c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney *all_lazy = al; 3180c0f4dfd4f90f1667d234d21f15153ea09a2eaa66Paul E. McKenney return hc; 318164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 318264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3183b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney/* 3184a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney * Helper function for _rcu_barrier() tracing. If tracing is disabled, 3185a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney * the compiler is expected to optimize this away. 3186a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney */ 3187e66c33d579ea566d10e8c8695a7168aae3e02992Steven Rostedt (Red Hat)static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s, 3188a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney int cpu, unsigned long done) 3189a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney{ 3190a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney trace_rcu_barrier(rsp->name, s, cpu, 3191a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney atomic_read(&rsp->barrier_cpu_count), done); 3192a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney} 3193a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney 3194a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney/* 3195b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * RCU callback function for _rcu_barrier(). If we are last, wake 3196b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * up the task executing _rcu_barrier(). 3197b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 319824ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenneystatic void rcu_barrier_callback(struct rcu_head *rhp) 3199d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney{ 320024ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head); 320124ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney struct rcu_state *rsp = rdp->rsp; 320224ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney 3203a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney if (atomic_dec_and_test(&rsp->barrier_cpu_count)) { 3204a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "LastCB", -1, rsp->n_barrier_done); 32057db74df88b52844f4e966901e2972bba725e6766Paul E. McKenney complete(&rsp->barrier_completion); 3206a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney } else { 3207a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "CB", -1, rsp->n_barrier_done); 3208a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney } 3209d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney} 3210d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 3211d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney/* 3212d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney * Called with preemption disabled, and from cross-cpu IRQ context. 3213d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney */ 3214d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenneystatic void rcu_barrier_func(void *type) 3215d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney{ 3216037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenney struct rcu_state *rsp = type; 3217fa07a58f71ee23a82597ce337126982d0cc60b72Christoph Lameter struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); 3218d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 3219a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "IRQ", -1, rsp->n_barrier_done); 322024ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney atomic_inc(&rsp->barrier_cpu_count); 322106668efa9180f4824fe846a8ff96338c18646bc7Paul E. McKenney rsp->call(&rdp->barrier_head, rcu_barrier_callback); 3222d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney} 3223d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 3224d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney/* 3225d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney * Orchestrate the specified type of RCU barrier, waiting for all 3226d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney * RCU callbacks of the specified type to complete. 3227d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney */ 3228037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenneystatic void _rcu_barrier(struct rcu_state *rsp) 3229d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney{ 3230b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney int cpu; 3231b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney struct rcu_data *rdp; 3232cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney unsigned long snap = ACCESS_ONCE(rsp->n_barrier_done); 3233cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney unsigned long snap_done; 3234b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3235a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "Begin", -1, snap); 3236b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3237e74f4c4564455c91a3b4075bb1721993c2a95ddaPaul E. McKenney /* Take mutex to serialize concurrent rcu_barrier() requests. */ 32387be7f0be907224445acc62b3884c892f38b7ff40Paul E. McKenney mutex_lock(&rsp->barrier_mutex); 3239b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3240cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney /* 3241cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * Ensure that all prior references, including to ->n_barrier_done, 3242cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * are ordered before the _rcu_barrier() machinery. 3243cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney */ 3244cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney smp_mb(); /* See above block comment. */ 3245cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney 3246cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney /* 3247cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * Recheck ->n_barrier_done to see if others did our work for us. 3248cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * This means checking ->n_barrier_done for an even-to-odd-to-even 3249cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * transition. The "if" expression below therefore rounds the old 3250cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * value up to the next even number and adds two before comparing. 3251cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney */ 3252458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney snap_done = rsp->n_barrier_done; 3253a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "Check", -1, snap_done); 3254458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney 3255458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney /* 3256458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * If the value in snap is odd, we needed to wait for the current 3257458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * rcu_barrier() to complete, then wait for the next one, in other 3258458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * words, we need the value of snap_done to be three larger than 3259458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * the value of snap. On the other hand, if the value in snap is 3260458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * even, we only had to wait for the next rcu_barrier() to complete, 3261458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * in other words, we need the value of snap_done to be only two 3262458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * greater than the value of snap. The "(snap + 3) & ~0x1" computes 3263458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney * this for us (thank you, Linus!). 3264458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney */ 3265458fb381eacdd23366cfa2fbdf5a467848683e3aPaul E. McKenney if (ULONG_CMP_GE(snap_done, (snap + 3) & ~0x1)) { 3266a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done); 3267cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney smp_mb(); /* caller's subsequent code after above check. */ 3268cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney mutex_unlock(&rsp->barrier_mutex); 3269cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney return; 3270cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney } 3271cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney 3272cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney /* 3273cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * Increment ->n_barrier_done to avoid duplicate work. Use 3274cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * ACCESS_ONCE() to prevent the compiler from speculating 3275cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney * the increment to precede the early-exit check. 3276cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney */ 3277a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; 3278cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1); 3279a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done); 3280cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */ 3281b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3282d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney /* 3283b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Initialize the count to one rather than to zero in order to 3284b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * avoid a too-soon return to zero in case of a short grace period 32851331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * (or preemption of this task). Exclude CPU-hotplug operations 32861331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * to ensure that no offline CPU has callbacks queued. 3287d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney */ 32887db74df88b52844f4e966901e2972bba725e6766Paul E. McKenney init_completion(&rsp->barrier_completion); 328924ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney atomic_set(&rsp->barrier_cpu_count, 1); 32901331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney get_online_cpus(); 3291b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3292b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* 32931331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * Force each CPU with callbacks to register a new callback. 32941331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * When that callback is invoked, we will know that all of the 32951331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney * corresponding CPU's preceding callbacks have been invoked. 3296b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 32973fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney for_each_possible_cpu(cpu) { 3298d1e43fa5f8bb25f83a86a29f11fcfb57ed4d7566Frederic Weisbecker if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu)) 32993fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney continue; 3300b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney rdp = per_cpu_ptr(rsp->rda, cpu); 3301d1e43fa5f8bb25f83a86a29f11fcfb57ed4d7566Frederic Weisbecker if (rcu_is_nocb_cpu(cpu)) { 3302d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) { 3303d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney _rcu_barrier_trace(rsp, "OfflineNoCB", cpu, 3304d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney rsp->n_barrier_done); 3305d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney } else { 3306d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney _rcu_barrier_trace(rsp, "OnlineNoCB", cpu, 3307d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney rsp->n_barrier_done); 3308d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney atomic_inc(&rsp->barrier_cpu_count); 3309d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney __call_rcu(&rdp->barrier_head, 3310d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney rcu_barrier_callback, rsp, cpu, 0); 3311d7e29933969e5ca7c112ce1368a07911f4485dc2Paul E. McKenney } 33123fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney } else if (ACCESS_ONCE(rdp->qlen)) { 3313a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "OnlineQ", cpu, 3314a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney rsp->n_barrier_done); 3315037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenney smp_call_function_single(cpu, rcu_barrier_func, rsp, 1); 3316b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney } else { 3317a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "OnlineNQ", cpu, 3318a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney rsp->n_barrier_done); 3319b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney } 3320b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney } 33211331e7a1bbe1f11b19c4327ba0853bee2a606543Paul E. McKenney put_online_cpus(); 3322b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3323b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* 3324b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * Now that we have an rcu_barrier_callback() callback on each 3325b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney * CPU, and thus each counted, remove the initial count. 3326b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney */ 332724ebbca8ecdd5129d7f829a7cb5146aaeb531f77Paul E. McKenney if (atomic_dec_and_test(&rsp->barrier_cpu_count)) 33287db74df88b52844f4e966901e2972bba725e6766Paul E. McKenney complete(&rsp->barrier_completion); 3329b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3330cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney /* Increment ->n_barrier_done to prevent duplicate work. */ 3331cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney smp_mb(); /* Keep increment after above mechanism. */ 3332a792563bd47632d85158c72e2acf4484eed0ec32Paul E. McKenney ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; 3333cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0); 3334a83eff0a82a7f3f14fea477fd41e6c082e7fc96aPaul E. McKenney _rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done); 3335cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney smp_mb(); /* Keep increment before caller's subsequent code. */ 3336cf3a9c4842b1e097dbe0854933c471d43dd24f69Paul E. McKenney 3337b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* Wait for all rcu_barrier_callback() callbacks to be invoked. */ 33387db74df88b52844f4e966901e2972bba725e6766Paul E. McKenney wait_for_completion(&rsp->barrier_completion); 3339b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney 3340b1420f1c8bfc30ecf6380a31d0f686884834b599Paul E. McKenney /* Other rcu_barrier() invocations can now safely proceed. */ 33417be7f0be907224445acc62b3884c892f38b7ff40Paul E. McKenney mutex_unlock(&rsp->barrier_mutex); 3342d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney} 3343d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 3344d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney/** 3345d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. 3346d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney */ 3347d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenneyvoid rcu_barrier_bh(void) 3348d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney{ 3349037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenney _rcu_barrier(&rcu_bh_state); 3350d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney} 3351d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_barrier_bh); 3352d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 3353d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney/** 3354d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. 3355d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney */ 3356d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenneyvoid rcu_barrier_sched(void) 3357d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney{ 3358037b64ed0bf2405a1a01542164d3418564b44fffPaul E. McKenney _rcu_barrier(&rcu_sched_state); 3359d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney} 3360d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenneyEXPORT_SYMBOL_GPL(rcu_barrier_sched); 3361d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney 336264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 336327569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney * Do boot-time initialization of a CPU's per-CPU RCU data. 336464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 336527569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenneystatic void __init 336627569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenneyrcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) 336764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 336864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 3369394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); 337027569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 337127569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney 337227569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney /* Set up local state, ensuring consistent view of global state. */ 33731304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 337427569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo); 33753f5d3ea64f1783f0d4ea0d35890ae3297f045a8bPaul E. McKenney init_callback_list(rdp); 3376486e259340fc4c60474f2c14703e3b3634bb58caPaul E. McKenney rdp->qlen_lazy = 0; 33771d1fb395f6dbc07b36285bbedcf01a73b57f7cb5Paul E. McKenney ACCESS_ONCE(rdp->qlen) = 0; 337827569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney rdp->dynticks = &per_cpu(rcu_dynticks, cpu); 337929e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE); 33809b2e4f1880b789be1f24f9684f7a54b90310b5c0Paul E. McKenney WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1); 338127569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney rdp->cpu = cpu; 3382d4c08f2ac311a360230eef7e5395b0ec8d8f0670Paul E. McKenney rdp->rsp = rsp; 33833fbfbf7a3b66ec424042d909f14ba2ddf4372ea8Paul E. McKenney rcu_boot_init_nocb_percpu_data(rdp); 33841304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 338527569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney} 338627569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney 338727569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney/* 338827569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney * Initialize a CPU's per-CPU RCU data. Note that only one online or 338927569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney * offline event can be happening at a given time. Note also that we 339027569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney * can accept some slop in the rsp->completed access due to the fact 339127569620c748ec13f801b4683b448a2ac2adaae4Paul E. McKenney * that this CPU cannot possibly have any RCU callbacks in flight yet. 339264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 339349fb4c6290c70c418a5c25eee996d6b55ea132d6Paul Gortmakerstatic void 33949b67122ae3da3018c966148233739116ed89502aIulia Mandarcu_init_percpu_data(int cpu, struct rcu_state *rsp) 339564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 339664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long flags; 339764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney unsigned long mask; 3398394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); 339964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp = rcu_get_root(rsp); 340064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3401a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney /* Exclude new grace periods. */ 3402a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_lock(&rsp->onoff_mutex); 3403a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney 340464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Set up local state, ensuring consistent view of global state. */ 34051304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 340664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->beenonline = 1; /* We have now been online. */ 340737c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney rdp->qlen_last_fqs_check = 0; 340837c72e56f6b234ea7387ba530434a80abf2658d8Paul E. McKenney rdp->n_force_qs_snap = rsp->n_force_qs; 340964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rdp->blimit = blimit; 34100d8ee37e2fcb7b77b9c5dee784beca5a215cad4cPaul E. McKenney init_callback_list(rdp); /* Re-enable callbacks on this CPU. */ 341129e37d814188ac8d60f2120583704d3ef6d634b4Paul E. McKenney rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; 34122333210b26cf7aaf48d71343029afb860103d9f9Paul E. McKenney rcu_sysidle_init_percpu_data(rdp->dynticks); 3413c92b131bdcf89bf79870f1631d07547241a98f6cPaul E. McKenney atomic_set(&rdp->dynticks->dynticks, 3414c92b131bdcf89bf79870f1631d07547241a98f6cPaul E. McKenney (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1); 34151304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ 341664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 341764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Add CPU to rcu_node bitmasks. */ 341864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rdp->mynode; 341964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney mask = rdp->grpmask; 342064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney do { 342164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Exclude any attempts to start a new GP on small systems. */ 34221304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock(&rnp->lock); /* irqs already disabled. */ 342364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->qsmaskinit |= mask; 342464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney mask = rnp->grpmask; 3425d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney if (rnp == rdp->mynode) { 342606ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney /* 342706ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney * If there is a grace period in progress, we will 342806ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney * set up to wait for it next time we run the 342906ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney * RCU core code. 343006ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney */ 343106ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney rdp->gpnum = rnp->completed; 3432d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney rdp->completed = rnp->completed; 343306ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney rdp->passed_quiesce = 0; 343406ae115a1d551cd952d80df06eaf8b5153351875Paul E. McKenney rdp->qs_pending = 0; 3435f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); 3436d09b62dfa336447c52a5ec9bb88adbc479b0f3b8Paul E. McKenney } 34371304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_unlock(&rnp->lock); /* irqs already disabled. */ 343864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rnp->parent; 343964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } while (rnp != NULL && !(rnp->qsmaskinit & mask)); 3440a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney local_irq_restore(flags); 344164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3442a4fbe35a124526e6759be07bd9c7ea796ba1e00dPaul E. McKenney mutex_unlock(&rsp->onoff_mutex); 344364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 344464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 344549fb4c6290c70c418a5c25eee996d6b55ea132d6Paul Gortmakerstatic void rcu_prepare_cpu(int cpu) 344664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 34476ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 34486ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney 34496ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 34509b67122ae3da3018c966148233739116ed89502aIulia Manda rcu_init_percpu_data(cpu, rsp); 345164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 345264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 345364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 3454f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney * Handle CPU online/offline notification events. 345564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 345649fb4c6290c70c418a5c25eee996d6b55ea132d6Paul Gortmakerstatic int rcu_cpu_notify(struct notifier_block *self, 34579f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney unsigned long action, void *hcpu) 345864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 345964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney long cpu = (long)hcpu; 3460e534165bbf6a04d001748c573c7d6a7bae3713a5Uma Sharma struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu); 3461a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney struct rcu_node *rnp = rdp->mynode; 34626ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney struct rcu_state *rsp; 346364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3464f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("Start CPU hotplug")); 346564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney switch (action) { 346664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_UP_PREPARE: 346764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_UP_PREPARE_FROZEN: 3468d72bce0e67e8afc6eb959f656013cbb577426f1ePeter Zijlstra rcu_prepare_cpu(cpu); 3469d72bce0e67e8afc6eb959f656013cbb577426f1ePeter Zijlstra rcu_prepare_kthreads(cpu); 347035ce7f29a44a888c45c0a9f202f69e10613c5306Paul E. McKenney rcu_spawn_all_nocb_kthreads(cpu); 3471a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney break; 3472a26ac2455ffcf3be5c6ef92bc6df7182700f2114Paul E. McKenney case CPU_ONLINE: 34730f962a5e7277c34987b77dc82fc9aefcedc95e27Paul E. McKenney case CPU_DOWN_FAILED: 34745d01bbd111d6ff9ea9d9847774f66dff39633776Thomas Gleixner rcu_boost_kthread_setaffinity(rnp, -1); 34750f962a5e7277c34987b77dc82fc9aefcedc95e27Paul E. McKenney break; 34760f962a5e7277c34987b77dc82fc9aefcedc95e27Paul E. McKenney case CPU_DOWN_PREPARE: 347734ed62461ae4970695974afb9a60ac3df0086830Paul E. McKenney rcu_boost_kthread_setaffinity(rnp, cpu); 347864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 3479d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney case CPU_DYING: 3480d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney case CPU_DYING_FROZEN: 34816ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 34826ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney rcu_cleanup_dying_cpu(rsp); 3483d0ec774cb2599c858be9d923bb873cf6697520d8Paul E. McKenney break; 348464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_DEAD: 348564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_DEAD_FROZEN: 348664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_UP_CANCELED: 348764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney case CPU_UP_CANCELED_FROZEN: 34886ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney for_each_rcu_flavor(rsp) 34896ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney rcu_cleanup_dead_cpu(cpu, rsp); 349064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 349164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney default: 349264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney break; 349364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 3494f7f7bac9cb1c50783f15937a11743655a5756a36Steven Rostedt (Red Hat) trace_rcu_utilization(TPS("End CPU hotplug")); 349534ed62461ae4970695974afb9a60ac3df0086830Paul E. McKenney return NOTIFY_OK; 349664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 349764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3498d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkovstatic int rcu_pm_notify(struct notifier_block *self, 3499d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov unsigned long action, void *hcpu) 3500d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov{ 3501d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov switch (action) { 3502d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov case PM_HIBERNATION_PREPARE: 3503d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov case PM_SUSPEND_PREPARE: 3504d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */ 3505d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov rcu_expedited = 1; 3506d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov break; 3507d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov case PM_POST_HIBERNATION: 3508d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov case PM_POST_SUSPEND: 3509d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov rcu_expedited = 0; 3510d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov break; 3511d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov default: 3512d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov break; 3513d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov } 3514d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov return NOTIFY_OK; 3515d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov} 3516d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov 351764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 35189386c0b75dda05f535a10ea1abf1817fe292c81cPaul E. McKenney * Spawn the kthreads that handle each RCU flavor's grace periods. 3519b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney */ 3520b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenneystatic int __init rcu_spawn_gp_kthread(void) 3521b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney{ 3522b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney unsigned long flags; 3523b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney struct rcu_node *rnp; 3524b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney struct rcu_state *rsp; 3525b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney struct task_struct *t; 3526b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 35279386c0b75dda05f535a10ea1abf1817fe292c81cPaul E. McKenney rcu_scheduler_fully_active = 1; 3528b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney for_each_rcu_flavor(rsp) { 3529f170168b9a0b61ea1e647b082b38f605f1d3de3eKees Cook t = kthread_run(rcu_gp_kthread, rsp, "%s", rsp->name); 3530b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney BUG_ON(IS_ERR(t)); 3531b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney rnp = rcu_get_root(rsp); 3532b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney raw_spin_lock_irqsave(&rnp->lock, flags); 3533b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney rsp->gp_kthread = t; 3534b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney raw_spin_unlock_irqrestore(&rnp->lock, flags); 3535b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney } 353635ce7f29a44a888c45c0a9f202f69e10613c5306Paul E. McKenney rcu_spawn_nocb_kthreads(); 35379386c0b75dda05f535a10ea1abf1817fe292c81cPaul E. McKenney rcu_spawn_boost_kthreads(); 3538b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney return 0; 3539b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney} 3540b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenneyearly_initcall(rcu_spawn_gp_kthread); 3541b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney 3542b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney/* 3543bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * This function is invoked towards the end of the scheduler's initialization 3544bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * process. Before this is called, the idle task might contain 3545bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * RCU read-side critical sections (during which time, this idle 3546bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * task is booting the system). After this function is called, the 3547bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * idle tasks are prohibited from containing RCU read-side critical 3548bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney * sections. This function also enables RCU lockdep checking. 3549bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney */ 3550bbad937983147c017c25406860287cb94da9af7cPaul E. McKenneyvoid rcu_scheduler_starting(void) 3551bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney{ 3552bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney WARN_ON(num_online_cpus() != 1); 3553bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney WARN_ON(nr_context_switches() > 0); 3554bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney rcu_scheduler_active = 1; 3555bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney} 3556bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney 3557bbad937983147c017c25406860287cb94da9af7cPaul E. McKenney/* 355864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Compute the per-level fanout, either using the exact fanout specified 355964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT. 356064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 356164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#ifdef CONFIG_RCU_FANOUT_EXACT 356264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void __init rcu_init_levelspread(struct rcu_state *rsp) 356364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 356464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int i; 356564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 356604f34650ca5e8445aae0ab3e0ff6704f141150a8Paul E. McKenney rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf; 356704f34650ca5e8445aae0ab3e0ff6704f141150a8Paul E. McKenney for (i = rcu_num_lvls - 2; i >= 0; i--) 356864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rsp->levelspread[i] = CONFIG_RCU_FANOUT; 356964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 357064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#else /* #ifdef CONFIG_RCU_FANOUT_EXACT */ 357164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenneystatic void __init rcu_init_levelspread(struct rcu_state *rsp) 357264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 357364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int ccur; 357464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int cprv; 357564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int i; 357664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 35774dbd6bb38dd1cbfa5cb21e56e51dffc74aa20038Paul E. McKenney cprv = nr_cpu_ids; 3578f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = rcu_num_lvls - 1; i >= 0; i--) { 357964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney ccur = rsp->levelcnt[i]; 358064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rsp->levelspread[i] = (cprv + ccur - 1) / ccur; 358164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney cprv = ccur; 358264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 358364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 358464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney#endif /* #else #ifdef CONFIG_RCU_FANOUT_EXACT */ 358564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 358664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney/* 358764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney * Helper function for rcu_init() that initializes one rcu_state structure. 358864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney */ 3589394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshanstatic void __init rcu_init_one(struct rcu_state *rsp, 3590394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan struct rcu_data __percpu *rda) 359164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 3592b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick static const char * const buf[] = { 3593b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_0", 3594b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_1", 3595b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_2", 3596b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_3" }; /* Match MAX_RCU_LVLS */ 3597b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick static const char * const fqs[] = { 3598b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_fqs_0", 3599b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_fqs_1", 3600b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_fqs_2", 3601b4426b49c65e0d266f8a9181ca51d5bf11407714Fabian Frederick "rcu_node_fqs_3" }; /* Match MAX_RCU_LVLS */ 36024a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney static u8 fl_mask = 0x1; 360364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int cpustride = 1; 360464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int i; 360564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney int j; 360664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney struct rcu_node *rnp; 360764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3608b6407e863934965cdc66cbc244d811ceeb6f4d77Paul E. McKenney BUILD_BUG_ON(MAX_RCU_LVLS > ARRAY_SIZE(buf)); /* Fix buf[] init! */ 3609b6407e863934965cdc66cbc244d811ceeb6f4d77Paul E. McKenney 36104930521ae10fd28ebc713107fd243c8044a95415Paul E. McKenney /* Silence gcc 4.8 warning about array index out of range. */ 36114930521ae10fd28ebc713107fd243c8044a95415Paul E. McKenney if (rcu_num_lvls > RCU_NUM_LVLS) 36124930521ae10fd28ebc713107fd243c8044a95415Paul E. McKenney panic("rcu_init_one: rcu_num_lvls overflow"); 36134930521ae10fd28ebc713107fd243c8044a95415Paul E. McKenney 361464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Initialize the level-tracking arrays. */ 361564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3616f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = 0; i < rcu_num_lvls; i++) 3617f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rsp->levelcnt[i] = num_rcu_lvl[i]; 3618f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = 1; i < rcu_num_lvls; i++) 361964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rsp->level[i] = rsp->level[i - 1] + rsp->levelcnt[i - 1]; 362064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rcu_init_levelspread(rsp); 36214a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney rsp->flavor_mask = fl_mask; 36224a81e8328d3791a4f99bf5b436d050f6dc5ffea3Paul E. McKenney fl_mask <<= 1; 362364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 362464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney /* Initialize the elements themselves, starting from the leaves. */ 362564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3626f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = rcu_num_lvls - 1; i >= 0; i--) { 362764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney cpustride *= rsp->levelspread[i]; 362864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp = rsp->level[i]; 362964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) { 36301304afb225288a2e250d6a7495462c28e5509cbbPaul E. McKenney raw_spin_lock_init(&rnp->lock); 3631b6407e863934965cdc66cbc244d811ceeb6f4d77Paul E. McKenney lockdep_set_class_and_name(&rnp->lock, 3632b6407e863934965cdc66cbc244d811ceeb6f4d77Paul E. McKenney &rcu_node_class[i], buf[i]); 3633394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney raw_spin_lock_init(&rnp->fqslock); 3634394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney lockdep_set_class_and_name(&rnp->fqslock, 3635394f2769aa0dbcf027bae6fb52835e25e05d332ePaul E. McKenney &rcu_fqs_class[i], fqs[i]); 363625d30cf4250f74e5ceb35f8f39739782408db633Paul E. McKenney rnp->gpnum = rsp->gpnum; 363725d30cf4250f74e5ceb35f8f39739782408db633Paul E. McKenney rnp->completed = rsp->completed; 363864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->qsmask = 0; 363964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->qsmaskinit = 0; 364064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grplo = j * cpustride; 364164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grphi = (j + 1) * cpustride - 1; 3642595f3900f6b4221403493c530138b8dad2bd87f3Himangi Saraogi if (rnp->grphi >= nr_cpu_ids) 3643595f3900f6b4221403493c530138b8dad2bd87f3Himangi Saraogi rnp->grphi = nr_cpu_ids - 1; 364464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney if (i == 0) { 364564db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grpnum = 0; 364664db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grpmask = 0; 364764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->parent = NULL; 364864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } else { 364964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grpnum = j % rsp->levelspread[i - 1]; 365064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->grpmask = 1UL << rnp->grpnum; 365164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->parent = rsp->level[i - 1] + 365264db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney j / rsp->levelspread[i - 1]; 365364db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 365464db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney rnp->level = i; 365512f5f524cafef3ab689929b118f2dfb8bf2be321Paul E. McKenney INIT_LIST_HEAD(&rnp->blkd_tasks); 3656dae6e64d2bcfd4b06304ab864c7e3a4f6b5fedf4Paul E. McKenney rcu_init_one_nocb(rnp); 365764db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 365864db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney } 36590c34029abdfdea64420cb4264c4e91a776b22157Lai Jiangshan 3660394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan rsp->rda = rda; 3661b3dbec76e5334fbb063987dea14e7b255602d7e4Paul E. McKenney init_waitqueue_head(&rsp->gp_wq); 3662f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rnp = rsp->level[rcu_num_lvls - 1]; 36630c34029abdfdea64420cb4264c4e91a776b22157Lai Jiangshan for_each_possible_cpu(i) { 36644a90a0681cf6cd21cd444184302aa045156486b3Paul E. McKenney while (i > rnp->grphi) 36650c34029abdfdea64420cb4264c4e91a776b22157Lai Jiangshan rnp++; 3666394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan per_cpu_ptr(rsp->rda, i)->mynode = rnp; 36670c34029abdfdea64420cb4264c4e91a776b22157Lai Jiangshan rcu_boot_init_percpu_data(i, rsp); 36680c34029abdfdea64420cb4264c4e91a776b22157Lai Jiangshan } 36696ce75a2326e6f8b3bdfb60e1de7934b89858e87bPaul E. McKenney list_add(&rsp->flavors, &rcu_struct_flavors); 367064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 367164db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 3672f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney/* 3673f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * Compute the rcu_node tree geometry from kernel parameters. This cannot 36744102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney * replace the definitions in tree.h because those are needed to size 3675f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * the ->node array in the rcu_state structure. 3676f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney */ 3677f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenneystatic void __init rcu_init_geometry(void) 3678f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney{ 3679026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney ulong d; 3680f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney int i; 3681f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney int j; 3682cca6f3931920a7547d02e68adc2ca635bea5600cPaul E. McKenney int n = nr_cpu_ids; 3683f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney int rcu_capacity[MAX_RCU_LVLS + 1]; 3684f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 3685026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney /* 3686026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney * Initialize any unspecified boot parameters. 3687026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney * The default values of jiffies_till_first_fqs and 3688026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS 3689026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney * value, which is a function of HZ, then adding one for each 3690026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney * RCU_JIFFIES_FQS_DIV CPUs that might be on the system. 3691026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney */ 3692026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV; 3693026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney if (jiffies_till_first_fqs == ULONG_MAX) 3694026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney jiffies_till_first_fqs = d; 3695026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney if (jiffies_till_next_fqs == ULONG_MAX) 3696026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney jiffies_till_next_fqs = d; 3697026ad2835ce6202069e7aa0b11f5f1be4de34550Paul E. McKenney 3698f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney /* If the compile-time values are accurate, just leave. */ 3699b17c7035f37f47c7f7cb08a5555ab2aebfa31f91Paul E. McKenney if (rcu_fanout_leaf == CONFIG_RCU_FANOUT_LEAF && 3700b17c7035f37f47c7f7cb08a5555ab2aebfa31f91Paul E. McKenney nr_cpu_ids == NR_CPUS) 3701f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney return; 37023947909814f38d524829bc41bd4c11068a15f0cdPaul E. McKenney pr_info("RCU: Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%d\n", 37033947909814f38d524829bc41bd4c11068a15f0cdPaul E. McKenney rcu_fanout_leaf, nr_cpu_ids); 3704f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 3705f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney /* 3706f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * Compute number of nodes that can be handled an rcu_node tree 3707f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * with the given number of levels. Setting rcu_capacity[0] makes 3708f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * some of the arithmetic easier. 3709f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney */ 3710f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_capacity[0] = 1; 3711f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_capacity[1] = rcu_fanout_leaf; 3712f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = 2; i <= MAX_RCU_LVLS; i++) 3713f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_capacity[i] = rcu_capacity[i - 1] * CONFIG_RCU_FANOUT; 3714f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 3715f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney /* 3716f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * The boot-time rcu_fanout_leaf parameter is only permitted 3717f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * to increase the leaf-level fanout, not decrease it. Of course, 3718f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * the leaf-level fanout cannot exceed the number of bits in 3719f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * the rcu_node masks. Finally, the tree must be able to accommodate 3720f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * the configured number of CPUs. Complain and fall back to the 3721f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney * compile-time values if these limits are exceeded. 3722f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney */ 3723f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney if (rcu_fanout_leaf < CONFIG_RCU_FANOUT_LEAF || 3724f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_fanout_leaf > sizeof(unsigned long) * 8 || 3725f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney n > rcu_capacity[MAX_RCU_LVLS]) { 3726f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney WARN_ON(1); 3727f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney return; 3728f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney } 3729f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 3730f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney /* Calculate the number of rcu_nodes at each level of the tree. */ 3731f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = 1; i <= MAX_RCU_LVLS; i++) 3732f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney if (n <= rcu_capacity[i]) { 3733f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (j = 0; j <= i; j++) 3734f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney num_rcu_lvl[j] = 3735f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney DIV_ROUND_UP(n, rcu_capacity[i - j]); 3736f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_num_lvls = i; 3737f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (j = i + 1; j <= MAX_RCU_LVLS; j++) 3738f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney num_rcu_lvl[j] = 0; 3739f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney break; 3740f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney } 3741f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 3742f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney /* Calculate the total number of rcu_node structures. */ 3743f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_num_nodes = 0; 3744f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney for (i = 0; i <= MAX_RCU_LVLS; i++) 3745f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_num_nodes += num_rcu_lvl[i]; 3746f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_num_nodes -= n; 3747f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney} 3748f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney 37499f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenneyvoid __init rcu_init(void) 375064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney{ 3751017c426138122c8e9b9f5057fbd0567c37b35247Paul E. McKenney int cpu; 37529f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney 3753f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney rcu_bootup_announce(); 3754f885b7f2b2de70be266d2cecc476f773a1e2ca5dPaul E. McKenney rcu_init_geometry(); 3755394f99a9007d4274f7076bb8553ab0ff9707688bLai Jiangshan rcu_init_one(&rcu_bh_state, &rcu_bh_data); 375669c8d28c96445e28f081fcd987e34ea2afa65039Paul E. McKenney rcu_init_one(&rcu_sched_state, &rcu_sched_data); 3757f41d911f8c49a5d65c86504c19e8204bb605c4fdPaul E. McKenney __rcu_init_preempt(); 3758b5b393601dbce0bce3f0717f29e6c8d1cf0295daJiang Fang open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); 37599f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney 37609f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney /* 37619f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney * We don't need protection against CPU-hotplug here because 37629f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney * this is called early in boot, before either interrupts 37639f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney * or the scheduler are operational. 37649f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney */ 37659f680ab41485edfdc96331b70afa7513aa0a7720Paul E. McKenney cpu_notifier(rcu_cpu_notify, 0); 3766d1d74d14e98a6be740a6f12456c7d9ad47be9c9cBorislav Petkov pm_notifier(rcu_pm_notify, 0); 3767017c426138122c8e9b9f5057fbd0567c37b35247Paul E. McKenney for_each_online_cpu(cpu) 3768017c426138122c8e9b9f5057fbd0567c37b35247Paul E. McKenney rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu); 376964db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney} 377064db4cfff99c04cd5f550357edcc8780f96b54a2Paul E. McKenney 37714102adab9189c8ea2f0cdd2f88345fd25d2790f1Paul E. McKenney#include "tree_plugin.h" 3772