1 /* 2 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright IBM Corporation, 2008 19 * 20 * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> 21 * 22 * For detailed explanation of Read-Copy Update mechanism see - 23 * Documentation/RCU 24 */ 25 #include <linux/completion.h> 26 #include <linux/interrupt.h> 27 #include <linux/notifier.h> 28 #include <linux/rcupdate_wait.h> 29 #include <linux/kernel.h> 30 #include <linux/export.h> 31 #include <linux/mutex.h> 32 #include <linux/sched.h> 33 #include <linux/types.h> 34 #include <linux/init.h> 35 #include <linux/time.h> 36 #include <linux/cpu.h> 37 #include <linux/prefetch.h> 38 39 #include "rcu.h" 40 41 /* Global control variables for rcupdate callback mechanism. */ 42 struct rcu_ctrlblk { 43 struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ 44 struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ 45 struct rcu_head **curtail; /* ->next pointer of last CB. */ 46 }; 47 48 /* Definition for rcupdate control block. */ 49 static struct rcu_ctrlblk rcu_ctrlblk = { 50 .donetail = &rcu_ctrlblk.rcucblist, 51 .curtail = &rcu_ctrlblk.rcucblist, 52 }; 53 54 void rcu_barrier(void) 55 { 56 wait_rcu_gp(call_rcu); 57 } 58 EXPORT_SYMBOL(rcu_barrier); 59 60 /* Record an rcu quiescent state. */ 61 void rcu_qs(void) 62 { 63 unsigned long flags; 64 65 local_irq_save(flags); 66 if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { 67 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; 68 raise_softirq(RCU_SOFTIRQ); 69 } 70 local_irq_restore(flags); 71 } 72 73 /* 74 * Check to see if the scheduling-clock interrupt came from an extended 75 * quiescent state, and, if so, tell RCU about it. This function must 76 * be called from hardirq context. It is normally called from the 77 * scheduling-clock interrupt. 78 */ 79 void rcu_check_callbacks(int user) 80 { 81 if (user) { 82 rcu_qs(); 83 } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { 84 set_tsk_need_resched(current); 85 set_preempt_need_resched(); 86 } 87 } 88 89 /* Invoke the RCU callbacks whose grace period has elapsed. */ 90 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) 91 { 92 struct rcu_head *next, *list; 93 unsigned long flags; 94 95 /* Move the ready-to-invoke callbacks to a local list. */ 96 local_irq_save(flags); 97 if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { 98 /* No callbacks ready, so just leave. */ 99 local_irq_restore(flags); 100 return; 101 } 102 list = rcu_ctrlblk.rcucblist; 103 rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; 104 *rcu_ctrlblk.donetail = NULL; 105 if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) 106 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; 107 rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; 108 local_irq_restore(flags); 109 110 /* Invoke the callbacks on the local list. */ 111 while (list) { 112 next = list->next; 113 prefetch(next); 114 debug_rcu_head_unqueue(list); 115 local_bh_disable(); 116 __rcu_reclaim("", list); 117 local_bh_enable(); 118 list = next; 119 } 120 } 121 122 /* 123 * Wait for a grace period to elapse. But it is illegal to invoke 124 * synchronize_rcu() from within an RCU read-side critical section. 125 * Therefore, any legal call to synchronize_rcu() is a quiescent 126 * state, and so on a UP system, synchronize_rcu() need do nothing. 127 * (But Lai Jiangshan points out the benefits of doing might_sleep() 128 * to reduce latency.) 129 * 130 * Cool, huh? (Due to Josh Triplett.) 131 */ 132 void synchronize_rcu(void) 133 { 134 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || 135 lock_is_held(&rcu_lock_map) || 136 lock_is_held(&rcu_sched_lock_map), 137 "Illegal synchronize_rcu() in RCU read-side critical section"); 138 } 139 EXPORT_SYMBOL_GPL(synchronize_rcu); 140 141 /* 142 * Post an RCU callback to be invoked after the end of an RCU grace 143 * period. But since we have but one CPU, that would be after any 144 * quiescent state. 145 */ 146 void call_rcu(struct rcu_head *head, rcu_callback_t func) 147 { 148 unsigned long flags; 149 150 debug_rcu_head_queue(head); 151 head->func = func; 152 head->next = NULL; 153 154 local_irq_save(flags); 155 *rcu_ctrlblk.curtail = head; 156 rcu_ctrlblk.curtail = &head->next; 157 local_irq_restore(flags); 158 159 if (unlikely(is_idle_task(current))) { 160 /* force scheduling for rcu_qs() */ 161 resched_cpu(0); 162 } 163 } 164 EXPORT_SYMBOL_GPL(call_rcu); 165 166 void __init rcu_init(void) 167 { 168 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); 169 rcu_early_boot_tests(); 170 srcu_init(); 171 } 172