1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. 4 * 5 * Copyright IBM Corporation, 2008 6 * 7 * Author: Paul E. McKenney <paulmck@linux.ibm.com> 8 * 9 * For detailed explanation of Read-Copy Update mechanism see - 10 * Documentation/RCU 11 */ 12 #include <linux/completion.h> 13 #include <linux/interrupt.h> 14 #include <linux/notifier.h> 15 #include <linux/rcupdate_wait.h> 16 #include <linux/kernel.h> 17 #include <linux/export.h> 18 #include <linux/mutex.h> 19 #include <linux/sched.h> 20 #include <linux/types.h> 21 #include <linux/init.h> 22 #include <linux/time.h> 23 #include <linux/cpu.h> 24 #include <linux/prefetch.h> 25 26 #include "rcu.h" 27 28 /* Global control variables for rcupdate callback mechanism. */ 29 struct rcu_ctrlblk { 30 struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ 31 struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ 32 struct rcu_head **curtail; /* ->next pointer of last CB. */ 33 }; 34 35 /* Definition for rcupdate control block. */ 36 static struct rcu_ctrlblk rcu_ctrlblk = { 37 .donetail = &rcu_ctrlblk.rcucblist, 38 .curtail = &rcu_ctrlblk.rcucblist, 39 }; 40 41 void rcu_barrier(void) 42 { 43 wait_rcu_gp(call_rcu); 44 } 45 EXPORT_SYMBOL(rcu_barrier); 46 47 /* Record an rcu quiescent state. */ 48 void rcu_qs(void) 49 { 50 unsigned long flags; 51 52 local_irq_save(flags); 53 if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { 54 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; 55 raise_softirq_irqoff(RCU_SOFTIRQ); 56 } 57 local_irq_restore(flags); 58 } 59 60 /* 61 * Check to see if the scheduling-clock interrupt came from an extended 62 * quiescent state, and, if so, tell RCU about it. This function must 63 * be called from hardirq context. It is normally called from the 64 * scheduling-clock interrupt. 65 */ 66 void rcu_sched_clock_irq(int user) 67 { 68 if (user) { 69 rcu_qs(); 70 } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { 71 set_tsk_need_resched(current); 72 set_preempt_need_resched(); 73 } 74 } 75 76 /* Invoke the RCU callbacks whose grace period has elapsed. */ 77 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) 78 { 79 struct rcu_head *next, *list; 80 unsigned long flags; 81 82 /* Move the ready-to-invoke callbacks to a local list. */ 83 local_irq_save(flags); 84 if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { 85 /* No callbacks ready, so just leave. */ 86 local_irq_restore(flags); 87 return; 88 } 89 list = rcu_ctrlblk.rcucblist; 90 rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; 91 *rcu_ctrlblk.donetail = NULL; 92 if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) 93 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; 94 rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; 95 local_irq_restore(flags); 96 97 /* Invoke the callbacks on the local list. */ 98 while (list) { 99 next = list->next; 100 prefetch(next); 101 debug_rcu_head_unqueue(list); 102 local_bh_disable(); 103 __rcu_reclaim("", list); 104 local_bh_enable(); 105 list = next; 106 } 107 } 108 109 /* 110 * Wait for a grace period to elapse. But it is illegal to invoke 111 * synchronize_rcu() from within an RCU read-side critical section. 112 * Therefore, any legal call to synchronize_rcu() is a quiescent 113 * state, and so on a UP system, synchronize_rcu() need do nothing. 114 * (But Lai Jiangshan points out the benefits of doing might_sleep() 115 * to reduce latency.) 116 * 117 * Cool, huh? (Due to Josh Triplett.) 118 */ 119 void synchronize_rcu(void) 120 { 121 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || 122 lock_is_held(&rcu_lock_map) || 123 lock_is_held(&rcu_sched_lock_map), 124 "Illegal synchronize_rcu() in RCU read-side critical section"); 125 } 126 EXPORT_SYMBOL_GPL(synchronize_rcu); 127 128 /* 129 * Post an RCU callback to be invoked after the end of an RCU grace 130 * period. But since we have but one CPU, that would be after any 131 * quiescent state. 132 */ 133 void call_rcu(struct rcu_head *head, rcu_callback_t func) 134 { 135 unsigned long flags; 136 137 debug_rcu_head_queue(head); 138 head->func = func; 139 head->next = NULL; 140 141 local_irq_save(flags); 142 *rcu_ctrlblk.curtail = head; 143 rcu_ctrlblk.curtail = &head->next; 144 local_irq_restore(flags); 145 146 if (unlikely(is_idle_task(current))) { 147 /* force scheduling for rcu_qs() */ 148 resched_cpu(0); 149 } 150 } 151 EXPORT_SYMBOL_GPL(call_rcu); 152 153 void __init rcu_init(void) 154 { 155 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); 156 rcu_early_boot_tests(); 157 srcu_init(); 158 } 159