1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Sleepable Read-Copy Update mechanism for mutual exclusion, 4 * tiny version for non-preemptible single-CPU use. 5 * 6 * Copyright (C) IBM Corporation, 2017 7 * 8 * Author: Paul McKenney <paulmck@linux.ibm.com> 9 */ 10 11 #include <linux/export.h> 12 #include <linux/mutex.h> 13 #include <linux/preempt.h> 14 #include <linux/rcupdate_wait.h> 15 #include <linux/sched.h> 16 #include <linux/delay.h> 17 #include <linux/srcu.h> 18 19 #include <linux/rcu_node_tree.h> 20 #include "rcu_segcblist.h" 21 #include "rcu.h" 22 23 int rcu_scheduler_active __read_mostly; 24 static LIST_HEAD(srcu_boot_list); 25 static bool srcu_init_done; 26 27 static int init_srcu_struct_fields(struct srcu_struct *ssp) 28 { 29 ssp->srcu_lock_nesting[0] = 0; 30 ssp->srcu_lock_nesting[1] = 0; 31 init_swait_queue_head(&ssp->srcu_wq); 32 ssp->srcu_cb_head = NULL; 33 ssp->srcu_cb_tail = &ssp->srcu_cb_head; 34 ssp->srcu_gp_running = false; 35 ssp->srcu_gp_waiting = false; 36 ssp->srcu_idx = 0; 37 INIT_WORK(&ssp->srcu_work, srcu_drive_gp); 38 INIT_LIST_HEAD(&ssp->srcu_work.entry); 39 return 0; 40 } 41 42 #ifdef CONFIG_DEBUG_LOCK_ALLOC 43 44 int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 45 struct lock_class_key *key) 46 { 47 /* Don't re-initialize a lock while it is held. */ 48 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); 49 lockdep_init_map(&ssp->dep_map, name, key, 0); 50 return init_srcu_struct_fields(ssp); 51 } 52 EXPORT_SYMBOL_GPL(__init_srcu_struct); 53 54 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 55 56 /* 57 * init_srcu_struct - initialize a sleep-RCU structure 58 * @ssp: structure to initialize. 59 * 60 * Must invoke this on a given srcu_struct before passing that srcu_struct 61 * to any other function. Each srcu_struct represents a separate domain 62 * of SRCU protection. 63 */ 64 int init_srcu_struct(struct srcu_struct *ssp) 65 { 66 return init_srcu_struct_fields(ssp); 67 } 68 EXPORT_SYMBOL_GPL(init_srcu_struct); 69 70 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 71 72 /* 73 * cleanup_srcu_struct - deconstruct a sleep-RCU structure 74 * @ssp: structure to clean up. 75 * 76 * Must invoke this after you are finished using a given srcu_struct that 77 * was initialized via init_srcu_struct(), else you leak memory. 78 */ 79 void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced) 80 { 81 WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); 82 if (quiesced) 83 WARN_ON(work_pending(&ssp->srcu_work)); 84 else 85 flush_work(&ssp->srcu_work); 86 WARN_ON(ssp->srcu_gp_running); 87 WARN_ON(ssp->srcu_gp_waiting); 88 WARN_ON(ssp->srcu_cb_head); 89 WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); 90 } 91 EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); 92 93 /* 94 * Removes the count for the old reader from the appropriate element of 95 * the srcu_struct. 96 */ 97 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) 98 { 99 int newval = ssp->srcu_lock_nesting[idx] - 1; 100 101 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval); 102 if (!newval && READ_ONCE(ssp->srcu_gp_waiting)) 103 swake_up_one(&ssp->srcu_wq); 104 } 105 EXPORT_SYMBOL_GPL(__srcu_read_unlock); 106 107 /* 108 * Workqueue handler to drive one grace period and invoke any callbacks 109 * that become ready as a result. Single-CPU and !PREEMPT operation 110 * means that we get away with murder on synchronization. ;-) 111 */ 112 void srcu_drive_gp(struct work_struct *wp) 113 { 114 int idx; 115 struct rcu_head *lh; 116 struct rcu_head *rhp; 117 struct srcu_struct *ssp; 118 119 ssp = container_of(wp, struct srcu_struct, srcu_work); 120 if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) 121 return; /* Already running or nothing to do. */ 122 123 /* Remove recently arrived callbacks and wait for readers. */ 124 WRITE_ONCE(ssp->srcu_gp_running, true); 125 local_irq_disable(); 126 lh = ssp->srcu_cb_head; 127 ssp->srcu_cb_head = NULL; 128 ssp->srcu_cb_tail = &ssp->srcu_cb_head; 129 local_irq_enable(); 130 idx = ssp->srcu_idx; 131 WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx); 132 WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ 133 swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx])); 134 WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ 135 136 /* Invoke the callbacks we removed above. */ 137 while (lh) { 138 rhp = lh; 139 lh = lh->next; 140 local_bh_disable(); 141 rhp->func(rhp); 142 local_bh_enable(); 143 } 144 145 /* 146 * Enable rescheduling, and if there are more callbacks, 147 * reschedule ourselves. This can race with a call_srcu() 148 * at interrupt level, but the ->srcu_gp_running checks will 149 * straighten that out. 150 */ 151 WRITE_ONCE(ssp->srcu_gp_running, false); 152 if (READ_ONCE(ssp->srcu_cb_head)) 153 schedule_work(&ssp->srcu_work); 154 } 155 EXPORT_SYMBOL_GPL(srcu_drive_gp); 156 157 /* 158 * Enqueue an SRCU callback on the specified srcu_struct structure, 159 * initiating grace-period processing if it is not already running. 160 */ 161 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, 162 rcu_callback_t func) 163 { 164 unsigned long flags; 165 166 rhp->func = func; 167 rhp->next = NULL; 168 local_irq_save(flags); 169 *ssp->srcu_cb_tail = rhp; 170 ssp->srcu_cb_tail = &rhp->next; 171 local_irq_restore(flags); 172 if (!READ_ONCE(ssp->srcu_gp_running)) { 173 if (likely(srcu_init_done)) 174 schedule_work(&ssp->srcu_work); 175 else if (list_empty(&ssp->srcu_work.entry)) 176 list_add(&ssp->srcu_work.entry, &srcu_boot_list); 177 } 178 } 179 EXPORT_SYMBOL_GPL(call_srcu); 180 181 /* 182 * synchronize_srcu - wait for prior SRCU read-side critical-section completion 183 */ 184 void synchronize_srcu(struct srcu_struct *ssp) 185 { 186 struct rcu_synchronize rs; 187 188 init_rcu_head_on_stack(&rs.head); 189 init_completion(&rs.completion); 190 call_srcu(ssp, &rs.head, wakeme_after_rcu); 191 wait_for_completion(&rs.completion); 192 destroy_rcu_head_on_stack(&rs.head); 193 } 194 EXPORT_SYMBOL_GPL(synchronize_srcu); 195 196 /* Lockdep diagnostics. */ 197 void __init rcu_scheduler_starting(void) 198 { 199 rcu_scheduler_active = RCU_SCHEDULER_RUNNING; 200 } 201 202 /* 203 * Queue work for srcu_struct structures with early boot callbacks. 204 * The work won't actually execute until the workqueue initialization 205 * phase that takes place after the scheduler starts. 206 */ 207 void __init srcu_init(void) 208 { 209 struct srcu_struct *ssp; 210 211 srcu_init_done = true; 212 while (!list_empty(&srcu_boot_list)) { 213 ssp = list_first_entry(&srcu_boot_list, 214 struct srcu_struct, srcu_work.entry); 215 list_del_init(&ssp->srcu_work.entry); 216 schedule_work(&ssp->srcu_work); 217 } 218 } 219