215 static bool __wake_nocb_gp(struct rcu_data *rdp_gp,  in __wake_nocb_gp()  argument
218 	__releases(rdp_gp->nocb_gp_lock)  in __wake_nocb_gp()
222 	if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {  in __wake_nocb_gp()
223 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in __wake_nocb_gp()
229 	if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {  in __wake_nocb_gp()
230 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);  in __wake_nocb_gp()
231 		del_timer(&rdp_gp->nocb_timer);  in __wake_nocb_gp()
234 	if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {  in __wake_nocb_gp()
235 		WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);  in __wake_nocb_gp()
238 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in __wake_nocb_gp()
241 		wake_up_process(rdp_gp->nocb_gp_kthread);  in __wake_nocb_gp()
253 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp()  local
255 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp()
256 	return __wake_nocb_gp(rdp_gp, rdp, force, flags);  in wake_nocb_gp()
292 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp_defer()  local
294 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp_defer()
302 		mod_timer(&rdp_gp->nocb_timer, jiffies + jiffies_till_flush);  in wake_nocb_gp_defer()
303 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);  in wake_nocb_gp_defer()
305 		mod_timer(&rdp_gp->nocb_timer, jiffies + 2);  in wake_nocb_gp_defer()
306 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);  in wake_nocb_gp_defer()
308 		if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE)  in wake_nocb_gp_defer()
309 			mod_timer(&rdp_gp->nocb_timer, jiffies + 1);  in wake_nocb_gp_defer()
310 		if (rdp_gp->nocb_defer_wakeup < waketype)  in wake_nocb_gp_defer()
311 			WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);  in wake_nocb_gp_defer()
314 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp_defer()
1022 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp,  in do_nocb_deferred_wakeup_common()  argument
1025 	__releases(rdp_gp->nocb_gp_lock)  in do_nocb_deferred_wakeup_common()
1030 	if (!rcu_nocb_need_deferred_wakeup(rdp_gp, level)) {  in do_nocb_deferred_wakeup_common()
1031 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup_common()
1035 	ndw = rdp_gp->nocb_defer_wakeup;  in do_nocb_deferred_wakeup_common()
1036 	ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);  in do_nocb_deferred_wakeup_common()
1064 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in do_nocb_deferred_wakeup()  local
1066 	if (!rdp_gp || !rcu_nocb_need_deferred_wakeup(rdp_gp, RCU_NOCB_WAKE))  in do_nocb_deferred_wakeup()
1069 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup()
1070 	return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);  in do_nocb_deferred_wakeup()
1084 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rdp_offload_toggle()  local
1099 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in rdp_offload_toggle()
1101 	WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);  in rdp_offload_toggle()
1102 	if (rdp_gp->nocb_gp_sleep) {  in rdp_offload_toggle()
1103 		rdp_gp->nocb_gp_sleep = false;  in rdp_offload_toggle()
1106 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in rdp_offload_toggle()
1117 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_deoffload()  local
1150 	mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);  in rcu_nocb_rdp_deoffload()
1151 	if (rdp_gp->nocb_gp_kthread) {  in rcu_nocb_rdp_deoffload()
1153 			wake_up_process(rdp_gp->nocb_gp_kthread);  in rcu_nocb_rdp_deoffload()
1181 	mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);  in rcu_nocb_rdp_deoffload()
1236 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_offload()  local
1246 	if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread))  in rcu_nocb_rdp_offload()
1275 		wake_up_process(rdp_gp->nocb_gp_kthread);  in rcu_nocb_rdp_offload()
1490 	struct rcu_data *rdp_gp;  in rcu_spawn_cpu_nocb_kthread()  local
1503 	rdp_gp = rdp->nocb_gp_rdp;  in rcu_spawn_cpu_nocb_kthread()
1504 	mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);  in rcu_spawn_cpu_nocb_kthread()
1505 	if (!rdp_gp->nocb_gp_kthread) {  in rcu_spawn_cpu_nocb_kthread()
1506 		t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,  in rcu_spawn_cpu_nocb_kthread()
1507 				"rcuog/%d", rdp_gp->cpu);  in rcu_spawn_cpu_nocb_kthread()
1509 			mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);  in rcu_spawn_cpu_nocb_kthread()
1512 		WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);  in rcu_spawn_cpu_nocb_kthread()
1516 	mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);  in rcu_spawn_cpu_nocb_kthread()
1528 	WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);  in rcu_spawn_cpu_nocb_kthread()
1555 	struct rcu_data *rdp_gp = NULL;  /* Suppress misguided gcc warn. */  in rcu_organize_nocb_kthreads()  local
1575 			rdp_gp = rdp;  in rcu_organize_nocb_kthreads()
1592 		rdp->nocb_gp_rdp = rdp_gp;  in rcu_organize_nocb_kthreads()
1594 			list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);  in rcu_organize_nocb_kthreads()