xref: /openbmc/linux/kernel/rcu/tree_stall.h (revision b181f7029bd71238ac2754ce7052dffd69432085)
1  // SPDX-License-Identifier: GPL-2.0+
2  /*
3   * RCU CPU stall warnings for normal RCU grace periods
4   *
5   * Copyright IBM Corporation, 2019
6   *
7   * Author: Paul E. McKenney <paulmck@linux.ibm.com>
8   */
9  
10  #include <linux/kvm_para.h>
11  
12  //////////////////////////////////////////////////////////////////////////////
13  //
14  // Controlling CPU stall warnings, including delay calculation.
15  
16  /* panic() on RCU Stall sysctl. */
17  int sysctl_panic_on_rcu_stall __read_mostly;
18  int sysctl_max_rcu_stall_to_panic __read_mostly;
19  
20  #ifdef CONFIG_PROVE_RCU
21  #define RCU_STALL_DELAY_DELTA		(5 * HZ)
22  #else
23  #define RCU_STALL_DELAY_DELTA		0
24  #endif
25  #define RCU_STALL_MIGHT_DIV		8
26  #define RCU_STALL_MIGHT_MIN		(2 * HZ)
27  
rcu_exp_jiffies_till_stall_check(void)28  int rcu_exp_jiffies_till_stall_check(void)
29  {
30  	int cpu_stall_timeout = READ_ONCE(rcu_exp_cpu_stall_timeout);
31  	int exp_stall_delay_delta = 0;
32  	int till_stall_check;
33  
34  	// Zero says to use rcu_cpu_stall_timeout, but in milliseconds.
35  	if (!cpu_stall_timeout)
36  		cpu_stall_timeout = jiffies_to_msecs(rcu_jiffies_till_stall_check());
37  
38  	// Limit check must be consistent with the Kconfig limits for
39  	// CONFIG_RCU_EXP_CPU_STALL_TIMEOUT, so check the allowed range.
40  	// The minimum clamped value is "2UL", because at least one full
41  	// tick has to be guaranteed.
42  	till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ);
43  
44  	if (cpu_stall_timeout && jiffies_to_msecs(till_stall_check) != cpu_stall_timeout)
45  		WRITE_ONCE(rcu_exp_cpu_stall_timeout, jiffies_to_msecs(till_stall_check));
46  
47  #ifdef CONFIG_PROVE_RCU
48  	/* Add extra ~25% out of till_stall_check. */
49  	exp_stall_delay_delta = ((till_stall_check * 25) / 100) + 1;
50  #endif
51  
52  	return till_stall_check + exp_stall_delay_delta;
53  }
54  EXPORT_SYMBOL_GPL(rcu_exp_jiffies_till_stall_check);
55  
56  /* Limit-check stall timeouts specified at boottime and runtime. */
rcu_jiffies_till_stall_check(void)57  int rcu_jiffies_till_stall_check(void)
58  {
59  	int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
60  
61  	/*
62  	 * Limit check must be consistent with the Kconfig limits
63  	 * for CONFIG_RCU_CPU_STALL_TIMEOUT.
64  	 */
65  	if (till_stall_check < 3) {
66  		WRITE_ONCE(rcu_cpu_stall_timeout, 3);
67  		till_stall_check = 3;
68  	} else if (till_stall_check > 300) {
69  		WRITE_ONCE(rcu_cpu_stall_timeout, 300);
70  		till_stall_check = 300;
71  	}
72  	return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
73  }
74  EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check);
75  
76  /**
77   * rcu_gp_might_be_stalled - Is it likely that the grace period is stalled?
78   *
79   * Returns @true if the current grace period is sufficiently old that
80   * it is reasonable to assume that it might be stalled.  This can be
81   * useful when deciding whether to allocate memory to enable RCU-mediated
82   * freeing on the one hand or just invoking synchronize_rcu() on the other.
83   * The latter is preferable when the grace period is stalled.
84   *
85   * Note that sampling of the .gp_start and .gp_seq fields must be done
86   * carefully to avoid false positives at the beginnings and ends of
87   * grace periods.
88   */
rcu_gp_might_be_stalled(void)89  bool rcu_gp_might_be_stalled(void)
90  {
91  	unsigned long d = rcu_jiffies_till_stall_check() / RCU_STALL_MIGHT_DIV;
92  	unsigned long j = jiffies;
93  
94  	if (d < RCU_STALL_MIGHT_MIN)
95  		d = RCU_STALL_MIGHT_MIN;
96  	smp_mb(); // jiffies before .gp_seq to avoid false positives.
97  	if (!rcu_gp_in_progress())
98  		return false;
99  	// Long delays at this point avoids false positive, but a delay
100  	// of ULONG_MAX/4 jiffies voids your no-false-positive warranty.
101  	smp_mb(); // .gp_seq before second .gp_start
102  	// And ditto here.
103  	return !time_before(j, READ_ONCE(rcu_state.gp_start) + d);
104  }
105  
106  /* Don't do RCU CPU stall warnings during long sysrq printouts. */
rcu_sysrq_start(void)107  void rcu_sysrq_start(void)
108  {
109  	if (!rcu_cpu_stall_suppress)
110  		rcu_cpu_stall_suppress = 2;
111  }
112  
rcu_sysrq_end(void)113  void rcu_sysrq_end(void)
114  {
115  	if (rcu_cpu_stall_suppress == 2)
116  		rcu_cpu_stall_suppress = 0;
117  }
118  
119  /* Don't print RCU CPU stall warnings during a kernel panic. */
rcu_panic(struct notifier_block * this,unsigned long ev,void * ptr)120  static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
121  {
122  	rcu_cpu_stall_suppress = 1;
123  	return NOTIFY_DONE;
124  }
125  
126  static struct notifier_block rcu_panic_block = {
127  	.notifier_call = rcu_panic,
128  };
129  
check_cpu_stall_init(void)130  static int __init check_cpu_stall_init(void)
131  {
132  	atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
133  	return 0;
134  }
135  early_initcall(check_cpu_stall_init);
136  
137  /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
panic_on_rcu_stall(void)138  static void panic_on_rcu_stall(void)
139  {
140  	static int cpu_stall;
141  
142  	if (++cpu_stall < sysctl_max_rcu_stall_to_panic)
143  		return;
144  
145  	if (sysctl_panic_on_rcu_stall)
146  		panic("RCU Stall\n");
147  }
148  
149  /**
150   * rcu_cpu_stall_reset - restart stall-warning timeout for current grace period
151   *
152   * To perform the reset request from the caller, disable stall detection until
153   * 3 fqs loops have passed. This is required to ensure a fresh jiffies is
154   * loaded.  It should be safe to do from the fqs loop as enough timer
155   * interrupts and context switches should have passed.
156   *
157   * The caller must disable hard irqs.
158   */
rcu_cpu_stall_reset(void)159  void rcu_cpu_stall_reset(void)
160  {
161  	WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 3);
162  	WRITE_ONCE(rcu_state.jiffies_stall, ULONG_MAX);
163  }
164  
165  //////////////////////////////////////////////////////////////////////////////
166  //
167  // Interaction with RCU grace periods
168  
169  /* Start of new grace period, so record stall time (and forcing times). */
record_gp_stall_check_time(void)170  static void record_gp_stall_check_time(void)
171  {
172  	unsigned long j = jiffies;
173  	unsigned long j1;
174  
175  	WRITE_ONCE(rcu_state.gp_start, j);
176  	j1 = rcu_jiffies_till_stall_check();
177  	smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq.
178  	WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 0);
179  	WRITE_ONCE(rcu_state.jiffies_stall, j + j1);
180  	rcu_state.jiffies_resched = j + j1 / 2;
181  	rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);
182  }
183  
184  /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */
zero_cpu_stall_ticks(struct rcu_data * rdp)185  static void zero_cpu_stall_ticks(struct rcu_data *rdp)
186  {
187  	rdp->ticks_this_gp = 0;
188  	rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
189  	WRITE_ONCE(rdp->last_fqs_resched, jiffies);
190  }
191  
192  /*
193   * If too much time has passed in the current grace period, and if
194   * so configured, go kick the relevant kthreads.
195   */
rcu_stall_kick_kthreads(void)196  static void rcu_stall_kick_kthreads(void)
197  {
198  	unsigned long j;
199  
200  	if (!READ_ONCE(rcu_kick_kthreads))
201  		return;
202  	j = READ_ONCE(rcu_state.jiffies_kick_kthreads);
203  	if (time_after(jiffies, j) && rcu_state.gp_kthread &&
204  	    (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {
205  		WARN_ONCE(1, "Kicking %s grace-period kthread\n",
206  			  rcu_state.name);
207  		rcu_ftrace_dump(DUMP_ALL);
208  		wake_up_process(rcu_state.gp_kthread);
209  		WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);
210  	}
211  }
212  
213  /*
214   * Handler for the irq_work request posted about halfway into the RCU CPU
215   * stall timeout, and used to detect excessive irq disabling.  Set state
216   * appropriately, but just complain if there is unexpected state on entry.
217   */
rcu_iw_handler(struct irq_work * iwp)218  static void rcu_iw_handler(struct irq_work *iwp)
219  {
220  	struct rcu_data *rdp;
221  	struct rcu_node *rnp;
222  
223  	rdp = container_of(iwp, struct rcu_data, rcu_iw);
224  	rnp = rdp->mynode;
225  	raw_spin_lock_rcu_node(rnp);
226  	if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
227  		rdp->rcu_iw_gp_seq = rnp->gp_seq;
228  		rdp->rcu_iw_pending = false;
229  	}
230  	raw_spin_unlock_rcu_node(rnp);
231  }
232  
233  //////////////////////////////////////////////////////////////////////////////
234  //
235  // Printing RCU CPU stall warnings
236  
237  #ifdef CONFIG_PREEMPT_RCU
238  
239  /*
240   * Dump detailed information for all tasks blocking the current RCU
241   * grace period on the specified rcu_node structure.
242   */
rcu_print_detail_task_stall_rnp(struct rcu_node * rnp)243  static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
244  {
245  	unsigned long flags;
246  	struct task_struct *t;
247  
248  	raw_spin_lock_irqsave_rcu_node(rnp, flags);
249  	if (!rcu_preempt_blocked_readers_cgp(rnp)) {
250  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
251  		return;
252  	}
253  	t = list_entry(rnp->gp_tasks->prev,
254  		       struct task_struct, rcu_node_entry);
255  	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
256  		/*
257  		 * We could be printing a lot while holding a spinlock.
258  		 * Avoid triggering hard lockup.
259  		 */
260  		touch_nmi_watchdog();
261  		sched_show_task(t);
262  	}
263  	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
264  }
265  
266  // Communicate task state back to the RCU CPU stall warning request.
267  struct rcu_stall_chk_rdr {
268  	int nesting;
269  	union rcu_special rs;
270  	bool on_blkd_list;
271  };
272  
273  /*
274   * Report out the state of a not-running task that is stalling the
275   * current RCU grace period.
276   */
check_slow_task(struct task_struct * t,void * arg)277  static int check_slow_task(struct task_struct *t, void *arg)
278  {
279  	struct rcu_stall_chk_rdr *rscrp = arg;
280  
281  	if (task_curr(t))
282  		return -EBUSY; // It is running, so decline to inspect it.
283  	rscrp->nesting = t->rcu_read_lock_nesting;
284  	rscrp->rs = t->rcu_read_unlock_special;
285  	rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry);
286  	return 0;
287  }
288  
289  /*
290   * Scan the current list of tasks blocked within RCU read-side critical
291   * sections, printing out the tid of each of the first few of them.
292   */
rcu_print_task_stall(struct rcu_node * rnp,unsigned long flags)293  static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
294  	__releases(rnp->lock)
295  {
296  	int i = 0;
297  	int ndetected = 0;
298  	struct rcu_stall_chk_rdr rscr;
299  	struct task_struct *t;
300  	struct task_struct *ts[8];
301  
302  	lockdep_assert_irqs_disabled();
303  	if (!rcu_preempt_blocked_readers_cgp(rnp)) {
304  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
305  		return 0;
306  	}
307  	pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
308  	       rnp->level, rnp->grplo, rnp->grphi);
309  	t = list_entry(rnp->gp_tasks->prev,
310  		       struct task_struct, rcu_node_entry);
311  	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
312  		get_task_struct(t);
313  		ts[i++] = t;
314  		if (i >= ARRAY_SIZE(ts))
315  			break;
316  	}
317  	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
318  	while (i) {
319  		t = ts[--i];
320  		if (task_call_func(t, check_slow_task, &rscr))
321  			pr_cont(" P%d", t->pid);
322  		else
323  			pr_cont(" P%d/%d:%c%c%c%c",
324  				t->pid, rscr.nesting,
325  				".b"[rscr.rs.b.blocked],
326  				".q"[rscr.rs.b.need_qs],
327  				".e"[rscr.rs.b.exp_hint],
328  				".l"[rscr.on_blkd_list]);
329  		lockdep_assert_irqs_disabled();
330  		put_task_struct(t);
331  		ndetected++;
332  	}
333  	pr_cont("\n");
334  	return ndetected;
335  }
336  
337  #else /* #ifdef CONFIG_PREEMPT_RCU */
338  
339  /*
340   * Because preemptible RCU does not exist, we never have to check for
341   * tasks blocked within RCU read-side critical sections.
342   */
rcu_print_detail_task_stall_rnp(struct rcu_node * rnp)343  static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
344  {
345  }
346  
347  /*
348   * Because preemptible RCU does not exist, we never have to check for
349   * tasks blocked within RCU read-side critical sections.
350   */
rcu_print_task_stall(struct rcu_node * rnp,unsigned long flags)351  static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
352  	__releases(rnp->lock)
353  {
354  	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
355  	return 0;
356  }
357  #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
358  
359  /*
360   * Dump stacks of all tasks running on stalled CPUs.  First try using
361   * NMIs, but fall back to manual remote stack tracing on architectures
362   * that don't support NMI-based stack dumps.  The NMI-triggered stack
363   * traces are more accurate because they are printed by the target CPU.
364   */
rcu_dump_cpu_stacks(void)365  static void rcu_dump_cpu_stacks(void)
366  {
367  	int cpu;
368  	unsigned long flags;
369  	struct rcu_node *rnp;
370  
371  	rcu_for_each_leaf_node(rnp) {
372  		raw_spin_lock_irqsave_rcu_node(rnp, flags);
373  		for_each_leaf_node_possible_cpu(rnp, cpu)
374  			if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
375  				if (cpu_is_offline(cpu))
376  					pr_err("Offline CPU %d blocking current GP.\n", cpu);
377  				else
378  					dump_cpu_task(cpu);
379  			}
380  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
381  	}
382  }
383  
384  static const char * const gp_state_names[] = {
385  	[RCU_GP_IDLE] = "RCU_GP_IDLE",
386  	[RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS",
387  	[RCU_GP_DONE_GPS] = "RCU_GP_DONE_GPS",
388  	[RCU_GP_ONOFF] = "RCU_GP_ONOFF",
389  	[RCU_GP_INIT] = "RCU_GP_INIT",
390  	[RCU_GP_WAIT_FQS] = "RCU_GP_WAIT_FQS",
391  	[RCU_GP_DOING_FQS] = "RCU_GP_DOING_FQS",
392  	[RCU_GP_CLEANUP] = "RCU_GP_CLEANUP",
393  	[RCU_GP_CLEANED] = "RCU_GP_CLEANED",
394  };
395  
396  /*
397   * Convert a ->gp_state value to a character string.
398   */
gp_state_getname(short gs)399  static const char *gp_state_getname(short gs)
400  {
401  	if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names))
402  		return "???";
403  	return gp_state_names[gs];
404  }
405  
406  /* Is the RCU grace-period kthread being starved of CPU time? */
rcu_is_gp_kthread_starving(unsigned long * jp)407  static bool rcu_is_gp_kthread_starving(unsigned long *jp)
408  {
409  	unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity);
410  
411  	if (jp)
412  		*jp = j;
413  	return j > 2 * HZ;
414  }
415  
rcu_is_rcuc_kthread_starving(struct rcu_data * rdp,unsigned long * jp)416  static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp)
417  {
418  	int cpu;
419  	struct task_struct *rcuc;
420  	unsigned long j;
421  
422  	rcuc = rdp->rcu_cpu_kthread_task;
423  	if (!rcuc)
424  		return false;
425  
426  	cpu = task_cpu(rcuc);
427  	if (cpu_is_offline(cpu) || idle_cpu(cpu))
428  		return false;
429  
430  	j = jiffies - READ_ONCE(rdp->rcuc_activity);
431  
432  	if (jp)
433  		*jp = j;
434  	return j > 2 * HZ;
435  }
436  
print_cpu_stat_info(int cpu)437  static void print_cpu_stat_info(int cpu)
438  {
439  	struct rcu_snap_record rsr, *rsrp;
440  	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
441  	struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu);
442  
443  	if (!rcu_cpu_stall_cputime)
444  		return;
445  
446  	rsrp = &rdp->snap_record;
447  	if (rsrp->gp_seq != rdp->gp_seq)
448  		return;
449  
450  	rsr.cputime_irq     = kcpustat_field(kcsp, CPUTIME_IRQ, cpu);
451  	rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu);
452  	rsr.cputime_system  = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu);
453  
454  	pr_err("\t         hardirqs   softirqs   csw/system\n");
455  	pr_err("\t number: %8ld %10d %12lld\n",
456  		kstat_cpu_irqs_sum(cpu) - rsrp->nr_hardirqs,
457  		kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs,
458  		nr_context_switches_cpu(cpu) - rsrp->nr_csw);
459  	pr_err("\tcputime: %8lld %10lld %12lld   ==> %d(ms)\n",
460  		div_u64(rsr.cputime_irq - rsrp->cputime_irq, NSEC_PER_MSEC),
461  		div_u64(rsr.cputime_softirq - rsrp->cputime_softirq, NSEC_PER_MSEC),
462  		div_u64(rsr.cputime_system - rsrp->cputime_system, NSEC_PER_MSEC),
463  		jiffies_to_msecs(jiffies - rsrp->jiffies));
464  }
465  
466  /*
467   * Print out diagnostic information for the specified stalled CPU.
468   *
469   * If the specified CPU is aware of the current RCU grace period, then
470   * print the number of scheduling clock interrupts the CPU has taken
471   * during the time that it has been aware.  Otherwise, print the number
472   * of RCU grace periods that this CPU is ignorant of, for example, "1"
473   * if the CPU was aware of the previous grace period.
474   *
475   * Also print out idle info.
476   */
print_cpu_stall_info(int cpu)477  static void print_cpu_stall_info(int cpu)
478  {
479  	unsigned long delta;
480  	bool falsepositive;
481  	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
482  	char *ticks_title;
483  	unsigned long ticks_value;
484  	bool rcuc_starved;
485  	unsigned long j;
486  	char buf[32];
487  
488  	/*
489  	 * We could be printing a lot while holding a spinlock.  Avoid
490  	 * triggering hard lockup.
491  	 */
492  	touch_nmi_watchdog();
493  
494  	ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);
495  	if (ticks_value) {
496  		ticks_title = "GPs behind";
497  	} else {
498  		ticks_title = "ticks this GP";
499  		ticks_value = rdp->ticks_this_gp;
500  	}
501  	delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
502  	falsepositive = rcu_is_gp_kthread_starving(NULL) &&
503  			rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu));
504  	rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j);
505  	if (rcuc_starved)
506  		// Print signed value, as negative values indicate a probable bug.
507  		snprintf(buf, sizeof(buf), " rcuc=%ld jiffies(starved)", j);
508  	pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%04x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n",
509  	       cpu,
510  	       "O."[!!cpu_online(cpu)],
511  	       "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
512  	       "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
513  	       !IS_ENABLED(CONFIG_IRQ_WORK) ? '?' :
514  			rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' :
515  				"!."[!delta],
516  	       ticks_value, ticks_title,
517  	       rcu_dynticks_snap(cpu) & 0xffff,
518  	       ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu),
519  	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
520  	       data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
521  	       rcuc_starved ? buf : "",
522  	       falsepositive ? " (false positive?)" : "");
523  
524  	print_cpu_stat_info(cpu);
525  }
526  
527  /* Complain about starvation of grace-period kthread.  */
rcu_check_gp_kthread_starvation(void)528  static void rcu_check_gp_kthread_starvation(void)
529  {
530  	int cpu;
531  	struct task_struct *gpk = rcu_state.gp_kthread;
532  	unsigned long j;
533  
534  	if (rcu_is_gp_kthread_starving(&j)) {
535  		cpu = gpk ? task_cpu(gpk) : -1;
536  		pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n",
537  		       rcu_state.name, j,
538  		       (long)rcu_seq_current(&rcu_state.gp_seq),
539  		       data_race(READ_ONCE(rcu_state.gp_flags)),
540  		       gp_state_getname(rcu_state.gp_state),
541  		       data_race(READ_ONCE(rcu_state.gp_state)),
542  		       gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu);
543  		if (gpk) {
544  			pr_err("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name);
545  			pr_err("RCU grace-period kthread stack dump:\n");
546  			sched_show_task(gpk);
547  			if (cpu >= 0) {
548  				if (cpu_is_offline(cpu)) {
549  					pr_err("RCU GP kthread last ran on offline CPU %d.\n", cpu);
550  				} else  {
551  					pr_err("Stack dump where RCU GP kthread last ran:\n");
552  					dump_cpu_task(cpu);
553  				}
554  			}
555  			wake_up_process(gpk);
556  		}
557  	}
558  }
559  
560  /* Complain about missing wakeups from expired fqs wait timer */
rcu_check_gp_kthread_expired_fqs_timer(void)561  static void rcu_check_gp_kthread_expired_fqs_timer(void)
562  {
563  	struct task_struct *gpk = rcu_state.gp_kthread;
564  	short gp_state;
565  	unsigned long jiffies_fqs;
566  	int cpu;
567  
568  	/*
569  	 * Order reads of .gp_state and .jiffies_force_qs.
570  	 * Matching smp_wmb() is present in rcu_gp_fqs_loop().
571  	 */
572  	gp_state = smp_load_acquire(&rcu_state.gp_state);
573  	jiffies_fqs = READ_ONCE(rcu_state.jiffies_force_qs);
574  
575  	if (gp_state == RCU_GP_WAIT_FQS &&
576  	    time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) &&
577  	    gpk && !READ_ONCE(gpk->on_rq)) {
578  		cpu = task_cpu(gpk);
579  		pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n",
580  		       rcu_state.name, (jiffies - jiffies_fqs),
581  		       (long)rcu_seq_current(&rcu_state.gp_seq),
582  		       data_race(rcu_state.gp_flags),
583  		       gp_state_getname(RCU_GP_WAIT_FQS), RCU_GP_WAIT_FQS,
584  		       data_race(READ_ONCE(gpk->__state)));
585  		pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n",
586  		       cpu, kstat_softirqs_cpu(TIMER_SOFTIRQ, cpu));
587  	}
588  }
589  
print_other_cpu_stall(unsigned long gp_seq,unsigned long gps)590  static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
591  {
592  	int cpu;
593  	unsigned long flags;
594  	unsigned long gpa;
595  	unsigned long j;
596  	int ndetected = 0;
597  	struct rcu_node *rnp;
598  	long totqlen = 0;
599  
600  	lockdep_assert_irqs_disabled();
601  
602  	/* Kick and suppress, if so configured. */
603  	rcu_stall_kick_kthreads();
604  	if (rcu_stall_is_suppressed())
605  		return;
606  
607  	/*
608  	 * OK, time to rat on our buddy...
609  	 * See Documentation/RCU/stallwarn.rst for info on how to debug
610  	 * RCU CPU stall warnings.
611  	 */
612  	trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected"));
613  	pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);
614  	rcu_for_each_leaf_node(rnp) {
615  		raw_spin_lock_irqsave_rcu_node(rnp, flags);
616  		if (rnp->qsmask != 0) {
617  			for_each_leaf_node_possible_cpu(rnp, cpu)
618  				if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
619  					print_cpu_stall_info(cpu);
620  					ndetected++;
621  				}
622  		}
623  		ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock.
624  		lockdep_assert_irqs_disabled();
625  	}
626  
627  	for_each_possible_cpu(cpu)
628  		totqlen += rcu_get_n_cbs_cpu(cpu);
629  	pr_err("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu ncpus=%d)\n",
630  	       smp_processor_id(), (long)(jiffies - gps),
631  	       (long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus);
632  	if (ndetected) {
633  		rcu_dump_cpu_stacks();
634  
635  		/* Complain about tasks blocking the grace period. */
636  		rcu_for_each_leaf_node(rnp)
637  			rcu_print_detail_task_stall_rnp(rnp);
638  	} else {
639  		if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {
640  			pr_err("INFO: Stall ended before state dump start\n");
641  		} else {
642  			j = jiffies;
643  			gpa = data_race(READ_ONCE(rcu_state.gp_activity));
644  			pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
645  			       rcu_state.name, j - gpa, j, gpa,
646  			       data_race(READ_ONCE(jiffies_till_next_fqs)),
647  			       data_race(READ_ONCE(rcu_get_root()->qsmask)));
648  		}
649  	}
650  	/* Rewrite if needed in case of slow consoles. */
651  	if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
652  		WRITE_ONCE(rcu_state.jiffies_stall,
653  			   jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
654  
655  	rcu_check_gp_kthread_expired_fqs_timer();
656  	rcu_check_gp_kthread_starvation();
657  
658  	panic_on_rcu_stall();
659  
660  	rcu_force_quiescent_state();  /* Kick them all. */
661  }
662  
print_cpu_stall(unsigned long gps)663  static void print_cpu_stall(unsigned long gps)
664  {
665  	int cpu;
666  	unsigned long flags;
667  	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
668  	struct rcu_node *rnp = rcu_get_root();
669  	long totqlen = 0;
670  
671  	lockdep_assert_irqs_disabled();
672  
673  	/* Kick and suppress, if so configured. */
674  	rcu_stall_kick_kthreads();
675  	if (rcu_stall_is_suppressed())
676  		return;
677  
678  	/*
679  	 * OK, time to rat on ourselves...
680  	 * See Documentation/RCU/stallwarn.rst for info on how to debug
681  	 * RCU CPU stall warnings.
682  	 */
683  	trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected"));
684  	pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);
685  	raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
686  	print_cpu_stall_info(smp_processor_id());
687  	raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
688  	for_each_possible_cpu(cpu)
689  		totqlen += rcu_get_n_cbs_cpu(cpu);
690  	pr_err("\t(t=%lu jiffies g=%ld q=%lu ncpus=%d)\n",
691  		jiffies - gps,
692  		(long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus);
693  
694  	rcu_check_gp_kthread_expired_fqs_timer();
695  	rcu_check_gp_kthread_starvation();
696  
697  	rcu_dump_cpu_stacks();
698  
699  	raw_spin_lock_irqsave_rcu_node(rnp, flags);
700  	/* Rewrite if needed in case of slow consoles. */
701  	if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
702  		WRITE_ONCE(rcu_state.jiffies_stall,
703  			   jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
704  	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
705  
706  	panic_on_rcu_stall();
707  
708  	/*
709  	 * Attempt to revive the RCU machinery by forcing a context switch.
710  	 *
711  	 * A context switch would normally allow the RCU state machine to make
712  	 * progress and it could be we're stuck in kernel space without context
713  	 * switches for an entirely unreasonable amount of time.
714  	 */
715  	set_tsk_need_resched(current);
716  	set_preempt_need_resched();
717  }
718  
check_cpu_stall(struct rcu_data * rdp)719  static void check_cpu_stall(struct rcu_data *rdp)
720  {
721  	bool didstall = false;
722  	unsigned long gs1;
723  	unsigned long gs2;
724  	unsigned long gps;
725  	unsigned long j;
726  	unsigned long jn;
727  	unsigned long js;
728  	struct rcu_node *rnp;
729  
730  	lockdep_assert_irqs_disabled();
731  	if ((rcu_stall_is_suppressed() && !READ_ONCE(rcu_kick_kthreads)) ||
732  	    !rcu_gp_in_progress())
733  		return;
734  	rcu_stall_kick_kthreads();
735  
736  	/*
737  	 * Check if it was requested (via rcu_cpu_stall_reset()) that the FQS
738  	 * loop has to set jiffies to ensure a non-stale jiffies value. This
739  	 * is required to have good jiffies value after coming out of long
740  	 * breaks of jiffies updates. Not doing so can cause false positives.
741  	 */
742  	if (READ_ONCE(rcu_state.nr_fqs_jiffies_stall) > 0)
743  		return;
744  
745  	j = jiffies;
746  
747  	/*
748  	 * Lots of memory barriers to reject false positives.
749  	 *
750  	 * The idea is to pick up rcu_state.gp_seq, then
751  	 * rcu_state.jiffies_stall, then rcu_state.gp_start, and finally
752  	 * another copy of rcu_state.gp_seq.  These values are updated in
753  	 * the opposite order with memory barriers (or equivalent) during
754  	 * grace-period initialization and cleanup.  Now, a false positive
755  	 * can occur if we get an new value of rcu_state.gp_start and a old
756  	 * value of rcu_state.jiffies_stall.  But given the memory barriers,
757  	 * the only way that this can happen is if one grace period ends
758  	 * and another starts between these two fetches.  This is detected
759  	 * by comparing the second fetch of rcu_state.gp_seq with the
760  	 * previous fetch from rcu_state.gp_seq.
761  	 *
762  	 * Given this check, comparisons of jiffies, rcu_state.jiffies_stall,
763  	 * and rcu_state.gp_start suffice to forestall false positives.
764  	 */
765  	gs1 = READ_ONCE(rcu_state.gp_seq);
766  	smp_rmb(); /* Pick up ->gp_seq first... */
767  	js = READ_ONCE(rcu_state.jiffies_stall);
768  	smp_rmb(); /* ...then ->jiffies_stall before the rest... */
769  	gps = READ_ONCE(rcu_state.gp_start);
770  	smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
771  	gs2 = READ_ONCE(rcu_state.gp_seq);
772  	if (gs1 != gs2 ||
773  	    ULONG_CMP_LT(j, js) ||
774  	    ULONG_CMP_GE(gps, js))
775  		return; /* No stall or GP completed since entering function. */
776  	rnp = rdp->mynode;
777  	jn = jiffies + ULONG_MAX / 2;
778  	if (rcu_gp_in_progress() &&
779  	    (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
780  	    cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
781  
782  		/*
783  		 * If a virtual machine is stopped by the host it can look to
784  		 * the watchdog like an RCU stall. Check to see if the host
785  		 * stopped the vm.
786  		 */
787  		if (kvm_check_and_clear_guest_paused())
788  			return;
789  
790  		/* We haven't checked in, so go dump stack. */
791  		print_cpu_stall(gps);
792  		if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
793  			rcu_ftrace_dump(DUMP_ALL);
794  		didstall = true;
795  
796  	} else if (rcu_gp_in_progress() &&
797  		   ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
798  		   cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
799  
800  		/*
801  		 * If a virtual machine is stopped by the host it can look to
802  		 * the watchdog like an RCU stall. Check to see if the host
803  		 * stopped the vm.
804  		 */
805  		if (kvm_check_and_clear_guest_paused())
806  			return;
807  
808  		/* They had a few time units to dump stack, so complain. */
809  		print_other_cpu_stall(gs2, gps);
810  		if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
811  			rcu_ftrace_dump(DUMP_ALL);
812  		didstall = true;
813  	}
814  	if (didstall && READ_ONCE(rcu_state.jiffies_stall) == jn) {
815  		jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
816  		WRITE_ONCE(rcu_state.jiffies_stall, jn);
817  	}
818  }
819  
820  //////////////////////////////////////////////////////////////////////////////
821  //
822  // RCU forward-progress mechanisms, including of callback invocation.
823  
824  
825  /*
826   * Check to see if a failure to end RCU priority inversion was due to
827   * a CPU not passing through a quiescent state.  When this happens, there
828   * is nothing that RCU priority boosting can do to help, so we shouldn't
829   * count this as an RCU priority boosting failure.  A return of true says
830   * RCU priority boosting is to blame, and false says otherwise.  If false
831   * is returned, the first of the CPUs to blame is stored through cpup.
832   * If there was no CPU blocking the current grace period, but also nothing
833   * in need of being boosted, *cpup is set to -1.  This can happen in case
834   * of vCPU preemption while the last CPU is reporting its quiscent state,
835   * for example.
836   *
837   * If cpup is NULL, then a lockless quick check is carried out, suitable
838   * for high-rate usage.  On the other hand, if cpup is non-NULL, each
839   * rcu_node structure's ->lock is acquired, ruling out high-rate usage.
840   */
rcu_check_boost_fail(unsigned long gp_state,int * cpup)841  bool rcu_check_boost_fail(unsigned long gp_state, int *cpup)
842  {
843  	bool atb = false;
844  	int cpu;
845  	unsigned long flags;
846  	struct rcu_node *rnp;
847  
848  	rcu_for_each_leaf_node(rnp) {
849  		if (!cpup) {
850  			if (data_race(READ_ONCE(rnp->qsmask))) {
851  				return false;
852  			} else {
853  				if (READ_ONCE(rnp->gp_tasks))
854  					atb = true;
855  				continue;
856  			}
857  		}
858  		*cpup = -1;
859  		raw_spin_lock_irqsave_rcu_node(rnp, flags);
860  		if (rnp->gp_tasks)
861  			atb = true;
862  		if (!rnp->qsmask) {
863  			// No CPUs without quiescent states for this rnp.
864  			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
865  			continue;
866  		}
867  		// Find the first holdout CPU.
868  		for_each_leaf_node_possible_cpu(rnp, cpu) {
869  			if (rnp->qsmask & (1UL << (cpu - rnp->grplo))) {
870  				raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
871  				*cpup = cpu;
872  				return false;
873  			}
874  		}
875  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
876  	}
877  	// Can't blame CPUs, so must blame RCU priority boosting.
878  	return atb;
879  }
880  EXPORT_SYMBOL_GPL(rcu_check_boost_fail);
881  
882  /*
883   * Show the state of the grace-period kthreads.
884   */
show_rcu_gp_kthreads(void)885  void show_rcu_gp_kthreads(void)
886  {
887  	unsigned long cbs = 0;
888  	int cpu;
889  	unsigned long j;
890  	unsigned long ja;
891  	unsigned long jr;
892  	unsigned long js;
893  	unsigned long jw;
894  	struct rcu_data *rdp;
895  	struct rcu_node *rnp;
896  	struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
897  
898  	j = jiffies;
899  	ja = j - data_race(READ_ONCE(rcu_state.gp_activity));
900  	jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity));
901  	js = j - data_race(READ_ONCE(rcu_state.gp_start));
902  	jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time));
903  	pr_info("%s: wait state: %s(%d) ->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n",
904  		rcu_state.name, gp_state_getname(rcu_state.gp_state),
905  		data_race(READ_ONCE(rcu_state.gp_state)),
906  		t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU,
907  		js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)),
908  		(long)data_race(READ_ONCE(rcu_state.gp_seq)),
909  		(long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)),
910  		data_race(READ_ONCE(rcu_state.gp_max)),
911  		data_race(READ_ONCE(rcu_state.gp_flags)));
912  	rcu_for_each_node_breadth_first(rnp) {
913  		if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) &&
914  		    !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) &&
915  		    !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks)))
916  			continue;
917  		pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n",
918  			rnp->grplo, rnp->grphi,
919  			(long)data_race(READ_ONCE(rnp->gp_seq)),
920  			(long)data_race(READ_ONCE(rnp->gp_seq_needed)),
921  			data_race(READ_ONCE(rnp->qsmask)),
922  			".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))],
923  			".B"[!!data_race(READ_ONCE(rnp->boost_tasks))],
924  			".E"[!!data_race(READ_ONCE(rnp->exp_tasks))],
925  			".G"[!!data_race(READ_ONCE(rnp->gp_tasks))],
926  			data_race(READ_ONCE(rnp->n_boosts)));
927  		if (!rcu_is_leaf_node(rnp))
928  			continue;
929  		for_each_leaf_node_possible_cpu(rnp, cpu) {
930  			rdp = per_cpu_ptr(&rcu_data, cpu);
931  			if (READ_ONCE(rdp->gpwrap) ||
932  			    ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
933  					 READ_ONCE(rdp->gp_seq_needed)))
934  				continue;
935  			pr_info("\tcpu %d ->gp_seq_needed %ld\n",
936  				cpu, (long)data_race(READ_ONCE(rdp->gp_seq_needed)));
937  		}
938  	}
939  	for_each_possible_cpu(cpu) {
940  		rdp = per_cpu_ptr(&rcu_data, cpu);
941  		cbs += data_race(READ_ONCE(rdp->n_cbs_invoked));
942  		if (rcu_segcblist_is_offloaded(&rdp->cblist))
943  			show_rcu_nocb_state(rdp);
944  	}
945  	pr_info("RCU callbacks invoked since boot: %lu\n", cbs);
946  	show_rcu_tasks_gp_kthreads();
947  }
948  EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
949  
950  /*
951   * This function checks for grace-period requests that fail to motivate
952   * RCU to come out of its idle mode.
953   */
rcu_check_gp_start_stall(struct rcu_node * rnp,struct rcu_data * rdp,const unsigned long gpssdelay)954  static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
955  				     const unsigned long gpssdelay)
956  {
957  	unsigned long flags;
958  	unsigned long j;
959  	struct rcu_node *rnp_root = rcu_get_root();
960  	static atomic_t warned = ATOMIC_INIT(0);
961  
962  	if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
963  	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
964  			 READ_ONCE(rnp_root->gp_seq_needed)) ||
965  	    !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
966  		return;
967  	j = jiffies; /* Expensive access, and in common case don't get here. */
968  	if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
969  	    time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
970  	    atomic_read(&warned))
971  		return;
972  
973  	raw_spin_lock_irqsave_rcu_node(rnp, flags);
974  	j = jiffies;
975  	if (rcu_gp_in_progress() ||
976  	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
977  			 READ_ONCE(rnp_root->gp_seq_needed)) ||
978  	    time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
979  	    time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
980  	    atomic_read(&warned)) {
981  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
982  		return;
983  	}
984  	/* Hold onto the leaf lock to make others see warned==1. */
985  
986  	if (rnp_root != rnp)
987  		raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
988  	j = jiffies;
989  	if (rcu_gp_in_progress() ||
990  	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
991  			 READ_ONCE(rnp_root->gp_seq_needed)) ||
992  	    time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
993  	    time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
994  	    atomic_xchg(&warned, 1)) {
995  		if (rnp_root != rnp)
996  			/* irqs remain disabled. */
997  			raw_spin_unlock_rcu_node(rnp_root);
998  		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
999  		return;
1000  	}
1001  	WARN_ON(1);
1002  	if (rnp_root != rnp)
1003  		raw_spin_unlock_rcu_node(rnp_root);
1004  	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1005  	show_rcu_gp_kthreads();
1006  }
1007  
1008  /*
1009   * Do a forward-progress check for rcutorture.  This is normally invoked
1010   * due to an OOM event.  The argument "j" gives the time period during
1011   * which rcutorture would like progress to have been made.
1012   */
rcu_fwd_progress_check(unsigned long j)1013  void rcu_fwd_progress_check(unsigned long j)
1014  {
1015  	unsigned long cbs;
1016  	int cpu;
1017  	unsigned long max_cbs = 0;
1018  	int max_cpu = -1;
1019  	struct rcu_data *rdp;
1020  
1021  	if (rcu_gp_in_progress()) {
1022  		pr_info("%s: GP age %lu jiffies\n",
1023  			__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start)));
1024  		show_rcu_gp_kthreads();
1025  	} else {
1026  		pr_info("%s: Last GP end %lu jiffies ago\n",
1027  			__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end)));
1028  		preempt_disable();
1029  		rdp = this_cpu_ptr(&rcu_data);
1030  		rcu_check_gp_start_stall(rdp->mynode, rdp, j);
1031  		preempt_enable();
1032  	}
1033  	for_each_possible_cpu(cpu) {
1034  		cbs = rcu_get_n_cbs_cpu(cpu);
1035  		if (!cbs)
1036  			continue;
1037  		if (max_cpu < 0)
1038  			pr_info("%s: callbacks", __func__);
1039  		pr_cont(" %d: %lu", cpu, cbs);
1040  		if (cbs <= max_cbs)
1041  			continue;
1042  		max_cbs = cbs;
1043  		max_cpu = cpu;
1044  	}
1045  	if (max_cpu >= 0)
1046  		pr_cont("\n");
1047  }
1048  EXPORT_SYMBOL_GPL(rcu_fwd_progress_check);
1049  
1050  /* Commandeer a sysrq key to dump RCU's tree. */
1051  static bool sysrq_rcu;
1052  module_param(sysrq_rcu, bool, 0444);
1053  
1054  /* Dump grace-period-request information due to commandeered sysrq. */
sysrq_show_rcu(u8 key)1055  static void sysrq_show_rcu(u8 key)
1056  {
1057  	show_rcu_gp_kthreads();
1058  }
1059  
1060  static const struct sysrq_key_op sysrq_rcudump_op = {
1061  	.handler = sysrq_show_rcu,
1062  	.help_msg = "show-rcu(y)",
1063  	.action_msg = "Show RCU tree",
1064  	.enable_mask = SYSRQ_ENABLE_DUMP,
1065  };
1066  
rcu_sysrq_init(void)1067  static int __init rcu_sysrq_init(void)
1068  {
1069  	if (sysrq_rcu)
1070  		return register_sysrq_key('y', &sysrq_rcudump_op);
1071  	return 0;
1072  }
1073  early_initcall(rcu_sysrq_init);
1074