tree.c (5d8a752e31aaa4c9703f201956a40b45ed791217) tree.c (10462d6f58fb6dbde7563e9343505d98d5bfba3d)
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright IBM Corporation, 2008
6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>

--- 135 unchanged lines hidden (view full) ---

144static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
145static void invoke_rcu_core(void);
146static void invoke_rcu_callbacks(struct rcu_data *rdp);
147static void rcu_report_exp_rdp(struct rcu_data *rdp);
148static void sync_sched_exp_online_cleanup(int cpu);
149
150/* rcuc/rcub kthread realtime priority */
151static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright IBM Corporation, 2008
6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>

--- 135 unchanged lines hidden (view full) ---

144static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
145static void invoke_rcu_core(void);
146static void invoke_rcu_callbacks(struct rcu_data *rdp);
147static void rcu_report_exp_rdp(struct rcu_data *rdp);
148static void sync_sched_exp_online_cleanup(int cpu);
149
150/* rcuc/rcub kthread realtime priority */
151static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
152module_param(kthread_prio, int, 0444);
152module_param(kthread_prio, int, 0644);
153
154/* Delay in jiffies for grace-period initialization delays, debug only. */
155
156static int gp_preinit_delay;
157module_param(gp_preinit_delay, int, 0444);
158static int gp_init_delay;
159module_param(gp_init_delay, int, 0444);
160static int gp_cleanup_delay;

--- 240 unchanged lines hidden (view full) ---

401static bool rcu_kick_kthreads;
402
403/*
404 * How long the grace period must be before we start recruiting
405 * quiescent-state help from rcu_note_context_switch().
406 */
407static ulong jiffies_till_sched_qs = ULONG_MAX;
408module_param(jiffies_till_sched_qs, ulong, 0444);
153
154/* Delay in jiffies for grace-period initialization delays, debug only. */
155
156static int gp_preinit_delay;
157module_param(gp_preinit_delay, int, 0444);
158static int gp_init_delay;
159module_param(gp_init_delay, int, 0444);
160static int gp_cleanup_delay;

--- 240 unchanged lines hidden (view full) ---

401static bool rcu_kick_kthreads;
402
403/*
404 * How long the grace period must be before we start recruiting
405 * quiescent-state help from rcu_note_context_switch().
406 */
407static ulong jiffies_till_sched_qs = ULONG_MAX;
408module_param(jiffies_till_sched_qs, ulong, 0444);
409static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
409static ulong jiffies_to_sched_qs; /* Adjusted version of above if not default */
410module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
411
412/*
413 * Make sure that we give the grace-period kthread time to detect any
414 * idle CPUs before taking active measures to force quiescent states.
415 * However, don't go below 100 milliseconds, adjusted upwards for really
416 * large systems.
417 */
418static void adjust_jiffies_till_sched_qs(void)
419{
420 unsigned long j;
421
422 /* If jiffies_till_sched_qs was specified, respect the request. */
423 if (jiffies_till_sched_qs != ULONG_MAX) {
424 WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
425 return;
426 }
410module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
411
412/*
413 * Make sure that we give the grace-period kthread time to detect any
414 * idle CPUs before taking active measures to force quiescent states.
415 * However, don't go below 100 milliseconds, adjusted upwards for really
416 * large systems.
417 */
418static void adjust_jiffies_till_sched_qs(void)
419{
420 unsigned long j;
421
422 /* If jiffies_till_sched_qs was specified, respect the request. */
423 if (jiffies_till_sched_qs != ULONG_MAX) {
424 WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
425 return;
426 }
427 /* Otherwise, set to third fqs scan, but bound below on large system. */
428 j = READ_ONCE(jiffies_till_first_fqs) +
429 2 * READ_ONCE(jiffies_till_next_fqs);
430 if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
431 j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
432 pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
433 WRITE_ONCE(jiffies_to_sched_qs, j);
434}
435

--- 1145 unchanged lines hidden (view full) ---

1581 * in the grace-period kthread's context? Because the kthread might have
1582 * been interrupted just as it was going to sleep, and just after the final
1583 * pre-sleep check of the awaken condition. In this case, a wakeup really
1584 * is required, and is therefore supplied.
1585 */
1586static void rcu_gp_kthread_wake(void)
1587{
1588 if ((current == rcu_state.gp_kthread &&
427 j = READ_ONCE(jiffies_till_first_fqs) +
428 2 * READ_ONCE(jiffies_till_next_fqs);
429 if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
430 j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
431 pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
432 WRITE_ONCE(jiffies_to_sched_qs, j);
433}
434

--- 1145 unchanged lines hidden (view full) ---

1580 * in the grace-period kthread's context? Because the kthread might have
1581 * been interrupted just as it was going to sleep, and just after the final
1582 * pre-sleep check of the awaken condition. In this case, a wakeup really
1583 * is required, and is therefore supplied.
1584 */
1585static void rcu_gp_kthread_wake(void)
1586{
1587 if ((current == rcu_state.gp_kthread &&
1589 !in_irq() && !in_serving_softirq()) ||
1588 !in_interrupt() && !in_serving_softirq()) ||
1590 !READ_ONCE(rcu_state.gp_flags) ||
1591 !rcu_state.gp_kthread)
1592 return;
1593 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1594 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
1595 swake_up_one(&rcu_state.gp_wq);
1596}
1597

--- 693 unchanged lines hidden (view full) ---

2291 * We will instead need a new quiescent state that lies
2292 * within the current grace period.
2293 */
2294 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
2295 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2296 return;
2297 }
2298 mask = rdp->grpmask;
1589 !READ_ONCE(rcu_state.gp_flags) ||
1590 !rcu_state.gp_kthread)
1591 return;
1592 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1593 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
1594 swake_up_one(&rcu_state.gp_wq);
1595}
1596

--- 693 unchanged lines hidden (view full) ---

2290 * We will instead need a new quiescent state that lies
2291 * within the current grace period.
2292 */
2293 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
2294 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2295 return;
2296 }
2297 mask = rdp->grpmask;
2299 rdp->core_needs_qs = false;
2300 if ((rnp->qsmask & mask) == 0) {
2301 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2302 } else {
2298 if ((rnp->qsmask & mask) == 0) {
2299 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2300 } else {
2301 rdp->core_needs_qs = false;
2302
2303 /*
2304 * This GP can't end until cpu checks in, so all of our
2305 * callbacks can be processed during the next GP.
2306 */
2307 needwake = rcu_accelerate_cbs(rnp, rdp);
2308
2309 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2310 /* ^^^ Released rnp->lock */

--- 232 unchanged lines hidden (view full) ---

2543 rcu_flavor_sched_clock_irq(user);
2544 if (rcu_pending())
2545 invoke_rcu_core();
2546
2547 trace_rcu_utilization(TPS("End scheduler-tick"));
2548}
2549
2550/*
2303 /*
2304 * This GP can't end until cpu checks in, so all of our
2305 * callbacks can be processed during the next GP.
2306 */
2307 needwake = rcu_accelerate_cbs(rnp, rdp);
2308
2309 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2310 /* ^^^ Released rnp->lock */

--- 232 unchanged lines hidden (view full) ---

2543 rcu_flavor_sched_clock_irq(user);
2544 if (rcu_pending())
2545 invoke_rcu_core();
2546
2547 trace_rcu_utilization(TPS("End scheduler-tick"));
2548}
2549
2550/*
2551 * Scan the leaf rcu_node structures. For each structure on which all
2552 * CPUs have reported a quiescent state and on which there are tasks
2553 * blocking the current grace period, initiate RCU priority boosting.
2554 * Otherwise, invoke the specified function to check dyntick state for
2555 * each CPU that has not yet reported a quiescent state.
2551 * Scan the leaf rcu_node structures, processing dyntick state for any that
2552 * have not yet encountered a quiescent state, using the function specified.
2553 * Also initiate boosting for any threads blocked on the root rcu_node.
2554 *
2555 * The caller must have suppressed start of new grace periods.
2556 */
2557static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
2558{
2559 int cpu;
2560 unsigned long flags;
2561 unsigned long mask;
2562 struct rcu_node *rnp;
2563

--- 990 unchanged lines hidden (view full) ---

3554 * and hibernation run faster.
3555 */
3556static int rcu_pm_notify(struct notifier_block *self,
3557 unsigned long action, void *hcpu)
3558{
3559 switch (action) {
3560 case PM_HIBERNATION_PREPARE:
3561 case PM_SUSPEND_PREPARE:
2556 */
2557static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
2558{
2559 int cpu;
2560 unsigned long flags;
2561 unsigned long mask;
2562 struct rcu_node *rnp;
2563

--- 990 unchanged lines hidden (view full) ---

3554 * and hibernation run faster.
3555 */
3556static int rcu_pm_notify(struct notifier_block *self,
3557 unsigned long action, void *hcpu)
3558{
3559 switch (action) {
3560 case PM_HIBERNATION_PREPARE:
3561 case PM_SUSPEND_PREPARE:
3562 rcu_expedite_gp();
3562 if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
3563 rcu_expedite_gp();
3563 break;
3564 case PM_POST_HIBERNATION:
3565 case PM_POST_SUSPEND:
3564 break;
3565 case PM_POST_HIBERNATION:
3566 case PM_POST_SUSPEND:
3566 rcu_unexpedite_gp();
3567 if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
3568 rcu_unexpedite_gp();
3567 break;
3568 default:
3569 break;
3570 }
3571 return NOTIFY_OK;
3572}
3573
3574/*

--- 160 unchanged lines hidden (view full) ---

3735 * value, which is a function of HZ, then adding one for each
3736 * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
3737 */
3738 d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
3739 if (jiffies_till_first_fqs == ULONG_MAX)
3740 jiffies_till_first_fqs = d;
3741 if (jiffies_till_next_fqs == ULONG_MAX)
3742 jiffies_till_next_fqs = d;
3569 break;
3570 default:
3571 break;
3572 }
3573 return NOTIFY_OK;
3574}
3575
3576/*

--- 160 unchanged lines hidden (view full) ---

3737 * value, which is a function of HZ, then adding one for each
3738 * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
3739 */
3740 d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
3741 if (jiffies_till_first_fqs == ULONG_MAX)
3742 jiffies_till_first_fqs = d;
3743 if (jiffies_till_next_fqs == ULONG_MAX)
3744 jiffies_till_next_fqs = d;
3743 adjust_jiffies_till_sched_qs();
3745 if (jiffies_till_sched_qs == ULONG_MAX)
3746 adjust_jiffies_till_sched_qs();
3744
3745 /* If the compile-time values are accurate, just leave. */
3746 if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
3747 nr_cpu_ids == NR_CPUS)
3748 return;
3749 pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
3750 rcu_fanout_leaf, nr_cpu_ids);
3751

--- 98 unchanged lines hidden (view full) ---

3850 /* Create workqueue for expedited GPs and for Tree SRCU. */
3851 rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
3852 WARN_ON(!rcu_gp_wq);
3853 rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3854 WARN_ON(!rcu_par_gp_wq);
3855 srcu_init();
3856}
3857
3747
3748 /* If the compile-time values are accurate, just leave. */
3749 if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
3750 nr_cpu_ids == NR_CPUS)
3751 return;
3752 pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
3753 rcu_fanout_leaf, nr_cpu_ids);
3754

--- 98 unchanged lines hidden (view full) ---

3853 /* Create workqueue for expedited GPs and for Tree SRCU. */
3854 rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
3855 WARN_ON(!rcu_gp_wq);
3856 rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3857 WARN_ON(!rcu_par_gp_wq);
3858 srcu_init();
3859}
3860
3861#include "tree_stall.h"
3858#include "tree_exp.h"
3859#include "tree_plugin.h"
3862#include "tree_exp.h"
3863#include "tree_plugin.h"