tree_plugin.h (ae91aa0adb14dc33114d566feca2f7cb7a96b8b7) tree_plugin.h (44c65ff2e3b0b48250a970183ab53b0602c25764)
1/*
2 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
3 * Internal non-public definitions that provide either classic
4 * or preemptible semantics.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or

--- 1282 unchanged lines hidden (view full) ---

1291 * an exported member of the RCU API.
1292 *
1293 * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1294 * any flavor of RCU.
1295 */
1296int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1297{
1298 *nextevt = KTIME_MAX;
1/*
2 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
3 * Internal non-public definitions that provide either classic
4 * or preemptible semantics.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or

--- 1282 unchanged lines hidden (view full) ---

1291 * an exported member of the RCU API.
1292 *
1293 * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1294 * any flavor of RCU.
1295 */
1296int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1297{
1298 *nextevt = KTIME_MAX;
1299 return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1300 ? 0 : rcu_cpu_has_callbacks(NULL);
1299 return rcu_cpu_has_callbacks(NULL);
1301}
1302
1303/*
1304 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1305 * after it.
1306 */
1307static void rcu_cleanup_after_idle(void)
1308{

--- 95 unchanged lines hidden (view full) ---

1404 * The caller must have disabled interrupts.
1405 */
1406int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1407{
1408 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1409 unsigned long dj;
1410
1411 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_needs_cpu() invoked with irqs enabled!!!");
1300}
1301
1302/*
1303 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1304 * after it.
1305 */
1306static void rcu_cleanup_after_idle(void)
1307{

--- 95 unchanged lines hidden (view full) ---

1403 * The caller must have disabled interrupts.
1404 */
1405int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1406{
1407 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1408 unsigned long dj;
1409
1410 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_needs_cpu() invoked with irqs enabled!!!");
1412 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1413 *nextevt = KTIME_MAX;
1414 return 0;
1415 }
1416
1417 /* Snapshot to detect later posting of non-lazy callback. */
1418 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1419
1420 /* If no callbacks, RCU doesn't need the CPU. */
1421 if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1422 *nextevt = KTIME_MAX;
1423 return 0;

--- 33 unchanged lines hidden (view full) ---

1457 bool needwake;
1458 struct rcu_data *rdp;
1459 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1460 struct rcu_node *rnp;
1461 struct rcu_state *rsp;
1462 int tne;
1463
1464 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_prepare_for_idle() invoked with irqs enabled!!!");
1411
1412 /* Snapshot to detect later posting of non-lazy callback. */
1413 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1414
1415 /* If no callbacks, RCU doesn't need the CPU. */
1416 if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1417 *nextevt = KTIME_MAX;
1418 return 0;

--- 33 unchanged lines hidden (view full) ---

1452 bool needwake;
1453 struct rcu_data *rdp;
1454 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1455 struct rcu_node *rnp;
1456 struct rcu_state *rsp;
1457 int tne;
1458
1459 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_prepare_for_idle() invoked with irqs enabled!!!");
1465 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1466 rcu_is_nocb_cpu(smp_processor_id()))
1460 if (rcu_is_nocb_cpu(smp_processor_id()))
1467 return;
1468
1469 /* Handle nohz enablement switches conservatively. */
1470 tne = READ_ONCE(tick_nohz_active);
1471 if (tne != rdtp->tick_nohz_enabled_snap) {
1472 if (rcu_cpu_has_callbacks(NULL))
1473 invoke_rcu_core(); /* force nohz to see update. */
1474 rdtp->tick_nohz_enabled_snap = tne;

--- 38 unchanged lines hidden (view full) ---

1513/*
1514 * Clean up for exit from idle. Attempt to advance callbacks based on
1515 * any grace periods that elapsed while the CPU was idle, and if any
1516 * callbacks are now ready to invoke, initiate invocation.
1517 */
1518static void rcu_cleanup_after_idle(void)
1519{
1520 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_cleanup_after_idle() invoked with irqs enabled!!!");
1461 return;
1462
1463 /* Handle nohz enablement switches conservatively. */
1464 tne = READ_ONCE(tick_nohz_active);
1465 if (tne != rdtp->tick_nohz_enabled_snap) {
1466 if (rcu_cpu_has_callbacks(NULL))
1467 invoke_rcu_core(); /* force nohz to see update. */
1468 rdtp->tick_nohz_enabled_snap = tne;

--- 38 unchanged lines hidden (view full) ---

1507/*
1508 * Clean up for exit from idle. Attempt to advance callbacks based on
1509 * any grace periods that elapsed while the CPU was idle, and if any
1510 * callbacks are now ready to invoke, initiate invocation.
1511 */
1512static void rcu_cleanup_after_idle(void)
1513{
1514 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_cleanup_after_idle() invoked with irqs enabled!!!");
1521 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1522 rcu_is_nocb_cpu(smp_processor_id()))
1515 if (rcu_is_nocb_cpu(smp_processor_id()))
1523 return;
1524 if (rcu_try_advance_all_cbs())
1525 invoke_rcu_core();
1526}
1527
1528/*
1529 * Keep a running count of the number of non-lazy callbacks posted
1530 * on this CPU. This running counter (which is never decremented) allows

--- 250 unchanged lines hidden (view full) ---

1781}
1782
1783static void rcu_init_one_nocb(struct rcu_node *rnp)
1784{
1785 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
1786 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
1787}
1788
1516 return;
1517 if (rcu_try_advance_all_cbs())
1518 invoke_rcu_core();
1519}
1520
1521/*
1522 * Keep a running count of the number of non-lazy callbacks posted
1523 * on this CPU. This running counter (which is never decremented) allows

--- 250 unchanged lines hidden (view full) ---

1774}
1775
1776static void rcu_init_one_nocb(struct rcu_node *rnp)
1777{
1778 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
1779 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
1780}
1781
1789#ifndef CONFIG_RCU_NOCB_CPU_ALL
1790/* Is the specified CPU a no-CBs CPU? */
1791bool rcu_is_nocb_cpu(int cpu)
1792{
1793 if (have_rcu_nocb_mask)
1794 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1795 return false;
1796}
1782/* Is the specified CPU a no-CBs CPU? */
1783bool rcu_is_nocb_cpu(int cpu)
1784{
1785 if (have_rcu_nocb_mask)
1786 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1787 return false;
1788}
1797#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1798
1799/*
1800 * Kick the leader kthread for this NOCB group.
1801 */
1802static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1803{
1804 struct rcu_data *rdp_leader = rdp->nocb_leader;
1805

--- 442 unchanged lines hidden (view full) ---

2248}
2249
2250void __init rcu_init_nohz(void)
2251{
2252 int cpu;
2253 bool need_rcu_nocb_mask = true;
2254 struct rcu_state *rsp;
2255
1789
1790/*
1791 * Kick the leader kthread for this NOCB group.
1792 */
1793static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1794{
1795 struct rcu_data *rdp_leader = rdp->nocb_leader;
1796

--- 442 unchanged lines hidden (view full) ---

2239}
2240
2241void __init rcu_init_nohz(void)
2242{
2243 int cpu;
2244 bool need_rcu_nocb_mask = true;
2245 struct rcu_state *rsp;
2246
2256#ifdef CONFIG_RCU_NOCB_CPU_NONE
2257 need_rcu_nocb_mask = false;
2258#endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2259
2260#if defined(CONFIG_NO_HZ_FULL)
2261 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2262 need_rcu_nocb_mask = true;
2263#endif /* #if defined(CONFIG_NO_HZ_FULL) */
2264
2265 if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2266 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2267 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2268 return;
2269 }
2270 have_rcu_nocb_mask = true;
2271 }
2272 if (!have_rcu_nocb_mask)
2273 return;
2274
2247#if defined(CONFIG_NO_HZ_FULL)
2248 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2249 need_rcu_nocb_mask = true;
2250#endif /* #if defined(CONFIG_NO_HZ_FULL) */
2251
2252 if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2253 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2254 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2255 return;
2256 }
2257 have_rcu_nocb_mask = true;
2258 }
2259 if (!have_rcu_nocb_mask)
2260 return;
2261
2275#ifdef CONFIG_RCU_NOCB_CPU_ZERO
2276 pr_info("\tOffload RCU callbacks from CPU 0\n");
2277 cpumask_set_cpu(0, rcu_nocb_mask);
2278#endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2279#ifdef CONFIG_RCU_NOCB_CPU_ALL
2280 pr_info("\tOffload RCU callbacks from all CPUs\n");
2281 cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2282#endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2283#if defined(CONFIG_NO_HZ_FULL)
2284 if (tick_nohz_full_running)
2285 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2286#endif /* #if defined(CONFIG_NO_HZ_FULL) */
2287
2288 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2289 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2290 cpumask_and(rcu_nocb_mask, cpu_possible_mask,

--- 292 unchanged lines hidden --- 		2262#if defined(CONFIG_NO_HZ_FULL)
2263 if (tick_nohz_full_running)
2264 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2265#endif /* #if defined(CONFIG_NO_HZ_FULL) */
2266
2267 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2268 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2269 cpumask_and(rcu_nocb_mask, cpu_possible_mask,

--- 292 unchanged lines hidden ---