Lines Matching refs:rcu_state
85 static struct rcu_state rcu_state = { variable
86 .level = { &rcu_state.node[0] },
89 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
90 .barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock),
93 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
94 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
227 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); in rcu_gp_in_progress()
484 return READ_ONCE(rcu_state.gp_seq); in rcu_get_gp_seq()
496 return rcu_state.expedited_sequence; in rcu_exp_batches_completed()
505 return &rcu_state.node[0]; in rcu_get_root()
516 *flags = READ_ONCE(rcu_state.gp_flags); in rcutorture_get_gp_data()
517 *gp_seq = rcu_seq_current(&rcu_state.gp_seq); in rcutorture_get_gp_data()
750 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
782 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
834 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || in rcu_implicit_dynticks_qs()
835 time_after(jiffies, rcu_state.jiffies_resched) || in rcu_implicit_dynticks_qs()
836 rcu_state.cbovld)) { in rcu_implicit_dynticks_qs()
840 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { in rcu_implicit_dynticks_qs()
854 rcu_state.cbovld)) { in rcu_implicit_dynticks_qs()
867 if (time_after(jiffies, rcu_state.jiffies_resched)) { in rcu_implicit_dynticks_qs()
907 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in trace_rcu_this_gp()
980 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); in rcu_start_this_gp()
981 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_start_this_gp()
982 if (!READ_ONCE(rcu_state.gp_kthread)) { in rcu_start_this_gp()
986 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); in rcu_start_this_gp()
1065 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); in rcu_gp_kthread_wake()
1068 !READ_ONCE(rcu_state.gp_flags) || !t) in rcu_gp_kthread_wake()
1070 WRITE_ONCE(rcu_state.gp_wake_time, jiffies); in rcu_gp_kthread_wake()
1071 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); in rcu_gp_kthread_wake()
1072 swake_up_one_online(&rcu_state.gp_wq); in rcu_gp_kthread_wake()
1111 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs()
1117 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1119 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1140 c = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs_unlocked()
1234 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1250 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1317 !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay))) in rcu_gp_slow()
1364 if (!rcu_seq_state(rcu_state.gp_seq_polled)) in rcu_poll_gp_seq_start()
1365 rcu_seq_start(&rcu_state.gp_seq_polled); in rcu_poll_gp_seq_start()
1368 *snap = rcu_state.gp_seq_polled; in rcu_poll_gp_seq_start()
1382 if (*snap && *snap == rcu_state.gp_seq_polled) { in rcu_poll_gp_seq_end()
1383 rcu_seq_end(&rcu_state.gp_seq_polled); in rcu_poll_gp_seq_end()
1384 rcu_state.gp_seq_polled_snap = 0; in rcu_poll_gp_seq_end()
1385 rcu_state.gp_seq_polled_exp_snap = 0; in rcu_poll_gp_seq_end()
1436 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1438 if (!READ_ONCE(rcu_state.gp_flags)) { in rcu_gp_init()
1443 WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */ in rcu_gp_init()
1457 rcu_seq_start(&rcu_state.gp_seq); in rcu_gp_init()
1458 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_init()
1459 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); in rcu_gp_init()
1460 rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap); in rcu_gp_init()
1472 WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF); in rcu_gp_init()
1476 arch_spin_lock(&rcu_state.ofl_lock); in rcu_gp_init()
1482 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1519 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1536 WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT); in rcu_gp_init()
1543 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1547 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1558 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1581 *gfp = READ_ONCE(rcu_state.gp_flags); in rcu_gp_fqs_check_wake()
1597 int nr_fqs = READ_ONCE(rcu_state.nr_fqs_jiffies_stall); in rcu_gp_fqs()
1600 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs()
1601 WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); in rcu_gp_fqs()
1607 WRITE_ONCE(rcu_state.jiffies_stall, in rcu_gp_fqs()
1610 WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, --nr_fqs); in rcu_gp_fqs()
1621 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
1623 WRITE_ONCE(rcu_state.gp_flags, in rcu_gp_fqs()
1624 READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
1641 if (rcu_state.cbovld) in rcu_gp_fqs_loop()
1645 if (rcu_state.cbovld) { in rcu_gp_fqs_loop()
1650 if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) { in rcu_gp_fqs_loop()
1651 WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j); in rcu_gp_fqs_loop()
1657 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, in rcu_gp_fqs_loop()
1660 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1662 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS); in rcu_gp_fqs_loop()
1663 (void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq, in rcu_gp_fqs_loop()
1666 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS); in rcu_gp_fqs_loop()
1681 if (!time_after(rcu_state.jiffies_force_qs, jiffies) || in rcu_gp_fqs_loop()
1683 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1689 gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0; in rcu_gp_fqs_loop()
1691 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1694 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1700 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1702 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1706 if (time_after(jiffies, rcu_state.jiffies_force_qs)) in rcu_gp_fqs_loop()
1709 j = rcu_state.jiffies_force_qs - j; in rcu_gp_fqs_loop()
1729 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
1731 rcu_state.gp_end = jiffies; in rcu_gp_cleanup()
1732 gp_duration = rcu_state.gp_end - rcu_state.gp_start; in rcu_gp_cleanup()
1733 if (gp_duration > rcu_state.gp_max) in rcu_gp_cleanup()
1734 rcu_state.gp_max = gp_duration; in rcu_gp_cleanup()
1744 rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap); in rcu_gp_cleanup()
1756 new_gp_seq = rcu_state.gp_seq; in rcu_gp_cleanup()
1781 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
1788 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end")); in rcu_gp_cleanup()
1789 rcu_seq_end(&rcu_state.gp_seq); in rcu_gp_cleanup()
1790 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_cleanup()
1791 WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE); in rcu_gp_cleanup()
1813 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
1814 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_gp_cleanup()
1815 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq")); in rcu_gp_cleanup()
1824 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
1843 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
1845 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS); in rcu_gp_kthread()
1846 swait_event_idle_exclusive(rcu_state.gp_wq, in rcu_gp_kthread()
1847 READ_ONCE(rcu_state.gp_flags) & in rcu_gp_kthread()
1850 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS); in rcu_gp_kthread()
1855 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_kthread()
1857 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
1865 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP); in rcu_gp_kthread()
1867 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED); in rcu_gp_kthread()
1885 WRITE_ONCE(rcu_state.gp_flags, in rcu_report_qs_rsp()
1886 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
1929 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
2135 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2137 trace_rcu_batch_end(rcu_state.name, 0, in rcu_do_batch()
2165 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2185 trace_rcu_invoke_callback(rcu_state.name, rhp); in rcu_do_batch()
2226 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2241 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2313 rcu_state.cbovld = rcu_state.cbovldnext; in force_qs_rnp()
2314 rcu_state.cbovldnext = false; in force_qs_rnp()
2321 rcu_state.cbovldnext |= !!rnp->cbovldmask; in force_qs_rnp()
2376 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) || in rcu_force_quiescent_state()
2389 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_force_quiescent_state()
2393 WRITE_ONCE(rcu_state.gp_flags, in rcu_force_quiescent_state()
2394 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_force_quiescent_state()
2628 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in __call_rcu_core()
2631 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in __call_rcu_core()
2737 trace_rcu_kvfree_callback(rcu_state.name, head, in __call_rcu_common()
2741 trace_rcu_callback(rcu_state.name, head, in __call_rcu_common()
3021 rcu_state.name, bnode->nr_records, in kvfree_rcu_bulk()
3028 rcu_state.name, bnode->records[i], 0); in kvfree_rcu_bulk()
3059 trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset); in kvfree_rcu_list()
3631 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); in synchronize_rcu()
3632 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); in synchronize_rcu()
3640 rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT); in synchronize_rcu()
3642 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in synchronize_rcu()
3676 return rcu_seq_snap(&rcu_state.gp_seq_polled); in get_state_synchronize_rcu()
3706 rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); in get_state_synchronize_rcu_full()
3732 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3817 rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) { in poll_state_synchronize_rcu()
3866 rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { in poll_state_synchronize_rcu_full()
3983 trace_rcu_barrier(rcu_state.name, s, cpu, in rcu_barrier_trace()
3984 atomic_read(&rcu_state.barrier_cpu_count), done); in rcu_barrier_trace()
3999 unsigned long __maybe_unused s = rcu_state.barrier_sequence; in rcu_barrier_callback()
4001 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) { in rcu_barrier_callback()
4003 complete(&rcu_state.barrier_completion); in rcu_barrier_callback()
4014 unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence); in rcu_barrier_entrain()
4019 lockdep_assert_held(&rcu_state.barrier_lock); in rcu_barrier_entrain()
4022 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_entrain()
4035 atomic_inc(&rcu_state.barrier_cpu_count); in rcu_barrier_entrain()
4038 rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_entrain()
4057 raw_spin_lock(&rcu_state.barrier_lock); in rcu_barrier_handler()
4059 raw_spin_unlock(&rcu_state.barrier_lock); in rcu_barrier_handler()
4076 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); in rcu_barrier()
4081 mutex_lock(&rcu_state.barrier_mutex); in rcu_barrier()
4084 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { in rcu_barrier()
4085 rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence); in rcu_barrier()
4087 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
4092 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcu_barrier()
4093 rcu_seq_start(&rcu_state.barrier_sequence); in rcu_barrier()
4094 gseq = rcu_state.barrier_sequence; in rcu_barrier()
4095 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence); in rcu_barrier()
4104 init_completion(&rcu_state.barrier_completion); in rcu_barrier()
4105 atomic_set(&rcu_state.barrier_cpu_count, 2); in rcu_barrier()
4106 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
4118 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcu_barrier()
4121 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
4122 rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
4128 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
4129 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
4132 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
4138 rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
4145 if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count)) in rcu_barrier()
4146 complete(&rcu_state.barrier_completion); in rcu_barrier()
4149 wait_for_completion(&rcu_state.barrier_completion); in rcu_barrier()
4152 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence); in rcu_barrier()
4153 rcu_seq_end(&rcu_state.barrier_sequence); in rcu_barrier()
4154 gseq = rcu_state.barrier_sequence; in rcu_barrier()
4162 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
4225 if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) in rcu_lockdep_current_cpu_online()
4238 return !!rcu_state.n_online_cpus; in rcu_init_invoked()
4255 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in rcutree_dying_cpu()
4316 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); in rcutree_dead_cpu()
4364 rdp->barrier_seq_snap = rcu_state.barrier_sequence; in rcu_boot_init_percpu_data()
4365 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4367 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4394 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
4420 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4424 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1); in rcutree_prepare_cpu()
4524 arch_spin_lock(&rcu_state.ofl_lock); in rcu_cpu_starting()
4526 raw_spin_lock(&rcu_state.barrier_lock); in rcu_cpu_starting()
4529 raw_spin_unlock(&rcu_state.barrier_lock); in rcu_cpu_starting()
4533 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */ in rcu_cpu_starting()
4534 ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus); in rcu_cpu_starting()
4536 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_cpu_starting()
4537 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_cpu_starting()
4551 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_cpu_starting()
4579 arch_spin_lock(&rcu_state.ofl_lock); in rcu_report_dead()
4581 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_report_dead()
4582 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_report_dead()
4591 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_report_dead()
4614 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcutree_migrate_callbacks()
4616 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcutree_migrate_callbacks()
4631 raw_spin_unlock(&rcu_state.barrier_lock); /* irqs remain disabled. */ in rcutree_migrate_callbacks()
4739 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name); in rcu_spawn_gp_kthread()
4748 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_spawn_gp_kthread()
4749 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_spawn_gp_kthread()
4751 smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */ in rcu_spawn_gp_kthread()
4791 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in rcu_scheduler_starting()
4824 rcu_state.level[i] = in rcu_init_one()
4825 rcu_state.level[i - 1] + num_rcu_lvl[i - 1]; in rcu_init_one()
4832 rnp = rcu_state.level[i]; in rcu_init_one()
4840 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
4841 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
4842 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()
4856 rnp->parent = rcu_state.level[i - 1] + in rcu_init_one()
4874 init_swait_queue_head(&rcu_state.gp_wq); in rcu_init_one()
4875 init_swait_queue_head(&rcu_state.expedited_wq); in rcu_init_one()