Lines Matching refs:rcu_data
79 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
150 static void rcu_report_exp_rdp(struct rcu_data *rdp);
152 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
153 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
154 static bool rcu_rdp_cpu_online(struct rcu_data *rdp);
236 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_n_cbs_cpu()
291 static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) in rcu_dynticks_in_eqs_since()
330 raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); in rcu_momentary_dyntick_idle()
476 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
549 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_irq_work_resched()
612 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in __rcu_irq_enter_check_tick()
660 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && in rcu_needs_cpu()
661 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data)); in rcu_needs_cpu()
669 static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) in rcu_disable_urgency_upon_qs()
721 smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true); in rcu_request_urgent_qs_task()
731 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf()
746 static int dyntick_save_progress_counter(struct rcu_data *rdp) in dyntick_save_progress_counter()
767 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) in rcu_implicit_dynticks_qs()
904 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp()
928 static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, in rcu_start_this_gp()
1006 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_future_gp_cleanup()
1087 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs()
1134 struct rcu_data *rdp) in rcu_accelerate_cbs_unlocked()
1163 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs()
1187 struct rcu_data *rdp) in rcu_advance_cbs_nowake()
1217 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes()
1266 static void note_gp_changes(struct rcu_data *rdp) in note_gp_changes()
1433 struct rcu_data *rdp; in rcu_gp_init()
1540 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_init()
1725 struct rcu_data *rdp; in rcu_gp_cleanup()
1766 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
1774 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_gp_cleanup()
1793 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
2009 rcu_report_qs_rdp(struct rcu_data *rdp) in rcu_report_qs_rdp()
2077 rcu_check_quiescent_state(struct rcu_data *rdp) in rcu_check_quiescent_state()
2119 static void rcu_do_batch(struct rcu_data *rdp) in rcu_do_batch()
2274 WARN_ON_ONCE(time_before(j, __this_cpu_read(rcu_data.last_sched_clock))); in rcu_sched_clock_irq()
2275 __this_cpu_write(rcu_data.last_sched_clock, j); in rcu_sched_clock_irq()
2279 raw_cpu_inc(rcu_data.ticks_this_gp); in rcu_sched_clock_irq()
2281 if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_sched_clock_irq()
2287 __this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_sched_clock_irq()
2306 static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) in force_qs_rnp()
2336 struct rcu_data *rdp; in force_qs_rnp()
2339 rdp = per_cpu_ptr(&rcu_data, cpu); in force_qs_rnp()
2373 rnp = raw_cpu_read(rcu_data.mynode); in rcu_force_quiescent_state()
2411 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); in rcu_core()
2498 __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); in invoke_rcu_core_kthread()
2499 t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); in invoke_rcu_core_kthread()
2501 rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); in invoke_rcu_core_kthread()
2520 per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; in rcu_cpu_kthread_park()
2525 return __this_cpu_read(rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread_should_run()
2535 unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status); in rcu_cpu_kthread()
2536 char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread()
2537 unsigned long *j = this_cpu_ptr(&rcu_data.rcuc_activity); in rcu_cpu_kthread()
2567 .store = &rcu_data.rcu_cpu_kthread_task,
2583 per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0; in rcu_spawn_core_kthreads()
2594 static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in __call_rcu_core()
2649 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked()
2672 static void check_cb_ovld(struct rcu_data *rdp) in check_cb_ovld()
2691 struct rcu_data *rdp; in __call_rcu_common()
2714 rdp = this_cpu_ptr(&rcu_data); in __call_rcu_common()
3628 for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent) in synchronize_rcu()
3705 struct rcu_data *rdp; in start_poll_synchronize_rcu_common()
3710 rdp = this_cpu_ptr(&rcu_data); in start_poll_synchronize_rcu_common()
3923 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_pending()
3999 static void rcu_barrier_entrain(struct rcu_data *rdp) in rcu_barrier_entrain()
4039 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier_handler()
4062 struct rcu_data *rdp; in rcu_barrier()
4101 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4143 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4169 static bool rcu_rdp_cpu_online(struct rcu_data *rdp) in rcu_rdp_cpu_online()
4176 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_online()
4198 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online()
4204 rdp = this_cpu_ptr(&rcu_data); in rcu_lockdep_current_cpu_online()
4235 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dying_cpu()
4344 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_boot_init_percpu_data()
4375 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_prepare_cpu()
4421 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_affinity_setting()
4431 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_beenfullyonline()
4443 struct rcu_data *rdp; in rcutree_online_cpu()
4446 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_online_cpu()
4468 struct rcu_data *rdp; in rcutree_offline_cpu()
4471 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_offline_cpu()
4499 struct rcu_data *rdp; in rcu_cpu_starting()
4504 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_starting()
4555 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_dead()
4593 struct rcu_data *my_rdp; in rcutree_migrate_callbacks()
4595 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_migrate_callbacks()
4605 my_rdp = this_cpu_ptr(&rcu_data); in rcutree_migrate_callbacks()
4718 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_spawn_gp_kthread()
4862 per_cpu_ptr(&rcu_data, i)->mynode = rnp; in rcu_init_one()