Lines Matching refs:srcu_sup
138 sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq; in init_srcu_struct_data()
139 sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq; in init_srcu_struct_data()
176 ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags); in init_srcu_struct_nodes()
177 if (!ssp->srcu_sup->node) in init_srcu_struct_nodes()
181 ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0]; in init_srcu_struct_nodes()
183 ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1]; in init_srcu_struct_nodes()
198 if (snp == &ssp->srcu_sup->node[0]) { in init_srcu_struct_nodes()
205 if (snp == ssp->srcu_sup->level[level + 1]) in init_srcu_struct_nodes()
207 snp->srcu_parent = ssp->srcu_sup->level[level - 1] + in init_srcu_struct_nodes()
208 (snp - ssp->srcu_sup->level[level]) / in init_srcu_struct_nodes()
217 snp_first = ssp->srcu_sup->level[level]; in init_srcu_struct_nodes()
228 smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER); in init_srcu_struct_nodes()
240 ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL); in init_srcu_struct_fields()
241 if (!ssp->srcu_sup) in init_srcu_struct_fields()
244 spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); in init_srcu_struct_fields()
245 ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL; in init_srcu_struct_fields()
246 ssp->srcu_sup->node = NULL; in init_srcu_struct_fields()
247 mutex_init(&ssp->srcu_sup->srcu_cb_mutex); in init_srcu_struct_fields()
248 mutex_init(&ssp->srcu_sup->srcu_gp_mutex); in init_srcu_struct_fields()
250 ssp->srcu_sup->srcu_gp_seq = 0; in init_srcu_struct_fields()
251 ssp->srcu_sup->srcu_barrier_seq = 0; in init_srcu_struct_fields()
252 mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); in init_srcu_struct_fields()
253 atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); in init_srcu_struct_fields()
254 INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu); in init_srcu_struct_fields()
255 ssp->srcu_sup->sda_is_static = is_static; in init_srcu_struct_fields()
260 kfree(ssp->srcu_sup); in init_srcu_struct_fields()
264 ssp->srcu_sup->srcu_gp_seq_needed_exp = 0; in init_srcu_struct_fields()
265 ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); in init_srcu_struct_fields()
266 if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { in init_srcu_struct_fields()
268 if (!ssp->srcu_sup->sda_is_static) { in init_srcu_struct_fields()
271 kfree(ssp->srcu_sup); in init_srcu_struct_fields()
275 WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); in init_srcu_struct_fields()
278 ssp->srcu_sup->srcu_ssp = ssp; in init_srcu_struct_fields()
279 smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ in init_srcu_struct_fields()
318 lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); in __srcu_transition_to_big()
319 smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); in __srcu_transition_to_big()
330 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) in srcu_transition_to_big()
332 spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); in srcu_transition_to_big()
333 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) { in srcu_transition_to_big()
334 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); in srcu_transition_to_big()
338 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); in srcu_transition_to_big()
349 if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state) in spin_lock_irqsave_check_contention()
352 if (ssp->srcu_sup->srcu_size_jiffies != j) { in spin_lock_irqsave_check_contention()
353 ssp->srcu_sup->srcu_size_jiffies = j; in spin_lock_irqsave_check_contention()
354 ssp->srcu_sup->srcu_n_lock_retries = 0; in spin_lock_irqsave_check_contention()
356 if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim) in spin_lock_irqsave_check_contention()
373 spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); in spin_lock_irqsave_sdp_contention()
375 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags); in spin_lock_irqsave_sdp_contention()
387 if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags)) in spin_lock_irqsave_ssp_contention()
389 spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); in spin_lock_irqsave_ssp_contention()
406 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/ in check_init_srcu_struct()
408 spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); in check_init_srcu_struct()
409 if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) { in check_init_srcu_struct()
410 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); in check_init_srcu_struct()
414 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); in check_init_srcu_struct()
619 struct srcu_usage *sup = ssp->srcu_sup; in srcu_get_delay()
647 struct srcu_usage *sup = ssp->srcu_sup; in cleanup_srcu_struct()
677 ssp->srcu_sup = NULL; in cleanup_srcu_struct()
776 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_gp_start()
780 lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); in srcu_gp_start()
781 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)); in srcu_gp_start()
784 rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); in srcu_gp_start()
787 WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies); in srcu_gp_start()
788 WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0); in srcu_gp_start()
790 rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq); in srcu_gp_start()
791 state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq); in srcu_gp_start()
864 struct srcu_usage *sup = ssp->srcu_sup; in srcu_gp_end()
966 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) || in srcu_funnel_exp_start()
979 if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s)) in srcu_funnel_exp_start()
980 WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s); in srcu_funnel_exp_start()
981 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); in srcu_funnel_exp_start()
1006 struct srcu_usage *sup = ssp->srcu_sup; in srcu_funnel_gp_start()
1191 tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end); in srcu_might_be_idle()
1197 curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq); in srcu_might_be_idle()
1199 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed))) in srcu_might_be_idle()
1202 if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)) in srcu_might_be_idle()
1236 ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state); in srcu_gp_start_if_needed()
1271 s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); in srcu_gp_start_if_needed()
1273 rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); in srcu_gp_start_if_needed()
1486 return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); in get_state_synchronize_srcu()
1533 if (!rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie)) in poll_state_synchronize_srcu()
1552 if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt)) in srcu_barrier_cb()
1553 complete(&ssp->srcu_sup->srcu_barrier_completion); in srcu_barrier_cb()
1567 atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1573 atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1586 unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); in srcu_barrier()
1589 mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex); in srcu_barrier()
1590 if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) { in srcu_barrier()
1592 mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex); in srcu_barrier()
1595 rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq); in srcu_barrier()
1596 init_completion(&ssp->srcu_sup->srcu_barrier_completion); in srcu_barrier()
1599 atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1); in srcu_barrier()
1602 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_barrier()
1610 if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt)) in srcu_barrier()
1611 complete(&ssp->srcu_sup->srcu_barrier_completion); in srcu_barrier()
1612 wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion); in srcu_barrier()
1614 rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq); in srcu_barrier()
1615 mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex); in srcu_barrier()
1641 mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); in srcu_advance_state()
1653 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */ in srcu_advance_state()
1655 spin_lock_irq_rcu_node(ssp->srcu_sup); in srcu_advance_state()
1656 if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { in srcu_advance_state()
1657 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); in srcu_advance_state()
1658 spin_unlock_irq_rcu_node(ssp->srcu_sup); in srcu_advance_state()
1659 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); in srcu_advance_state()
1662 idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); in srcu_advance_state()
1665 spin_unlock_irq_rcu_node(ssp->srcu_sup); in srcu_advance_state()
1667 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); in srcu_advance_state()
1672 if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) { in srcu_advance_state()
1675 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); in srcu_advance_state()
1679 spin_lock_irq_rcu_node(ssp->srcu_sup); in srcu_advance_state()
1680 rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2); in srcu_advance_state()
1681 ssp->srcu_sup->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1682 spin_unlock_irq_rcu_node(ssp->srcu_sup); in srcu_advance_state()
1685 if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) { in srcu_advance_state()
1693 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); in srcu_advance_state()
1696 ssp->srcu_sup->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1723 rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); in srcu_invoke_callbacks()
1765 spin_lock_irq_rcu_node(ssp->srcu_sup); in srcu_reschedule()
1766 if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { in srcu_reschedule()
1767 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) { in srcu_reschedule()
1771 } else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) { in srcu_reschedule()
1775 spin_unlock_irq_rcu_node(ssp->srcu_sup); in srcu_reschedule()
1778 queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay); in srcu_reschedule()
1819 *gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq); in srcutorture_get_gp_data()
1841 int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state); in srcu_torture_stats_print()
1848 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state, in srcu_torture_stats_print()
1962 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) && in srcu_module_going()
1963 !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static)) in srcu_module_going()