tree.c (dd84cfff3cc3b79c9d616f85bd1178df135cbd1a) tree.c (e67198cc05b8ecbb7b8e2d8ef9fb5c8d26821873)
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 *
5 * Copyright IBM Corporation, 2008
6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>

--- 617 unchanged lines hidden (view full) ---

626 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
627 rdp->dynticks_nesting == 0);
628 if (rdp->dynticks_nesting != 1) {
629 // RCU will still be watching, so just do accounting and leave.
630 rdp->dynticks_nesting--;
631 return;
632 }
633
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 *
5 * Copyright IBM Corporation, 2008
6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>

--- 617 unchanged lines hidden (view full) ---

626 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
627 rdp->dynticks_nesting == 0);
628 if (rdp->dynticks_nesting != 1) {
629 // RCU will still be watching, so just do accounting and leave.
630 rdp->dynticks_nesting--;
631 return;
632 }
633
634 lockdep_assert_irqs_disabled();
635 instrumentation_begin();
634 instrumentation_begin();
635 lockdep_assert_irqs_disabled();
636 trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
637 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
638 rcu_preempt_deferred_qs(current);
639
640 // instrumentation for the noinstr rcu_dynticks_eqs_enter()
641 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
642
643 instrumentation_end();

--- 10 unchanged lines hidden (view full) ---

654 * Enter idle mode, in other words, -leave- the mode in which RCU
655 * read-side critical sections can occur. (Though RCU read-side
656 * critical sections can occur in irq handlers in idle, a possibility
657 * handled by irq_enter() and irq_exit().)
658 *
659 * If you add or remove a call to rcu_idle_enter(), be sure to test with
660 * CONFIG_RCU_EQS_DEBUG=y.
661 */
636 trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
637 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
638 rcu_preempt_deferred_qs(current);
639
640 // instrumentation for the noinstr rcu_dynticks_eqs_enter()
641 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
642
643 instrumentation_end();

--- 10 unchanged lines hidden (view full) ---

654 * Enter idle mode, in other words, -leave- the mode in which RCU
655 * read-side critical sections can occur. (Though RCU read-side
656 * critical sections can occur in irq handlers in idle, a possibility
657 * handled by irq_enter() and irq_exit().)
658 *
659 * If you add or remove a call to rcu_idle_enter(), be sure to test with
660 * CONFIG_RCU_EQS_DEBUG=y.
661 */
662void rcu_idle_enter(void)
662void noinstr rcu_idle_enter(void)
663{
663{
664 lockdep_assert_irqs_disabled();
664 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled());
665 rcu_eqs_enter(false);
666}
665 rcu_eqs_enter(false);
666}
667EXPORT_SYMBOL_GPL(rcu_idle_enter);
668
669#ifdef CONFIG_NO_HZ_FULL
670
671#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)
672/*
673 * An empty function that will trigger a reschedule on
674 * IRQ tail once IRQs get re-enabled on userspace/guest resume.
675 */

--- 180 unchanged lines hidden (view full) ---

856 * allow for the possibility of usermode upcalls messing up our count of
857 * interrupt nesting level during the busy period that is just now starting.
858 */
859static void noinstr rcu_eqs_exit(bool user)
860{
861 struct rcu_data *rdp;
862 long oldval;
863
667
668#ifdef CONFIG_NO_HZ_FULL
669
670#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)
671/*
672 * An empty function that will trigger a reschedule on
673 * IRQ tail once IRQs get re-enabled on userspace/guest resume.
674 */

--- 180 unchanged lines hidden (view full) ---

855 * allow for the possibility of usermode upcalls messing up our count of
856 * interrupt nesting level during the busy period that is just now starting.
857 */
858static void noinstr rcu_eqs_exit(bool user)
859{
860 struct rcu_data *rdp;
861 long oldval;
862
864 lockdep_assert_irqs_disabled();
863 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled());
865 rdp = this_cpu_ptr(&rcu_data);
866 oldval = rdp->dynticks_nesting;
867 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
868 if (oldval) {
869 // RCU was already watching, so just do accounting and leave.
870 rdp->dynticks_nesting++;
871 return;
872 }

--- 18 unchanged lines hidden (view full) ---

891 * rcu_idle_exit - inform RCU that current CPU is leaving idle
892 *
893 * Exit idle mode, in other words, -enter- the mode in which RCU
894 * read-side critical sections can occur.
895 *
896 * If you add or remove a call to rcu_idle_exit(), be sure to test with
897 * CONFIG_RCU_EQS_DEBUG=y.
898 */
864 rdp = this_cpu_ptr(&rcu_data);
865 oldval = rdp->dynticks_nesting;
866 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
867 if (oldval) {
868 // RCU was already watching, so just do accounting and leave.
869 rdp->dynticks_nesting++;
870 return;
871 }

--- 18 unchanged lines hidden (view full) ---

890 * rcu_idle_exit - inform RCU that current CPU is leaving idle
891 *
892 * Exit idle mode, in other words, -enter- the mode in which RCU
893 * read-side critical sections can occur.
894 *
895 * If you add or remove a call to rcu_idle_exit(), be sure to test with
896 * CONFIG_RCU_EQS_DEBUG=y.
897 */
899void rcu_idle_exit(void)
898void noinstr rcu_idle_exit(void)
900{
901 unsigned long flags;
902
899{
900 unsigned long flags;
901
903 local_irq_save(flags);
902 raw_local_irq_save(flags);
904 rcu_eqs_exit(false);
903 rcu_eqs_exit(false);
905 local_irq_restore(flags);
904 raw_local_irq_restore(flags);
906}
905}
907EXPORT_SYMBOL_GPL(rcu_idle_exit);
908
909#ifdef CONFIG_NO_HZ_FULL
910/**
911 * rcu_user_exit - inform RCU that we are exiting userspace.
912 *
913 * Exit RCU idle mode while entering the kernel because it can
914 * run a RCU read side critical section anytime.
915 *

--- 4019 unchanged lines hidden --- 		906
907#ifdef CONFIG_NO_HZ_FULL
908/**
909 * rcu_user_exit - inform RCU that we are exiting userspace.
910 *
911 * Exit RCU idle mode while entering the kernel because it can
912 * run a RCU read side critical section anytime.
913 *

--- 4019 unchanged lines hidden ---