rcutorture.c (61670adcb4a9f66ff3fa8a9e846a623d9a9e1553) rcutorture.c (e0aff97355575ac6a28a48a4217533a3953095c5)
1/*
2 * Read-Copy Update module-based torture test facility
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *

--- 42 unchanged lines hidden (view full) ---

51#include <linux/srcu.h>
52#include <linux/slab.h>
53#include <linux/trace_clock.h>
54#include <asm/byteorder.h>
55#include <linux/torture.h>
56#include <linux/vmalloc.h>
57#include <linux/sched/debug.h>
58#include <linux/sched/sysctl.h>
1/*
2 * Read-Copy Update module-based torture test facility
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *

--- 42 unchanged lines hidden (view full) ---

51#include <linux/srcu.h>
52#include <linux/slab.h>
53#include <linux/trace_clock.h>
54#include <asm/byteorder.h>
55#include <linux/torture.h>
56#include <linux/vmalloc.h>
57#include <linux/sched/debug.h>
58#include <linux/sched/sysctl.h>
59#include <linux/oom.h>
59
60#include "rcu.h"
61
62MODULE_LICENSE("GPL");
63MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
64
65
66/* Bits for ->extendables field, extendables param, and related definitions. */

--- 1552 unchanged lines hidden (view full) ---

1619 struct rcu_fwd_cb *rfc_next;
1620 int rfc_gps;
1621};
1622static DEFINE_SPINLOCK(rcu_fwd_lock);
1623static struct rcu_fwd_cb *rcu_fwd_cb_head;
1624static struct rcu_fwd_cb **rcu_fwd_cb_tail = &rcu_fwd_cb_head;
1625static long n_launders_cb;
1626static unsigned long rcu_fwd_startat;
60
61#include "rcu.h"
62
63MODULE_LICENSE("GPL");
64MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
65
66
67/* Bits for ->extendables field, extendables param, and related definitions. */

--- 1552 unchanged lines hidden (view full) ---

1620 struct rcu_fwd_cb *rfc_next;
1621 int rfc_gps;
1622};
1623static DEFINE_SPINLOCK(rcu_fwd_lock);
1624static struct rcu_fwd_cb *rcu_fwd_cb_head;
1625static struct rcu_fwd_cb **rcu_fwd_cb_tail = &rcu_fwd_cb_head;
1626static long n_launders_cb;
1627static unsigned long rcu_fwd_startat;
1628static bool rcu_fwd_emergency_stop;
1627#define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */
1628#define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */
1629#define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */
1630static long n_launders_hist[2 * MAX_FWD_CB_JIFFIES / HZ];
1631
1632/* Callback function for continuous-flood RCU callbacks. */
1633static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
1634{

--- 41 unchanged lines hidden (view full) ---

1676 }
1677 cver = READ_ONCE(rcu_torture_current_version);
1678 gps = cur_ops->get_gp_seq();
1679 sd = cur_ops->stall_dur() + 1;
1680 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
1681 dur = sd4 + torture_random(&trs) % (sd - sd4);
1682 WRITE_ONCE(rcu_fwd_startat, jiffies);
1683 stopat = rcu_fwd_startat + dur;
1629#define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */
1630#define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */
1631#define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */
1632static long n_launders_hist[2 * MAX_FWD_CB_JIFFIES / HZ];
1633
1634/* Callback function for continuous-flood RCU callbacks. */
1635static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
1636{

--- 41 unchanged lines hidden (view full) ---

1678 }
1679 cver = READ_ONCE(rcu_torture_current_version);
1680 gps = cur_ops->get_gp_seq();
1681 sd = cur_ops->stall_dur() + 1;
1682 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
1683 dur = sd4 + torture_random(&trs) % (sd - sd4);
1684 WRITE_ONCE(rcu_fwd_startat, jiffies);
1685 stopat = rcu_fwd_startat + dur;
1684 while (time_before(jiffies, stopat) && !torture_must_stop()) {
1686 while (time_before(jiffies, stopat) &&
1687 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
1685 idx = cur_ops->readlock();
1686 udelay(10);
1687 cur_ops->readunlock(idx);
1688 if (!fwd_progress_need_resched || need_resched())
1689 cond_resched();
1690 }
1691 (*tested_tries)++;
1688 idx = cur_ops->readlock();
1689 udelay(10);
1690 cur_ops->readunlock(idx);
1691 if (!fwd_progress_need_resched || need_resched())
1692 cond_resched();
1693 }
1694 (*tested_tries)++;
1692 if (!time_before(jiffies, stopat) && !torture_must_stop()) {
1695 if (!time_before(jiffies, stopat) &&
1696 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
1693 (*tested)++;
1694 cver = READ_ONCE(rcu_torture_current_version) - cver;
1695 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
1696 WARN_ON(!cver && gps < 2);
1697 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);
1698 }
1699 if (selfpropcb) {
1700 WRITE_ONCE(fcs.stop, 1);

--- 33 unchanged lines hidden (view full) ---

1734 n_launders_cb = 0;
1735 n_launders_sa = 0;
1736 n_max_cbs = 0;
1737 n_max_gps = 0;
1738 for (i = 0; i < ARRAY_SIZE(n_launders_hist); i++)
1739 n_launders_hist[i] = 0;
1740 cver = READ_ONCE(rcu_torture_current_version);
1741 gps = cur_ops->get_gp_seq();
1697 (*tested)++;
1698 cver = READ_ONCE(rcu_torture_current_version) - cver;
1699 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
1700 WARN_ON(!cver && gps < 2);
1701 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);
1702 }
1703 if (selfpropcb) {
1704 WRITE_ONCE(fcs.stop, 1);

--- 33 unchanged lines hidden (view full) ---

1738 n_launders_cb = 0;
1739 n_launders_sa = 0;
1740 n_max_cbs = 0;
1741 n_max_gps = 0;
1742 for (i = 0; i < ARRAY_SIZE(n_launders_hist); i++)
1743 n_launders_hist[i] = 0;
1744 cver = READ_ONCE(rcu_torture_current_version);
1745 gps = cur_ops->get_gp_seq();
1742 while (time_before(jiffies, stopat) && !torture_must_stop()) {
1746 while (time_before(jiffies, stopat) &&
1747 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
1743 rfcp = READ_ONCE(rcu_fwd_cb_head);
1744 rfcpn = NULL;
1745 if (rfcp)
1746 rfcpn = READ_ONCE(rfcp->rfc_next);
1747 if (rfcpn) {
1748 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
1749 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
1750 break;

--- 40 unchanged lines hidden (view full) ---

1791 break;
1792 pr_alert("Callback-invocation histogram:");
1793 for (j = 0; j <= i; j++)
1794 pr_cont(" %ds: %ld", j + 1, n_launders_hist[j]);
1795 pr_cont("\n");
1796 }
1797}
1798
1748 rfcp = READ_ONCE(rcu_fwd_cb_head);
1749 rfcpn = NULL;
1750 if (rfcp)
1751 rfcpn = READ_ONCE(rfcp->rfc_next);
1752 if (rfcpn) {
1753 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
1754 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
1755 break;

--- 40 unchanged lines hidden (view full) ---

1796 break;
1797 pr_alert("Callback-invocation histogram:");
1798 for (j = 0; j <= i; j++)
1799 pr_cont(" %ds: %ld", j + 1, n_launders_hist[j]);
1800 pr_cont("\n");
1801 }
1802}
1803
1804
1805/*
1806 * OOM notifier, but this only prints diagnostic information for the
1807 * current forward-progress test.
1808 */
1809static int rcutorture_oom_notify(struct notifier_block *self,
1810 unsigned long notused, void *nfreed)
1811{
1812 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rcu_fwd_startat) / 2));
1813 WRITE_ONCE(rcu_fwd_emergency_stop, true);
1814 return NOTIFY_OK;
1815}
1816
1817static struct notifier_block rcutorture_oom_nb = {
1818 .notifier_call = rcutorture_oom_notify
1819};
1820
1799/* Carry out grace-period forward-progress testing. */
1800static int rcu_torture_fwd_prog(void *args)
1801{
1802 int tested = 0;
1803 int tested_tries = 0;
1804
1805 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
1806 rcu_bind_current_to_nocb();
1807 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
1808 set_user_nice(current, MAX_NICE);
1809 do {
1810 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
1821/* Carry out grace-period forward-progress testing. */
1822static int rcu_torture_fwd_prog(void *args)
1823{
1824 int tested = 0;
1825 int tested_tries = 0;
1826
1827 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
1828 rcu_bind_current_to_nocb();
1829 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
1830 set_user_nice(current, MAX_NICE);
1831 do {
1832 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
1833 WRITE_ONCE(rcu_fwd_emergency_stop, false);
1834 register_oom_notifier(&rcutorture_oom_nb);
1811 rcu_torture_fwd_prog_nr(&tested, &tested_tries);
1812 rcu_torture_fwd_prog_cr();
1835 rcu_torture_fwd_prog_nr(&tested, &tested_tries);
1836 rcu_torture_fwd_prog_cr();
1837 unregister_oom_notifier(&rcutorture_oom_nb);
1813
1814 /* Avoid slow periods, better to test when busy. */
1815 stutter_wait("rcu_torture_fwd_prog");
1816 } while (!torture_must_stop());
1817 /* Short runs might not contain a valid forward-progress attempt. */
1818 WARN_ON(!tested && tested_tries >= 5);
1819 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
1820 torture_kthread_stopping("rcu_torture_fwd_prog");

--- 521 unchanged lines hidden --- 		1838
1839 /* Avoid slow periods, better to test when busy. */
1840 stutter_wait("rcu_torture_fwd_prog");
1841 } while (!torture_must_stop());
1842 /* Short runs might not contain a valid forward-progress attempt. */
1843 WARN_ON(!tested && tested_tries >= 5);
1844 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
1845 torture_kthread_stopping("rcu_torture_fwd_prog");

--- 521 unchanged lines hidden ---