tree_exp.h (3c345825c899df0751b01143b159ddaefb91a220) tree_exp.h (031aeee000e895247eafa4233c2d193dd39d4ea2)
1/*
2 * RCU expedited grace periods
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *

--- 278 unchanged lines hidden (view full) ---

287 spin_lock(&rnp->exp_lock);
288 if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {
289
290 /* Someone else doing GP, so wait for them. */
291 spin_unlock(&rnp->exp_lock);
292 trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
293 rnp->grplo, rnp->grphi,
294 TPS("wait"));
1/*
2 * RCU expedited grace periods
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *

--- 278 unchanged lines hidden (view full) ---

287 spin_lock(&rnp->exp_lock);
288 if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {
289
290 /* Someone else doing GP, so wait for them. */
291 spin_unlock(&rnp->exp_lock);
292 trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
293 rnp->grplo, rnp->grphi,
294 TPS("wait"));
295 wait_event(rnp->exp_wq[(s >> 1) & 0x3],
295 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
296 sync_exp_work_done(rsp,
297 &rdp->exp_workdone2, s));
298 return true;
299 }
300 rnp->exp_seq_rq = s; /* Followers can wait on us. */
301 spin_unlock(&rnp->exp_lock);
302 trace_rcu_exp_funnel_lock(rsp->name, rnp->level, rnp->grplo,
303 rnp->grphi, TPS("nxtlvl"));

--- 225 unchanged lines hidden (view full) ---

529 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {
530 spin_lock(&rnp->exp_lock);
531 /* Recheck, avoid hang in case someone just arrived. */
532 if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
533 rnp->exp_seq_rq = s;
534 spin_unlock(&rnp->exp_lock);
535 }
536 smp_mb(); /* All above changes before wakeup. */
296 sync_exp_work_done(rsp,
297 &rdp->exp_workdone2, s));
298 return true;
299 }
300 rnp->exp_seq_rq = s; /* Followers can wait on us. */
301 spin_unlock(&rnp->exp_lock);
302 trace_rcu_exp_funnel_lock(rsp->name, rnp->level, rnp->grplo,
303 rnp->grphi, TPS("nxtlvl"));

--- 225 unchanged lines hidden (view full) ---

529 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {
530 spin_lock(&rnp->exp_lock);
531 /* Recheck, avoid hang in case someone just arrived. */
532 if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
533 rnp->exp_seq_rq = s;
534 spin_unlock(&rnp->exp_lock);
535 }
536 smp_mb(); /* All above changes before wakeup. */
537 wake_up_all(&rnp->exp_wq[(rsp->expedited_sequence >> 1) & 0x3]);
537 wake_up_all(&rnp->exp_wq[rcu_seq_ctr(rsp->expedited_sequence) & 0x3]);
538 }
539 trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));
540 mutex_unlock(&rsp->exp_wake_mutex);
541}
542
543/* Let the workqueue handler know what it is supposed to do. */
544struct rcu_exp_work {
545 smp_call_func_t rew_func;

--- 61 unchanged lines hidden (view full) ---

607 rew.rew_s = s;
608 INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
609 schedule_work(&rew.rew_work);
610 }
611
612 /* Wait for expedited grace period to complete. */
613 rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
614 rnp = rcu_get_root(rsp);
538 }
539 trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));
540 mutex_unlock(&rsp->exp_wake_mutex);
541}
542
543/* Let the workqueue handler know what it is supposed to do. */
544struct rcu_exp_work {
545 smp_call_func_t rew_func;

--- 61 unchanged lines hidden (view full) ---

607 rew.rew_s = s;
608 INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
609 schedule_work(&rew.rew_work);
610 }
611
612 /* Wait for expedited grace period to complete. */
613 rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
614 rnp = rcu_get_root(rsp);
615 wait_event(rnp->exp_wq[(s >> 1) & 0x3],
616 sync_exp_work_done(rsp,
617 &rdp->exp_workdone0, s));
615 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
616 sync_exp_work_done(rsp, &rdp->exp_workdone0, s));
618 smp_mb(); /* Workqueue actions happen before return. */
619
620 /* Let the next expedited grace period start. */
621 mutex_unlock(&rsp->exp_mutex);
622}
623
624/**
625 * synchronize_sched_expedited - Brute-force RCU-sched grace period

--- 116 unchanged lines hidden --- 		617 smp_mb(); /* Workqueue actions happen before return. */
618
619 /* Let the next expedited grace period start. */
620 mutex_unlock(&rsp->exp_mutex);
621}
622
623/**
624 * synchronize_sched_expedited - Brute-force RCU-sched grace period

--- 116 unchanged lines hidden ---