1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * Read-Copy Update definitions shared among RCU implementations. 4 * 5 * Copyright IBM Corporation, 2011 6 * 7 * Author: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #ifndef __LINUX_RCU_H 11 #define __LINUX_RCU_H 12 13 #include <trace/events/rcu.h> 14 15 /* 16 * Grace-period counter management. 17 * 18 * The two least significant bits contain the control flags. 19 * The most significant bits contain the grace-period sequence counter. 20 * 21 * When both control flags are zero, no grace period is in progress. 22 * When either bit is non-zero, a grace period has started and is in 23 * progress. When the grace period completes, the control flags are reset 24 * to 0 and the grace-period sequence counter is incremented. 25 * 26 * However some specific RCU usages make use of custom values. 27 * 28 * SRCU special control values: 29 * 30 * SRCU_SNP_INIT_SEQ : Invalid/init value set when SRCU node 31 * is initialized. 32 * 33 * SRCU_STATE_IDLE : No SRCU gp is in progress 34 * 35 * SRCU_STATE_SCAN1 : State set by rcu_seq_start(). Indicates 36 * we are scanning the readers on the slot 37 * defined as inactive (there might well 38 * be pending readers that will use that 39 * index, but their number is bounded). 40 * 41 * SRCU_STATE_SCAN2 : State set manually via rcu_seq_set_state() 42 * Indicates we are flipping the readers 43 * index and then scanning the readers on the 44 * slot newly designated as inactive (again, 45 * the number of pending readers that will use 46 * this inactive index is bounded). 47 * 48 * RCU polled GP special control value: 49 * 50 * RCU_GET_STATE_COMPLETED : State value indicating an already-completed 51 * polled GP has completed. This value covers 52 * both the state and the counter of the 53 * grace-period sequence number. 54 */ 55 56 #define RCU_SEQ_CTR_SHIFT 2 57 #define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) 58 59 /* Low-order bit definition for polled grace-period APIs. */ 60 #define RCU_GET_STATE_COMPLETED 0x1 61 62 extern int sysctl_sched_rt_runtime; 63 64 /* 65 * Return the counter portion of a sequence number previously returned 66 * by rcu_seq_snap() or rcu_seq_current(). 67 */ 68 static inline unsigned long rcu_seq_ctr(unsigned long s) 69 { 70 return s >> RCU_SEQ_CTR_SHIFT; 71 } 72 73 /* 74 * Return the state portion of a sequence number previously returned 75 * by rcu_seq_snap() or rcu_seq_current(). 76 */ 77 static inline int rcu_seq_state(unsigned long s) 78 { 79 return s & RCU_SEQ_STATE_MASK; 80 } 81 82 /* 83 * Set the state portion of the pointed-to sequence number. 84 * The caller is responsible for preventing conflicting updates. 85 */ 86 static inline void rcu_seq_set_state(unsigned long *sp, int newstate) 87 { 88 WARN_ON_ONCE(newstate & ~RCU_SEQ_STATE_MASK); 89 WRITE_ONCE(*sp, (*sp & ~RCU_SEQ_STATE_MASK) + newstate); 90 } 91 92 /* Adjust sequence number for start of update-side operation. */ 93 static inline void rcu_seq_start(unsigned long *sp) 94 { 95 WRITE_ONCE(*sp, *sp + 1); 96 smp_mb(); /* Ensure update-side operation after counter increment. */ 97 WARN_ON_ONCE(rcu_seq_state(*sp) != 1); 98 } 99 100 /* Compute the end-of-grace-period value for the specified sequence number. */ 101 static inline unsigned long rcu_seq_endval(unsigned long *sp) 102 { 103 return (*sp | RCU_SEQ_STATE_MASK) + 1; 104 } 105 106 /* Adjust sequence number for end of update-side operation. */ 107 static inline void rcu_seq_end(unsigned long *sp) 108 { 109 smp_mb(); /* Ensure update-side operation before counter increment. */ 110 WARN_ON_ONCE(!rcu_seq_state(*sp)); 111 WRITE_ONCE(*sp, rcu_seq_endval(sp)); 112 } 113 114 /* 115 * rcu_seq_snap - Take a snapshot of the update side's sequence number. 116 * 117 * This function returns the earliest value of the grace-period sequence number 118 * that will indicate that a full grace period has elapsed since the current 119 * time. Once the grace-period sequence number has reached this value, it will 120 * be safe to invoke all callbacks that have been registered prior to the 121 * current time. This value is the current grace-period number plus two to the 122 * power of the number of low-order bits reserved for state, then rounded up to 123 * the next value in which the state bits are all zero. 124 */ 125 static inline unsigned long rcu_seq_snap(unsigned long *sp) 126 { 127 unsigned long s; 128 129 s = (READ_ONCE(*sp) + 2 * RCU_SEQ_STATE_MASK + 1) & ~RCU_SEQ_STATE_MASK; 130 smp_mb(); /* Above access must not bleed into critical section. */ 131 return s; 132 } 133 134 /* Return the current value the update side's sequence number, no ordering. */ 135 static inline unsigned long rcu_seq_current(unsigned long *sp) 136 { 137 return READ_ONCE(*sp); 138 } 139 140 /* 141 * Given a snapshot from rcu_seq_snap(), determine whether or not the 142 * corresponding update-side operation has started. 143 */ 144 static inline bool rcu_seq_started(unsigned long *sp, unsigned long s) 145 { 146 return ULONG_CMP_LT((s - 1) & ~RCU_SEQ_STATE_MASK, READ_ONCE(*sp)); 147 } 148 149 /* 150 * Given a snapshot from rcu_seq_snap(), determine whether or not a 151 * full update-side operation has occurred. 152 */ 153 static inline bool rcu_seq_done(unsigned long *sp, unsigned long s) 154 { 155 return ULONG_CMP_GE(READ_ONCE(*sp), s); 156 } 157 158 /* 159 * Given a snapshot from rcu_seq_snap(), determine whether or not a 160 * full update-side operation has occurred, but do not allow the 161 * (ULONG_MAX / 2) safety-factor/guard-band. 162 */ 163 static inline bool rcu_seq_done_exact(unsigned long *sp, unsigned long s) 164 { 165 unsigned long cur_s = READ_ONCE(*sp); 166 167 return ULONG_CMP_GE(cur_s, s) || ULONG_CMP_LT(cur_s, s - (2 * RCU_SEQ_STATE_MASK + 1)); 168 } 169 170 /* 171 * Has a grace period completed since the time the old gp_seq was collected? 172 */ 173 static inline bool rcu_seq_completed_gp(unsigned long old, unsigned long new) 174 { 175 return ULONG_CMP_LT(old, new & ~RCU_SEQ_STATE_MASK); 176 } 177 178 /* 179 * Has a grace period started since the time the old gp_seq was collected? 180 */ 181 static inline bool rcu_seq_new_gp(unsigned long old, unsigned long new) 182 { 183 return ULONG_CMP_LT((old + RCU_SEQ_STATE_MASK) & ~RCU_SEQ_STATE_MASK, 184 new); 185 } 186 187 /* 188 * Roughly how many full grace periods have elapsed between the collection 189 * of the two specified grace periods? 190 */ 191 static inline unsigned long rcu_seq_diff(unsigned long new, unsigned long old) 192 { 193 unsigned long rnd_diff; 194 195 if (old == new) 196 return 0; 197 /* 198 * Compute the number of grace periods (still shifted up), plus 199 * one if either of new and old is not an exact grace period. 200 */ 201 rnd_diff = (new & ~RCU_SEQ_STATE_MASK) - 202 ((old + RCU_SEQ_STATE_MASK) & ~RCU_SEQ_STATE_MASK) + 203 ((new & RCU_SEQ_STATE_MASK) || (old & RCU_SEQ_STATE_MASK)); 204 if (ULONG_CMP_GE(RCU_SEQ_STATE_MASK, rnd_diff)) 205 return 1; /* Definitely no grace period has elapsed. */ 206 return ((rnd_diff - RCU_SEQ_STATE_MASK - 1) >> RCU_SEQ_CTR_SHIFT) + 2; 207 } 208 209 /* 210 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally 211 * by call_rcu() and rcu callback execution, and are therefore not part 212 * of the RCU API. These are in rcupdate.h because they are used by all 213 * RCU implementations. 214 */ 215 216 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 217 # define STATE_RCU_HEAD_READY 0 218 # define STATE_RCU_HEAD_QUEUED 1 219 220 extern const struct debug_obj_descr rcuhead_debug_descr; 221 222 static inline int debug_rcu_head_queue(struct rcu_head *head) 223 { 224 int r1; 225 226 r1 = debug_object_activate(head, &rcuhead_debug_descr); 227 debug_object_active_state(head, &rcuhead_debug_descr, 228 STATE_RCU_HEAD_READY, 229 STATE_RCU_HEAD_QUEUED); 230 return r1; 231 } 232 233 static inline void debug_rcu_head_unqueue(struct rcu_head *head) 234 { 235 debug_object_active_state(head, &rcuhead_debug_descr, 236 STATE_RCU_HEAD_QUEUED, 237 STATE_RCU_HEAD_READY); 238 debug_object_deactivate(head, &rcuhead_debug_descr); 239 } 240 #else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 241 static inline int debug_rcu_head_queue(struct rcu_head *head) 242 { 243 return 0; 244 } 245 246 static inline void debug_rcu_head_unqueue(struct rcu_head *head) 247 { 248 } 249 #endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 250 251 extern int rcu_cpu_stall_suppress_at_boot; 252 253 static inline bool rcu_stall_is_suppressed_at_boot(void) 254 { 255 return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended(); 256 } 257 258 #ifdef CONFIG_RCU_STALL_COMMON 259 260 extern int rcu_cpu_stall_ftrace_dump; 261 extern int rcu_cpu_stall_suppress; 262 extern int rcu_cpu_stall_timeout; 263 extern int rcu_exp_cpu_stall_timeout; 264 extern int rcu_cpu_stall_cputime; 265 extern bool rcu_exp_stall_task_details __read_mostly; 266 int rcu_jiffies_till_stall_check(void); 267 int rcu_exp_jiffies_till_stall_check(void); 268 269 static inline bool rcu_stall_is_suppressed(void) 270 { 271 return rcu_stall_is_suppressed_at_boot() || rcu_cpu_stall_suppress; 272 } 273 274 #define rcu_ftrace_dump_stall_suppress() \ 275 do { \ 276 if (!rcu_cpu_stall_suppress) \ 277 rcu_cpu_stall_suppress = 3; \ 278 } while (0) 279 280 #define rcu_ftrace_dump_stall_unsuppress() \ 281 do { \ 282 if (rcu_cpu_stall_suppress == 3) \ 283 rcu_cpu_stall_suppress = 0; \ 284 } while (0) 285 286 #else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */ 287 288 static inline bool rcu_stall_is_suppressed(void) 289 { 290 return rcu_stall_is_suppressed_at_boot(); 291 } 292 #define rcu_ftrace_dump_stall_suppress() 293 #define rcu_ftrace_dump_stall_unsuppress() 294 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */ 295 296 /* 297 * Strings used in tracepoints need to be exported via the 298 * tracing system such that tools like perf and trace-cmd can 299 * translate the string address pointers to actual text. 300 */ 301 #define TPS(x) tracepoint_string(x) 302 303 /* 304 * Dump the ftrace buffer, but only one time per callsite per boot. 305 */ 306 #define rcu_ftrace_dump(oops_dump_mode) \ 307 do { \ 308 static atomic_t ___rfd_beenhere = ATOMIC_INIT(0); \ 309 \ 310 if (!atomic_read(&___rfd_beenhere) && \ 311 !atomic_xchg(&___rfd_beenhere, 1)) { \ 312 tracing_off(); \ 313 rcu_ftrace_dump_stall_suppress(); \ 314 ftrace_dump(oops_dump_mode); \ 315 rcu_ftrace_dump_stall_unsuppress(); \ 316 } \ 317 } while (0) 318 319 void rcu_early_boot_tests(void); 320 void rcu_test_sync_prims(void); 321 322 /* 323 * This function really isn't for public consumption, but RCU is special in 324 * that context switches can allow the state machine to make progress. 325 */ 326 extern void resched_cpu(int cpu); 327 328 #if !defined(CONFIG_TINY_RCU) 329 330 #include <linux/rcu_node_tree.h> 331 332 extern int rcu_num_lvls; 333 extern int num_rcu_lvl[]; 334 extern int rcu_num_nodes; 335 static bool rcu_fanout_exact; 336 static int rcu_fanout_leaf; 337 338 /* 339 * Compute the per-level fanout, either using the exact fanout specified 340 * or balancing the tree, depending on the rcu_fanout_exact boot parameter. 341 */ 342 static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) 343 { 344 int i; 345 346 for (i = 0; i < RCU_NUM_LVLS; i++) 347 levelspread[i] = INT_MIN; 348 if (rcu_fanout_exact) { 349 levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf; 350 for (i = rcu_num_lvls - 2; i >= 0; i--) 351 levelspread[i] = RCU_FANOUT; 352 } else { 353 int ccur; 354 int cprv; 355 356 cprv = nr_cpu_ids; 357 for (i = rcu_num_lvls - 1; i >= 0; i--) { 358 ccur = levelcnt[i]; 359 levelspread[i] = (cprv + ccur - 1) / ccur; 360 cprv = ccur; 361 } 362 } 363 } 364 365 extern void rcu_init_geometry(void); 366 367 /* Returns a pointer to the first leaf rcu_node structure. */ 368 #define rcu_first_leaf_node() (rcu_state.level[rcu_num_lvls - 1]) 369 370 /* Is this rcu_node a leaf? */ 371 #define rcu_is_leaf_node(rnp) ((rnp)->level == rcu_num_lvls - 1) 372 373 /* Is this rcu_node the last leaf? */ 374 #define rcu_is_last_leaf_node(rnp) ((rnp) == &rcu_state.node[rcu_num_nodes - 1]) 375 376 /* 377 * Do a full breadth-first scan of the {s,}rcu_node structures for the 378 * specified state structure (for SRCU) or the only rcu_state structure 379 * (for RCU). 380 */ 381 #define _rcu_for_each_node_breadth_first(sp, rnp) \ 382 for ((rnp) = &(sp)->node[0]; \ 383 (rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++) 384 #define rcu_for_each_node_breadth_first(rnp) \ 385 _rcu_for_each_node_breadth_first(&rcu_state, rnp) 386 #define srcu_for_each_node_breadth_first(ssp, rnp) \ 387 _rcu_for_each_node_breadth_first(ssp->srcu_sup, rnp) 388 389 /* 390 * Scan the leaves of the rcu_node hierarchy for the rcu_state structure. 391 * Note that if there is a singleton rcu_node tree with but one rcu_node 392 * structure, this loop -will- visit the rcu_node structure. It is still 393 * a leaf node, even if it is also the root node. 394 */ 395 #define rcu_for_each_leaf_node(rnp) \ 396 for ((rnp) = rcu_first_leaf_node(); \ 397 (rnp) < &rcu_state.node[rcu_num_nodes]; (rnp)++) 398 399 /* 400 * Iterate over all possible CPUs in a leaf RCU node. 401 */ 402 #define for_each_leaf_node_possible_cpu(rnp, cpu) \ 403 for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \ 404 (cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \ 405 (cpu) <= rnp->grphi; \ 406 (cpu) = cpumask_next((cpu), cpu_possible_mask)) 407 408 /* 409 * Iterate over all CPUs in a leaf RCU node's specified mask. 410 */ 411 #define rcu_find_next_bit(rnp, cpu, mask) \ 412 ((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu))) 413 #define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \ 414 for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \ 415 (cpu) = rcu_find_next_bit((rnp), 0, (mask)); \ 416 (cpu) <= rnp->grphi; \ 417 (cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask))) 418 419 #endif /* !defined(CONFIG_TINY_RCU) */ 420 421 #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_TASKS_RCU_GENERIC) 422 423 /* 424 * Wrappers for the rcu_node::lock acquire and release. 425 * 426 * Because the rcu_nodes form a tree, the tree traversal locking will observe 427 * different lock values, this in turn means that an UNLOCK of one level 428 * followed by a LOCK of another level does not imply a full memory barrier; 429 * and most importantly transitivity is lost. 430 * 431 * In order to restore full ordering between tree levels, augment the regular 432 * lock acquire functions with smp_mb__after_unlock_lock(). 433 * 434 * As ->lock of struct rcu_node is a __private field, therefore one should use 435 * these wrappers rather than directly call raw_spin_{lock,unlock}* on ->lock. 436 */ 437 #define raw_spin_lock_rcu_node(p) \ 438 do { \ 439 raw_spin_lock(&ACCESS_PRIVATE(p, lock)); \ 440 smp_mb__after_unlock_lock(); \ 441 } while (0) 442 443 #define raw_spin_unlock_rcu_node(p) \ 444 do { \ 445 lockdep_assert_irqs_disabled(); \ 446 raw_spin_unlock(&ACCESS_PRIVATE(p, lock)); \ 447 } while (0) 448 449 #define raw_spin_lock_irq_rcu_node(p) \ 450 do { \ 451 raw_spin_lock_irq(&ACCESS_PRIVATE(p, lock)); \ 452 smp_mb__after_unlock_lock(); \ 453 } while (0) 454 455 #define raw_spin_unlock_irq_rcu_node(p) \ 456 do { \ 457 lockdep_assert_irqs_disabled(); \ 458 raw_spin_unlock_irq(&ACCESS_PRIVATE(p, lock)); \ 459 } while (0) 460 461 #define raw_spin_lock_irqsave_rcu_node(p, flags) \ 462 do { \ 463 raw_spin_lock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ 464 smp_mb__after_unlock_lock(); \ 465 } while (0) 466 467 #define raw_spin_unlock_irqrestore_rcu_node(p, flags) \ 468 do { \ 469 lockdep_assert_irqs_disabled(); \ 470 raw_spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags); \ 471 } while (0) 472 473 #define raw_spin_trylock_rcu_node(p) \ 474 ({ \ 475 bool ___locked = raw_spin_trylock(&ACCESS_PRIVATE(p, lock)); \ 476 \ 477 if (___locked) \ 478 smp_mb__after_unlock_lock(); \ 479 ___locked; \ 480 }) 481 482 #define raw_lockdep_assert_held_rcu_node(p) \ 483 lockdep_assert_held(&ACCESS_PRIVATE(p, lock)) 484 485 #endif // #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_TASKS_RCU_GENERIC) 486 487 #ifdef CONFIG_TINY_RCU 488 /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */ 489 static inline bool rcu_gp_is_normal(void) { return true; } 490 static inline bool rcu_gp_is_expedited(void) { return false; } 491 static inline bool rcu_async_should_hurry(void) { return false; } 492 static inline void rcu_expedite_gp(void) { } 493 static inline void rcu_unexpedite_gp(void) { } 494 static inline void rcu_async_hurry(void) { } 495 static inline void rcu_async_relax(void) { } 496 static inline bool rcu_cpu_online(int cpu) { return true; } 497 #else /* #ifdef CONFIG_TINY_RCU */ 498 bool rcu_gp_is_normal(void); /* Internal RCU use. */ 499 bool rcu_gp_is_expedited(void); /* Internal RCU use. */ 500 bool rcu_async_should_hurry(void); /* Internal RCU use. */ 501 void rcu_expedite_gp(void); 502 void rcu_unexpedite_gp(void); 503 void rcu_async_hurry(void); 504 void rcu_async_relax(void); 505 void rcupdate_announce_bootup_oddness(void); 506 bool rcu_cpu_online(int cpu); 507 #ifdef CONFIG_TASKS_RCU_GENERIC 508 void show_rcu_tasks_gp_kthreads(void); 509 #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ 510 static inline void show_rcu_tasks_gp_kthreads(void) {} 511 #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ 512 #endif /* #else #ifdef CONFIG_TINY_RCU */ 513 514 #ifdef CONFIG_TASKS_RCU 515 struct task_struct *get_rcu_tasks_gp_kthread(void); 516 #endif // # ifdef CONFIG_TASKS_RCU 517 518 #ifdef CONFIG_TASKS_RUDE_RCU 519 struct task_struct *get_rcu_tasks_rude_gp_kthread(void); 520 #endif // # ifdef CONFIG_TASKS_RUDE_RCU 521 522 #define RCU_SCHEDULER_INACTIVE 0 523 #define RCU_SCHEDULER_INIT 1 524 #define RCU_SCHEDULER_RUNNING 2 525 526 enum rcutorture_type { 527 RCU_FLAVOR, 528 RCU_TASKS_FLAVOR, 529 RCU_TASKS_RUDE_FLAVOR, 530 RCU_TASKS_TRACING_FLAVOR, 531 RCU_TRIVIAL_FLAVOR, 532 SRCU_FLAVOR, 533 INVALID_RCU_FLAVOR 534 }; 535 536 #if defined(CONFIG_RCU_LAZY) 537 unsigned long rcu_lazy_get_jiffies_till_flush(void); 538 void rcu_lazy_set_jiffies_till_flush(unsigned long j); 539 #else 540 static inline unsigned long rcu_lazy_get_jiffies_till_flush(void) { return 0; } 541 static inline void rcu_lazy_set_jiffies_till_flush(unsigned long j) { } 542 #endif 543 544 #if defined(CONFIG_TREE_RCU) 545 void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, 546 unsigned long *gp_seq); 547 void do_trace_rcu_torture_read(const char *rcutorturename, 548 struct rcu_head *rhp, 549 unsigned long secs, 550 unsigned long c_old, 551 unsigned long c); 552 void rcu_gp_set_torture_wait(int duration); 553 #else 554 static inline void rcutorture_get_gp_data(enum rcutorture_type test_type, 555 int *flags, unsigned long *gp_seq) 556 { 557 *flags = 0; 558 *gp_seq = 0; 559 } 560 #ifdef CONFIG_RCU_TRACE 561 void do_trace_rcu_torture_read(const char *rcutorturename, 562 struct rcu_head *rhp, 563 unsigned long secs, 564 unsigned long c_old, 565 unsigned long c); 566 #else 567 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ 568 do { } while (0) 569 #endif 570 static inline void rcu_gp_set_torture_wait(int duration) { } 571 #endif 572 573 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) 574 long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask); 575 #endif 576 577 #ifdef CONFIG_TINY_SRCU 578 579 static inline void srcutorture_get_gp_data(enum rcutorture_type test_type, 580 struct srcu_struct *sp, int *flags, 581 unsigned long *gp_seq) 582 { 583 if (test_type != SRCU_FLAVOR) 584 return; 585 *flags = 0; 586 *gp_seq = sp->srcu_idx; 587 } 588 589 #elif defined(CONFIG_TREE_SRCU) 590 591 void srcutorture_get_gp_data(enum rcutorture_type test_type, 592 struct srcu_struct *sp, int *flags, 593 unsigned long *gp_seq); 594 595 #endif 596 597 #ifdef CONFIG_TINY_RCU 598 static inline bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { return false; } 599 static inline unsigned long rcu_get_gp_seq(void) { return 0; } 600 static inline unsigned long rcu_exp_batches_completed(void) { return 0; } 601 static inline unsigned long 602 srcu_batches_completed(struct srcu_struct *sp) { return 0; } 603 static inline void rcu_force_quiescent_state(void) { } 604 static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { return true; } 605 static inline void show_rcu_gp_kthreads(void) { } 606 static inline int rcu_get_gp_kthreads_prio(void) { return 0; } 607 static inline void rcu_fwd_progress_check(unsigned long j) { } 608 static inline void rcu_gp_slow_register(atomic_t *rgssp) { } 609 static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { } 610 #else /* #ifdef CONFIG_TINY_RCU */ 611 bool rcu_dynticks_zero_in_eqs(int cpu, int *vp); 612 unsigned long rcu_get_gp_seq(void); 613 unsigned long rcu_exp_batches_completed(void); 614 unsigned long srcu_batches_completed(struct srcu_struct *sp); 615 bool rcu_check_boost_fail(unsigned long gp_state, int *cpup); 616 void show_rcu_gp_kthreads(void); 617 int rcu_get_gp_kthreads_prio(void); 618 void rcu_fwd_progress_check(unsigned long j); 619 void rcu_force_quiescent_state(void); 620 extern struct workqueue_struct *rcu_gp_wq; 621 #ifdef CONFIG_RCU_EXP_KTHREAD 622 extern struct kthread_worker *rcu_exp_gp_kworker; 623 extern struct kthread_worker *rcu_exp_par_gp_kworker; 624 #else /* !CONFIG_RCU_EXP_KTHREAD */ 625 extern struct workqueue_struct *rcu_par_gp_wq; 626 #endif /* CONFIG_RCU_EXP_KTHREAD */ 627 void rcu_gp_slow_register(atomic_t *rgssp); 628 void rcu_gp_slow_unregister(atomic_t *rgssp); 629 #endif /* #else #ifdef CONFIG_TINY_RCU */ 630 631 #ifdef CONFIG_RCU_NOCB_CPU 632 void rcu_bind_current_to_nocb(void); 633 #else 634 static inline void rcu_bind_current_to_nocb(void) { } 635 #endif 636 637 #if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_RCU) 638 void show_rcu_tasks_classic_gp_kthread(void); 639 #else 640 static inline void show_rcu_tasks_classic_gp_kthread(void) {} 641 #endif 642 #if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_RUDE_RCU) 643 void show_rcu_tasks_rude_gp_kthread(void); 644 #else 645 static inline void show_rcu_tasks_rude_gp_kthread(void) {} 646 #endif 647 #if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_TRACE_RCU) 648 void show_rcu_tasks_trace_gp_kthread(void); 649 #else 650 static inline void show_rcu_tasks_trace_gp_kthread(void) {} 651 #endif 652 653 #ifdef CONFIG_TINY_RCU 654 static inline bool rcu_cpu_beenfullyonline(int cpu) { return true; } 655 #else 656 bool rcu_cpu_beenfullyonline(int cpu); 657 #endif 658 659 #endif /* __LINUX_RCU_H */ 660