1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * RCU segmented callback lists, internal-to-rcu header file 4 * 5 * Copyright IBM Corporation, 2017 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #include <linux/rcu_segcblist.h> 11 12 /* Return number of callbacks in the specified callback list. */ 13 static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) 14 { 15 return READ_ONCE(rclp->len); 16 } 17 18 long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg); 19 20 /* Return number of callbacks in segmented callback list by summing seglen. */ 21 long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); 22 23 void rcu_cblist_init(struct rcu_cblist *rclp); 24 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); 25 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, 26 struct rcu_cblist *srclp, 27 struct rcu_head *rhp); 28 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); 29 30 /* 31 * Is the specified rcu_segcblist structure empty? 32 * 33 * But careful! The fact that the ->head field is NULL does not 34 * necessarily imply that there are no callbacks associated with 35 * this structure. When callbacks are being invoked, they are 36 * removed as a group. If callback invocation must be preempted, 37 * the remaining callbacks will be added back to the list. Either 38 * way, the counts are updated later. 39 * 40 * So it is often the case that rcu_segcblist_n_cbs() should be used 41 * instead. 42 */ 43 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) 44 { 45 return !READ_ONCE(rsclp->head); 46 } 47 48 /* Return number of callbacks in segmented callback list. */ 49 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) 50 { 51 #ifdef CONFIG_RCU_NOCB_CPU 52 return atomic_long_read(&rsclp->len); 53 #else 54 return READ_ONCE(rsclp->len); 55 #endif 56 } 57 58 static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp, 59 int flags) 60 { 61 WRITE_ONCE(rsclp->flags, rsclp->flags | flags); 62 } 63 64 static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp, 65 int flags) 66 { 67 WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags); 68 } 69 70 static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp, 71 int flags) 72 { 73 return READ_ONCE(rsclp->flags) & flags; 74 } 75 76 /* 77 * Is the specified rcu_segcblist enabled, for example, not corresponding 78 * to an offline CPU? 79 */ 80 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) 81 { 82 return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); 83 } 84 85 /* 86 * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the 87 * [de]offloading process)? 88 */ 89 static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) 90 { 91 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && 92 rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING)) 93 return true; 94 95 return false; 96 } 97 98 static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp) 99 { 100 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && 101 !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE)) 102 return true; 103 104 return false; 105 } 106 107 /* 108 * Are all segments following the specified segment of the specified 109 * rcu_segcblist structure empty of callbacks? (The specified 110 * segment might well contain callbacks.) 111 */ 112 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) 113 { 114 return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); 115 } 116 117 /* 118 * Is the specified segment of the specified rcu_segcblist structure 119 * empty of callbacks? 120 */ 121 static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg) 122 { 123 if (seg == RCU_DONE_TAIL) 124 return &rsclp->head == rsclp->tails[RCU_DONE_TAIL]; 125 return rsclp->tails[seg - 1] == rsclp->tails[seg]; 126 } 127 128 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); 129 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v); 130 void rcu_segcblist_init(struct rcu_segcblist *rsclp); 131 void rcu_segcblist_disable(struct rcu_segcblist *rsclp); 132 void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload); 133 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); 134 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); 135 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); 136 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); 137 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); 138 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, 139 struct rcu_head *rhp); 140 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, 141 struct rcu_head *rhp); 142 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, 143 struct rcu_cblist *rclp); 144 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, 145 struct rcu_cblist *rclp); 146 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, 147 struct rcu_cblist *rclp); 148 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, 149 struct rcu_cblist *rclp); 150 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, 151 struct rcu_cblist *rclp); 152 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); 153 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); 154 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, 155 struct rcu_segcblist *src_rsclp); 156