1 #ifndef QEMU_RCU_H 2 #define QEMU_RCU_H 3 4 /* 5 * urcu-mb.h 6 * 7 * Userspace RCU header with explicit memory barrier. 8 * 9 * This library is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; either 12 * version 2.1 of the License, or (at your option) any later version. 13 * 14 * This library is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this library; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 * IBM's contributions to this file may be relicensed under LGPLv2 or later. 24 */ 25 26 27 #include "qemu/thread.h" 28 #include "qemu/queue.h" 29 #include "qemu/atomic.h" 30 #include "qemu/notify.h" 31 #include "qemu/sys_membarrier.h" 32 #include "qemu/coroutine-tls.h" 33 34 /* 35 * Important ! 36 * 37 * Each thread containing read-side critical sections must be registered 38 * with rcu_register_thread() before calling rcu_read_lock(). 39 * rcu_unregister_thread() should be called before the thread exits. 40 */ 41 42 #ifdef DEBUG_RCU 43 #define rcu_assert(args...) assert(args) 44 #else 45 #define rcu_assert(args...) 46 #endif 47 48 /* 49 * Global quiescent period counter with low-order bits unused. 50 * Using a int rather than a char to eliminate false register dependencies 51 * causing stalls on some architectures. 52 */ 53 extern unsigned long rcu_gp_ctr; 54 55 extern QemuEvent rcu_gp_event; 56 57 struct rcu_reader_data { 58 /* Data used by both reader and synchronize_rcu() */ 59 unsigned long ctr; 60 bool waiting; 61 62 /* Data used by reader only */ 63 unsigned depth; 64 65 /* Data used for registry, protected by rcu_registry_lock */ 66 QLIST_ENTRY(rcu_reader_data) node; 67 68 /* 69 * NotifierList used to force an RCU grace period. Accessed under 70 * rcu_registry_lock. Note that the notifier is called _outside_ 71 * the thread! 72 */ 73 NotifierList force_rcu; 74 }; 75 76 QEMU_DECLARE_CO_TLS(struct rcu_reader_data, rcu_reader) 77 78 static inline void rcu_read_lock(void) 79 { 80 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 81 unsigned ctr; 82 83 if (p_rcu_reader->depth++ > 0) { 84 return; 85 } 86 87 ctr = qatomic_read(&rcu_gp_ctr); 88 qatomic_set(&p_rcu_reader->ctr, ctr); 89 90 /* 91 * Read rcu_gp_ptr and write p_rcu_reader->ctr before reading 92 * RCU-protected pointers. 93 */ 94 smp_mb_placeholder(); 95 } 96 97 static inline void rcu_read_unlock(void) 98 { 99 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 100 101 assert(p_rcu_reader->depth != 0); 102 if (--p_rcu_reader->depth > 0) { 103 return; 104 } 105 106 /* Ensure that the critical section is seen to precede the 107 * store to p_rcu_reader->ctr. Together with the following 108 * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr 109 * are sequentially consistent. 110 */ 111 qatomic_store_release(&p_rcu_reader->ctr, 0); 112 113 /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */ 114 smp_mb_placeholder(); 115 if (unlikely(qatomic_read(&p_rcu_reader->waiting))) { 116 qatomic_set(&p_rcu_reader->waiting, false); 117 qemu_event_set(&rcu_gp_event); 118 } 119 } 120 121 void synchronize_rcu(void); 122 123 /* 124 * Reader thread registration. 125 */ 126 void rcu_register_thread(void); 127 void rcu_unregister_thread(void); 128 129 /* 130 * Support for fork(). fork() support is enabled at startup. 131 */ 132 void rcu_enable_atfork(void); 133 void rcu_disable_atfork(void); 134 135 struct rcu_head; 136 typedef void RCUCBFunc(struct rcu_head *head); 137 138 struct rcu_head { 139 struct rcu_head *next; 140 RCUCBFunc *func; 141 }; 142 143 void call_rcu1(struct rcu_head *head, RCUCBFunc *func); 144 void drain_call_rcu(void); 145 146 /* The operands of the minus operator must have the same type, 147 * which must be the one that we specify in the cast. 148 */ 149 #define call_rcu(head, func, field) \ 150 call_rcu1(({ \ 151 char __attribute__((unused)) \ 152 offset_must_be_zero[-offsetof(typeof(*(head)), field)], \ 153 func_type_invalid = (func) - (void (*)(typeof(head)))(func); \ 154 &(head)->field; \ 155 }), \ 156 (RCUCBFunc *)(func)) 157 158 #define g_free_rcu(obj, field) \ 159 call_rcu1(({ \ 160 char __attribute__((unused)) \ 161 offset_must_be_zero[-offsetof(typeof(*(obj)), field)]; \ 162 &(obj)->field; \ 163 }), \ 164 (RCUCBFunc *)g_free); 165 166 typedef void RCUReadAuto; 167 static inline RCUReadAuto *rcu_read_auto_lock(void) 168 { 169 rcu_read_lock(); 170 /* Anything non-NULL causes the cleanup function to be called */ 171 return (void *)(uintptr_t)0x1; 172 } 173 174 static inline void rcu_read_auto_unlock(RCUReadAuto *r) 175 { 176 rcu_read_unlock(); 177 } 178 179 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock) 180 181 #define WITH_RCU_READ_LOCK_GUARD() \ 182 WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__)) 183 184 #define WITH_RCU_READ_LOCK_GUARD_(var) \ 185 for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \ 186 (var); rcu_read_auto_unlock(var), (var) = NULL) 187 188 #define RCU_READ_LOCK_GUARD() \ 189 g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock() 190 191 /* 192 * Force-RCU notifiers tell readers that they should exit their 193 * read-side critical section. 194 */ 195 void rcu_add_force_rcu_notifier(Notifier *n); 196 void rcu_remove_force_rcu_notifier(Notifier *n); 197 198 #endif /* QEMU_RCU_H */ 199