1 /* 2 * Split spinlock implementation out into its own file, so it can be 3 * compiled in a FTRACE-compatible way. 4 */ 5 #include <linux/kernel_stat.h> 6 #include <linux/spinlock.h> 7 #include <linux/debugfs.h> 8 #include <linux/log2.h> 9 #include <linux/gfp.h> 10 #include <linux/slab.h> 11 12 #include <asm/paravirt.h> 13 14 #include <xen/interface/xen.h> 15 #include <xen/events.h> 16 17 #include "xen-ops.h" 18 #include "debugfs.h" 19 20 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; 21 static DEFINE_PER_CPU(char *, irq_name); 22 static bool xen_pvspin = true; 23 24 #ifdef CONFIG_QUEUED_SPINLOCKS 25 26 #include <asm/qspinlock.h> 27 28 static void xen_qlock_kick(int cpu) 29 { 30 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 31 } 32 33 /* 34 * Halt the current CPU & release it back to the host 35 */ 36 static void xen_qlock_wait(u8 *byte, u8 val) 37 { 38 int irq = __this_cpu_read(lock_kicker_irq); 39 40 /* If kicker interrupts not initialized yet, just spin */ 41 if (irq == -1) 42 return; 43 44 /* clear pending */ 45 xen_clear_irq_pending(irq); 46 barrier(); 47 48 /* 49 * We check the byte value after clearing pending IRQ to make sure 50 * that we won't miss a wakeup event because of the clearing. 51 * 52 * The sync_clear_bit() call in xen_clear_irq_pending() is atomic. 53 * So it is effectively a memory barrier for x86. 54 */ 55 if (READ_ONCE(*byte) != val) 56 return; 57 58 /* 59 * If an interrupt happens here, it will leave the wakeup irq 60 * pending, which will cause xen_poll_irq() to return 61 * immediately. 62 */ 63 64 /* Block until irq becomes pending (or perhaps a spurious wakeup) */ 65 xen_poll_irq(irq); 66 } 67 68 #else /* CONFIG_QUEUED_SPINLOCKS */ 69 70 enum xen_contention_stat { 71 TAKEN_SLOW, 72 TAKEN_SLOW_PICKUP, 73 TAKEN_SLOW_SPURIOUS, 74 RELEASED_SLOW, 75 RELEASED_SLOW_KICKED, 76 NR_CONTENTION_STATS 77 }; 78 79 80 #ifdef CONFIG_XEN_DEBUG_FS 81 #define HISTO_BUCKETS 30 82 static struct xen_spinlock_stats 83 { 84 u32 contention_stats[NR_CONTENTION_STATS]; 85 u32 histo_spin_blocked[HISTO_BUCKETS+1]; 86 u64 time_blocked; 87 } spinlock_stats; 88 89 static u8 zero_stats; 90 91 static inline void check_zero(void) 92 { 93 u8 ret; 94 u8 old = READ_ONCE(zero_stats); 95 if (unlikely(old)) { 96 ret = cmpxchg(&zero_stats, old, 0); 97 /* This ensures only one fellow resets the stat */ 98 if (ret == old) 99 memset(&spinlock_stats, 0, sizeof(spinlock_stats)); 100 } 101 } 102 103 static inline void add_stats(enum xen_contention_stat var, u32 val) 104 { 105 check_zero(); 106 spinlock_stats.contention_stats[var] += val; 107 } 108 109 static inline u64 spin_time_start(void) 110 { 111 return xen_clocksource_read(); 112 } 113 114 static void __spin_time_accum(u64 delta, u32 *array) 115 { 116 unsigned index = ilog2(delta); 117 118 check_zero(); 119 120 if (index < HISTO_BUCKETS) 121 array[index]++; 122 else 123 array[HISTO_BUCKETS]++; 124 } 125 126 static inline void spin_time_accum_blocked(u64 start) 127 { 128 u32 delta = xen_clocksource_read() - start; 129 130 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); 131 spinlock_stats.time_blocked += delta; 132 } 133 #else /* !CONFIG_XEN_DEBUG_FS */ 134 static inline void add_stats(enum xen_contention_stat var, u32 val) 135 { 136 } 137 138 static inline u64 spin_time_start(void) 139 { 140 return 0; 141 } 142 143 static inline void spin_time_accum_blocked(u64 start) 144 { 145 } 146 #endif /* CONFIG_XEN_DEBUG_FS */ 147 148 struct xen_lock_waiting { 149 struct arch_spinlock *lock; 150 __ticket_t want; 151 }; 152 153 static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting); 154 static cpumask_t waiting_cpus; 155 156 __visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want) 157 { 158 int irq = __this_cpu_read(lock_kicker_irq); 159 struct xen_lock_waiting *w = this_cpu_ptr(&lock_waiting); 160 int cpu = smp_processor_id(); 161 u64 start; 162 __ticket_t head; 163 unsigned long flags; 164 165 /* If kicker interrupts not initialized yet, just spin */ 166 if (irq == -1) 167 return; 168 169 start = spin_time_start(); 170 171 /* 172 * Make sure an interrupt handler can't upset things in a 173 * partially setup state. 174 */ 175 local_irq_save(flags); 176 /* 177 * We don't really care if we're overwriting some other 178 * (lock,want) pair, as that would mean that we're currently 179 * in an interrupt context, and the outer context had 180 * interrupts enabled. That has already kicked the VCPU out 181 * of xen_poll_irq(), so it will just return spuriously and 182 * retry with newly setup (lock,want). 183 * 184 * The ordering protocol on this is that the "lock" pointer 185 * may only be set non-NULL if the "want" ticket is correct. 186 * If we're updating "want", we must first clear "lock". 187 */ 188 w->lock = NULL; 189 smp_wmb(); 190 w->want = want; 191 smp_wmb(); 192 w->lock = lock; 193 194 /* This uses set_bit, which atomic and therefore a barrier */ 195 cpumask_set_cpu(cpu, &waiting_cpus); 196 add_stats(TAKEN_SLOW, 1); 197 198 /* clear pending */ 199 xen_clear_irq_pending(irq); 200 201 /* Only check lock once pending cleared */ 202 barrier(); 203 204 /* 205 * Mark entry to slowpath before doing the pickup test to make 206 * sure we don't deadlock with an unlocker. 207 */ 208 __ticket_enter_slowpath(lock); 209 210 /* make sure enter_slowpath, which is atomic does not cross the read */ 211 smp_mb__after_atomic(); 212 213 /* 214 * check again make sure it didn't become free while 215 * we weren't looking 216 */ 217 head = READ_ONCE(lock->tickets.head); 218 if (__tickets_equal(head, want)) { 219 add_stats(TAKEN_SLOW_PICKUP, 1); 220 goto out; 221 } 222 223 /* Allow interrupts while blocked */ 224 local_irq_restore(flags); 225 226 /* 227 * If an interrupt happens here, it will leave the wakeup irq 228 * pending, which will cause xen_poll_irq() to return 229 * immediately. 230 */ 231 232 /* Block until irq becomes pending (or perhaps a spurious wakeup) */ 233 xen_poll_irq(irq); 234 add_stats(TAKEN_SLOW_SPURIOUS, !xen_test_irq_pending(irq)); 235 236 local_irq_save(flags); 237 238 kstat_incr_irq_this_cpu(irq); 239 out: 240 cpumask_clear_cpu(cpu, &waiting_cpus); 241 w->lock = NULL; 242 243 local_irq_restore(flags); 244 245 spin_time_accum_blocked(start); 246 } 247 PV_CALLEE_SAVE_REGS_THUNK(xen_lock_spinning); 248 249 static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next) 250 { 251 int cpu; 252 253 add_stats(RELEASED_SLOW, 1); 254 255 for_each_cpu(cpu, &waiting_cpus) { 256 const struct xen_lock_waiting *w = &per_cpu(lock_waiting, cpu); 257 258 /* Make sure we read lock before want */ 259 if (READ_ONCE(w->lock) == lock && 260 READ_ONCE(w->want) == next) { 261 add_stats(RELEASED_SLOW_KICKED, 1); 262 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 263 break; 264 } 265 } 266 } 267 #endif /* CONFIG_QUEUED_SPINLOCKS */ 268 269 static irqreturn_t dummy_handler(int irq, void *dev_id) 270 { 271 BUG(); 272 return IRQ_HANDLED; 273 } 274 275 void xen_init_lock_cpu(int cpu) 276 { 277 int irq; 278 char *name; 279 280 if (!xen_pvspin) 281 return; 282 283 WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", 284 cpu, per_cpu(lock_kicker_irq, cpu)); 285 286 name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); 287 irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, 288 cpu, 289 dummy_handler, 290 IRQF_PERCPU|IRQF_NOBALANCING, 291 name, 292 NULL); 293 294 if (irq >= 0) { 295 disable_irq(irq); /* make sure it's never delivered */ 296 per_cpu(lock_kicker_irq, cpu) = irq; 297 per_cpu(irq_name, cpu) = name; 298 } 299 300 printk("cpu %d spinlock event irq %d\n", cpu, irq); 301 } 302 303 void xen_uninit_lock_cpu(int cpu) 304 { 305 if (!xen_pvspin) 306 return; 307 308 unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); 309 per_cpu(lock_kicker_irq, cpu) = -1; 310 kfree(per_cpu(irq_name, cpu)); 311 per_cpu(irq_name, cpu) = NULL; 312 } 313 314 315 /* 316 * Our init of PV spinlocks is split in two init functions due to us 317 * using paravirt patching and jump labels patching and having to do 318 * all of this before SMP code is invoked. 319 * 320 * The paravirt patching needs to be done _before_ the alternative asm code 321 * is started, otherwise we would not patch the core kernel code. 322 */ 323 void __init xen_init_spinlocks(void) 324 { 325 326 if (!xen_pvspin) { 327 printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); 328 return; 329 } 330 printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); 331 #ifdef CONFIG_QUEUED_SPINLOCKS 332 __pv_init_lock_hash(); 333 pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 334 pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 335 pv_lock_ops.wait = xen_qlock_wait; 336 pv_lock_ops.kick = xen_qlock_kick; 337 #else 338 pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning); 339 pv_lock_ops.unlock_kick = xen_unlock_kick; 340 #endif 341 } 342 343 /* 344 * While the jump_label init code needs to happend _after_ the jump labels are 345 * enabled and before SMP is started. Hence we use pre-SMP initcall level 346 * init. We cannot do it in xen_init_spinlocks as that is done before 347 * jump labels are activated. 348 */ 349 static __init int xen_init_spinlocks_jump(void) 350 { 351 if (!xen_pvspin) 352 return 0; 353 354 if (!xen_domain()) 355 return 0; 356 357 static_key_slow_inc(¶virt_ticketlocks_enabled); 358 return 0; 359 } 360 early_initcall(xen_init_spinlocks_jump); 361 362 static __init int xen_parse_nopvspin(char *arg) 363 { 364 xen_pvspin = false; 365 return 0; 366 } 367 early_param("xen_nopvspin", xen_parse_nopvspin); 368 369 #if defined(CONFIG_XEN_DEBUG_FS) && !defined(CONFIG_QUEUED_SPINLOCKS) 370 371 static struct dentry *d_spin_debug; 372 373 static int __init xen_spinlock_debugfs(void) 374 { 375 struct dentry *d_xen = xen_init_debugfs(); 376 377 if (d_xen == NULL) 378 return -ENOMEM; 379 380 if (!xen_pvspin) 381 return 0; 382 383 d_spin_debug = debugfs_create_dir("spinlocks", d_xen); 384 385 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); 386 387 debugfs_create_u32("taken_slow", 0444, d_spin_debug, 388 &spinlock_stats.contention_stats[TAKEN_SLOW]); 389 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, 390 &spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]); 391 debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, 392 &spinlock_stats.contention_stats[TAKEN_SLOW_SPURIOUS]); 393 394 debugfs_create_u32("released_slow", 0444, d_spin_debug, 395 &spinlock_stats.contention_stats[RELEASED_SLOW]); 396 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, 397 &spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]); 398 399 debugfs_create_u64("time_blocked", 0444, d_spin_debug, 400 &spinlock_stats.time_blocked); 401 402 debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, 403 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); 404 405 return 0; 406 } 407 fs_initcall(xen_spinlock_debugfs); 408 409 #endif /* CONFIG_XEN_DEBUG_FS */ 410