1 /* 2 * Queue read/write lock 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. 15 * 16 * Authors: Waiman Long <waiman.long@hp.com> 17 */ 18 #include <linux/smp.h> 19 #include <linux/bug.h> 20 #include <linux/cpumask.h> 21 #include <linux/percpu.h> 22 #include <linux/hardirq.h> 23 #include <asm/qrwlock.h> 24 25 /** 26 * rspin_until_writer_unlock - inc reader count & spin until writer is gone 27 * @lock : Pointer to queue rwlock structure 28 * @writer: Current queue rwlock writer status byte 29 * 30 * In interrupt context or at the head of the queue, the reader will just 31 * increment the reader count & wait until the writer releases the lock. 32 */ 33 static __always_inline void 34 rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts) 35 { 36 while ((cnts & _QW_WMASK) == _QW_LOCKED) { 37 cpu_relax_lowlatency(); 38 cnts = smp_load_acquire((u32 *)&lock->cnts); 39 } 40 } 41 42 /** 43 * queue_read_lock_slowpath - acquire read lock of a queue rwlock 44 * @lock: Pointer to queue rwlock structure 45 */ 46 void queue_read_lock_slowpath(struct qrwlock *lock) 47 { 48 u32 cnts; 49 50 /* 51 * Readers come here when they cannot get the lock without waiting 52 */ 53 if (unlikely(in_interrupt())) { 54 /* 55 * Readers in interrupt context will spin until the lock is 56 * available without waiting in the queue. 57 */ 58 cnts = smp_load_acquire((u32 *)&lock->cnts); 59 rspin_until_writer_unlock(lock, cnts); 60 return; 61 } 62 atomic_sub(_QR_BIAS, &lock->cnts); 63 64 /* 65 * Put the reader into the wait queue 66 */ 67 arch_spin_lock(&lock->lock); 68 69 /* 70 * At the head of the wait queue now, wait until the writer state 71 * goes to 0 and then try to increment the reader count and get 72 * the lock. It is possible that an incoming writer may steal the 73 * lock in the interim, so it is necessary to check the writer byte 74 * to make sure that the write lock isn't taken. 75 */ 76 while (atomic_read(&lock->cnts) & _QW_WMASK) 77 cpu_relax_lowlatency(); 78 79 cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS; 80 rspin_until_writer_unlock(lock, cnts); 81 82 /* 83 * Signal the next one in queue to become queue head 84 */ 85 arch_spin_unlock(&lock->lock); 86 } 87 EXPORT_SYMBOL(queue_read_lock_slowpath); 88 89 /** 90 * queue_write_lock_slowpath - acquire write lock of a queue rwlock 91 * @lock : Pointer to queue rwlock structure 92 */ 93 void queue_write_lock_slowpath(struct qrwlock *lock) 94 { 95 u32 cnts; 96 97 /* Put the writer into the wait queue */ 98 arch_spin_lock(&lock->lock); 99 100 /* Try to acquire the lock directly if no reader is present */ 101 if (!atomic_read(&lock->cnts) && 102 (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0)) 103 goto unlock; 104 105 /* 106 * Set the waiting flag to notify readers that a writer is pending, 107 * or wait for a previous writer to go away. 108 */ 109 for (;;) { 110 cnts = atomic_read(&lock->cnts); 111 if (!(cnts & _QW_WMASK) && 112 (atomic_cmpxchg(&lock->cnts, cnts, 113 cnts | _QW_WAITING) == cnts)) 114 break; 115 116 cpu_relax_lowlatency(); 117 } 118 119 /* When no more readers, set the locked flag */ 120 for (;;) { 121 cnts = atomic_read(&lock->cnts); 122 if ((cnts == _QW_WAITING) && 123 (atomic_cmpxchg(&lock->cnts, _QW_WAITING, 124 _QW_LOCKED) == _QW_WAITING)) 125 break; 126 127 cpu_relax_lowlatency(); 128 } 129 unlock: 130 arch_spin_unlock(&lock->lock); 131 } 132 EXPORT_SYMBOL(queue_write_lock_slowpath); 133