1 /* 2 * Queue read/write lock 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. 15 * 16 * Authors: Waiman Long <waiman.long@hp.com> 17 */ 18 #include <linux/smp.h> 19 #include <linux/bug.h> 20 #include <linux/cpumask.h> 21 #include <linux/percpu.h> 22 #include <linux/hardirq.h> 23 #include <linux/mutex.h> 24 #include <asm/qrwlock.h> 25 26 /** 27 * rspin_until_writer_unlock - inc reader count & spin until writer is gone 28 * @lock : Pointer to queue rwlock structure 29 * @writer: Current queue rwlock writer status byte 30 * 31 * In interrupt context or at the head of the queue, the reader will just 32 * increment the reader count & wait until the writer releases the lock. 33 */ 34 static __always_inline void 35 rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts) 36 { 37 while ((cnts & _QW_WMASK) == _QW_LOCKED) { 38 arch_mutex_cpu_relax(); 39 cnts = smp_load_acquire((u32 *)&lock->cnts); 40 } 41 } 42 43 /** 44 * queue_read_lock_slowpath - acquire read lock of a queue rwlock 45 * @lock: Pointer to queue rwlock structure 46 */ 47 void queue_read_lock_slowpath(struct qrwlock *lock) 48 { 49 u32 cnts; 50 51 /* 52 * Readers come here when they cannot get the lock without waiting 53 */ 54 if (unlikely(in_interrupt())) { 55 /* 56 * Readers in interrupt context will spin until the lock is 57 * available without waiting in the queue. 58 */ 59 cnts = smp_load_acquire((u32 *)&lock->cnts); 60 rspin_until_writer_unlock(lock, cnts); 61 return; 62 } 63 atomic_sub(_QR_BIAS, &lock->cnts); 64 65 /* 66 * Put the reader into the wait queue 67 */ 68 arch_spin_lock(&lock->lock); 69 70 /* 71 * At the head of the wait queue now, wait until the writer state 72 * goes to 0 and then try to increment the reader count and get 73 * the lock. It is possible that an incoming writer may steal the 74 * lock in the interim, so it is necessary to check the writer byte 75 * to make sure that the write lock isn't taken. 76 */ 77 while (atomic_read(&lock->cnts) & _QW_WMASK) 78 arch_mutex_cpu_relax(); 79 80 cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS; 81 rspin_until_writer_unlock(lock, cnts); 82 83 /* 84 * Signal the next one in queue to become queue head 85 */ 86 arch_spin_unlock(&lock->lock); 87 } 88 EXPORT_SYMBOL(queue_read_lock_slowpath); 89 90 /** 91 * queue_write_lock_slowpath - acquire write lock of a queue rwlock 92 * @lock : Pointer to queue rwlock structure 93 */ 94 void queue_write_lock_slowpath(struct qrwlock *lock) 95 { 96 u32 cnts; 97 98 /* Put the writer into the wait queue */ 99 arch_spin_lock(&lock->lock); 100 101 /* Try to acquire the lock directly if no reader is present */ 102 if (!atomic_read(&lock->cnts) && 103 (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0)) 104 goto unlock; 105 106 /* 107 * Set the waiting flag to notify readers that a writer is pending, 108 * or wait for a previous writer to go away. 109 */ 110 for (;;) { 111 cnts = atomic_read(&lock->cnts); 112 if (!(cnts & _QW_WMASK) && 113 (atomic_cmpxchg(&lock->cnts, cnts, 114 cnts | _QW_WAITING) == cnts)) 115 break; 116 117 arch_mutex_cpu_relax(); 118 } 119 120 /* When no more readers, set the locked flag */ 121 for (;;) { 122 cnts = atomic_read(&lock->cnts); 123 if ((cnts == _QW_WAITING) && 124 (atomic_cmpxchg(&lock->cnts, _QW_WAITING, 125 _QW_LOCKED) == _QW_WAITING)) 126 break; 127 128 arch_mutex_cpu_relax(); 129 } 130 unlock: 131 arch_spin_unlock(&lock->lock); 132 } 133 EXPORT_SYMBOL(queue_write_lock_slowpath); 134