xref: /openbmc/linux/arch/ia64/include/asm/spinlock.h (revision 7bcae826) 
1 #ifndef _ASM_IA64_SPINLOCK_H
2 #define _ASM_IA64_SPINLOCK_H
3 
4 /*
5  * Copyright (C) 1998-2003 Hewlett-Packard Co
6  *	David Mosberger-Tang <davidm@hpl.hp.com>
7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
8  *
9  * This file is used for SMP configurations only.
10  */
11 
12 #include <linux/compiler.h>
13 #include <linux/kernel.h>
14 #include <linux/bitops.h>
15 
16 #include <linux/atomic.h>
17 #include <asm/intrinsics.h>
18 #include <asm/barrier.h>
19 #include <asm/processor.h>
20 
21 #define arch_spin_lock_init(x)			((x)->lock = 0)
22 
23 /*
24  * Ticket locks are conceptually two parts, one indicating the current head of
25  * the queue, and the other indicating the current tail. The lock is acquired
26  * by atomically noting the tail and incrementing it by one (thus adding
27  * ourself to the queue and noting our position), then waiting until the head
28  * becomes equal to the the initial value of the tail.
29  * The pad bits in the middle are used to prevent the next_ticket number
30  * overflowing into the now_serving number.
31  *
32  *   31             17  16    15  14                    0
33  *  +----------------------------------------------------+
34  *  |  now_serving     | padding |   next_ticket         |
35  *  +----------------------------------------------------+
36  */
37 
38 #define TICKET_SHIFT	17
39 #define TICKET_BITS	15
40 #define	TICKET_MASK	((1 << TICKET_BITS) - 1)
41 
42 static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
43 {
44 	int	*p = (int *)&lock->lock, ticket, serve;
45 
46 	ticket = ia64_fetchadd(1, p, acq);
47 
48 	if (!(((ticket >> TICKET_SHIFT) ^ ticket) & TICKET_MASK))
49 		return;
50 
51 	ia64_invala();
52 
53 	for (;;) {
54 		asm volatile ("ld4.c.nc %0=[%1]" : "=r"(serve) : "r"(p) : "memory");
55 
56 		if (!(((serve >> TICKET_SHIFT) ^ ticket) & TICKET_MASK))
57 			return;
58 		cpu_relax();
59 	}
60 }
61 
62 static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
63 {
64 	int tmp = ACCESS_ONCE(lock->lock);
65 
66 	if (!(((tmp >> TICKET_SHIFT) ^ tmp) & TICKET_MASK))
67 		return ia64_cmpxchg(acq, &lock->lock, tmp, tmp + 1, sizeof (tmp)) == tmp;
68 	return 0;
69 }
70 
71 static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
72 {
73 	unsigned short	*p = (unsigned short *)&lock->lock + 1, tmp;
74 
75 	asm volatile ("ld2.bias %0=[%1]" : "=r"(tmp) : "r"(p));
76 	ACCESS_ONCE(*p) = (tmp + 2) & ~1;
77 }
78 
79 static __always_inline void __ticket_spin_unlock_wait(arch_spinlock_t *lock)
80 {
81 	int	*p = (int *)&lock->lock, ticket;
82 
83 	ia64_invala();
84 
85 	for (;;) {
86 		asm volatile ("ld4.c.nc %0=[%1]" : "=r"(ticket) : "r"(p) : "memory");
87 		if (!(((ticket >> TICKET_SHIFT) ^ ticket) & TICKET_MASK))
88 			return;
89 		cpu_relax();
90 	}
91 
92 	smp_acquire__after_ctrl_dep();
93 }
94 
95 static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)
96 {
97 	long tmp = ACCESS_ONCE(lock->lock);
98 
99 	return !!(((tmp >> TICKET_SHIFT) ^ tmp) & TICKET_MASK);
100 }
101 
102 static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)
103 {
104 	long tmp = ACCESS_ONCE(lock->lock);
105 
106 	return ((tmp - (tmp >> TICKET_SHIFT)) & TICKET_MASK) > 1;
107 }
108 
109 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
110 {
111 	return !(((lock.lock >> TICKET_SHIFT) ^ lock.lock) & TICKET_MASK);
112 }
113 
114 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
115 {
116 	return __ticket_spin_is_locked(lock);
117 }
118 
119 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
120 {
121 	return __ticket_spin_is_contended(lock);
122 }
123 #define arch_spin_is_contended	arch_spin_is_contended
124 
125 static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
126 {
127 	__ticket_spin_lock(lock);
128 }
129 
130 static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
131 {
132 	return __ticket_spin_trylock(lock);
133 }
134 
135 static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
136 {
137 	__ticket_spin_unlock(lock);
138 }
139 
140 static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
141 						  unsigned long flags)
142 {
143 	arch_spin_lock(lock);
144 }
145 
146 static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
147 {
148 	__ticket_spin_unlock_wait(lock);
149 }
150 
151 #define arch_read_can_lock(rw)		(*(volatile int *)(rw) >= 0)
152 #define arch_write_can_lock(rw)	(*(volatile int *)(rw) == 0)
153 
154 #ifdef ASM_SUPPORTED
155 
156 static __always_inline void
157 arch_read_lock_flags(arch_rwlock_t *lock, unsigned long flags)
158 {
159 	__asm__ __volatile__ (
160 		"tbit.nz p6, p0 = %1,%2\n"
161 		"br.few 3f\n"
162 		"1:\n"
163 		"fetchadd4.rel r2 = [%0], -1;;\n"
164 		"(p6) ssm psr.i\n"
165 		"2:\n"
166 		"hint @pause\n"
167 		"ld4 r2 = [%0];;\n"
168 		"cmp4.lt p7,p0 = r2, r0\n"
169 		"(p7) br.cond.spnt.few 2b\n"
170 		"(p6) rsm psr.i\n"
171 		";;\n"
172 		"3:\n"
173 		"fetchadd4.acq r2 = [%0], 1;;\n"
174 		"cmp4.lt p7,p0 = r2, r0\n"
175 		"(p7) br.cond.spnt.few 1b\n"
176 		: : "r"(lock), "r"(flags), "i"(IA64_PSR_I_BIT)
177 		: "p6", "p7", "r2", "memory");
178 }
179 
180 #define arch_read_lock(lock) arch_read_lock_flags(lock, 0)
181 
182 #else /* !ASM_SUPPORTED */
183 
184 #define arch_read_lock_flags(rw, flags) arch_read_lock(rw)
185 
186 #define arch_read_lock(rw)								\
187 do {											\
188 	arch_rwlock_t *__read_lock_ptr = (rw);						\
189 											\
190 	while (unlikely(ia64_fetchadd(1, (int *) __read_lock_ptr, acq) < 0)) {		\
191 		ia64_fetchadd(-1, (int *) __read_lock_ptr, rel);			\
192 		while (*(volatile int *)__read_lock_ptr < 0)				\
193 			cpu_relax();							\
194 	}										\
195 } while (0)
196 
197 #endif /* !ASM_SUPPORTED */
198 
199 #define arch_read_unlock(rw)					\
200 do {								\
201 	arch_rwlock_t *__read_lock_ptr = (rw);			\
202 	ia64_fetchadd(-1, (int *) __read_lock_ptr, rel);	\
203 } while (0)
204 
205 #ifdef ASM_SUPPORTED
206 
207 static __always_inline void
208 arch_write_lock_flags(arch_rwlock_t *lock, unsigned long flags)
209 {
210 	__asm__ __volatile__ (
211 		"tbit.nz p6, p0 = %1, %2\n"
212 		"mov ar.ccv = r0\n"
213 		"dep r29 = -1, r0, 31, 1\n"
214 		"br.few 3f;;\n"
215 		"1:\n"
216 		"(p6) ssm psr.i\n"
217 		"2:\n"
218 		"hint @pause\n"
219 		"ld4 r2 = [%0];;\n"
220 		"cmp4.eq p0,p7 = r0, r2\n"
221 		"(p7) br.cond.spnt.few 2b\n"
222 		"(p6) rsm psr.i\n"
223 		";;\n"
224 		"3:\n"
225 		"cmpxchg4.acq r2 = [%0], r29, ar.ccv;;\n"
226 		"cmp4.eq p0,p7 = r0, r2\n"
227 		"(p7) br.cond.spnt.few 1b;;\n"
228 		: : "r"(lock), "r"(flags), "i"(IA64_PSR_I_BIT)
229 		: "ar.ccv", "p6", "p7", "r2", "r29", "memory");
230 }
231 
232 #define arch_write_lock(rw) arch_write_lock_flags(rw, 0)
233 
234 #define arch_write_trylock(rw)							\
235 ({										\
236 	register long result;							\
237 										\
238 	__asm__ __volatile__ (							\
239 		"mov ar.ccv = r0\n"						\
240 		"dep r29 = -1, r0, 31, 1;;\n"					\
241 		"cmpxchg4.acq %0 = [%1], r29, ar.ccv\n"				\
242 		: "=r"(result) : "r"(rw) : "ar.ccv", "r29", "memory");		\
243 	(result == 0);								\
244 })
245 
246 static inline void arch_write_unlock(arch_rwlock_t *x)
247 {
248 	u8 *y = (u8 *)x;
249 	barrier();
250 	asm volatile ("st1.rel.nta [%0] = r0\n\t" :: "r"(y+3) : "memory" );
251 }
252 
253 #else /* !ASM_SUPPORTED */
254 
255 #define arch_write_lock_flags(l, flags) arch_write_lock(l)
256 
257 #define arch_write_lock(l)								\
258 ({											\
259 	__u64 ia64_val, ia64_set_val = ia64_dep_mi(-1, 0, 31, 1);			\
260 	__u32 *ia64_write_lock_ptr = (__u32 *) (l);					\
261 	do {										\
262 		while (*ia64_write_lock_ptr)						\
263 			ia64_barrier();							\
264 		ia64_val = ia64_cmpxchg4_acq(ia64_write_lock_ptr, ia64_set_val, 0);	\
265 	} while (ia64_val);								\
266 })
267 
268 #define arch_write_trylock(rw)						\
269 ({									\
270 	__u64 ia64_val;							\
271 	__u64 ia64_set_val = ia64_dep_mi(-1, 0, 31,1);			\
272 	ia64_val = ia64_cmpxchg4_acq((__u32 *)(rw), ia64_set_val, 0);	\
273 	(ia64_val == 0);						\
274 })
275 
276 static inline void arch_write_unlock(arch_rwlock_t *x)
277 {
278 	barrier();
279 	x->write_lock = 0;
280 }
281 
282 #endif /* !ASM_SUPPORTED */
283 
284 static inline int arch_read_trylock(arch_rwlock_t *x)
285 {
286 	union {
287 		arch_rwlock_t lock;
288 		__u32 word;
289 	} old, new;
290 	old.lock = new.lock = *x;
291 	old.lock.write_lock = new.lock.write_lock = 0;
292 	++new.lock.read_counter;
293 	return (u32)ia64_cmpxchg4_acq((__u32 *)(x), new.word, old.word) == old.word;
294 }
295 
296 #define arch_spin_relax(lock)	cpu_relax()
297 #define arch_read_relax(lock)	cpu_relax()
298 #define arch_write_relax(lock)	cpu_relax()
299 
300 #endif /*  _ASM_IA64_SPINLOCK_H */
301