xref: /openbmc/linux/arch/ia64/include/asm/spinlock.h (revision f1288bdb) 
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_IA64_SPINLOCK_H
3 #define _ASM_IA64_SPINLOCK_H
4 
5 /*
6  * Copyright (C) 1998-2003 Hewlett-Packard Co
7  *	David Mosberger-Tang <davidm@hpl.hp.com>
8  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
9  *
10  * This file is used for SMP configurations only.
11  */
12 
13 #include <linux/compiler.h>
14 #include <linux/kernel.h>
15 #include <linux/bitops.h>
16 
17 #include <linux/atomic.h>
18 #include <asm/intrinsics.h>
19 #include <asm/barrier.h>
20 #include <asm/processor.h>
21 
22 #define arch_spin_lock_init(x)			((x)->lock = 0)
23 
24 /*
25  * Ticket locks are conceptually two parts, one indicating the current head of
26  * the queue, and the other indicating the current tail. The lock is acquired
27  * by atomically noting the tail and incrementing it by one (thus adding
28  * ourself to the queue and noting our position), then waiting until the head
29  * becomes equal to the initial value of the tail.
30  * The pad bits in the middle are used to prevent the next_ticket number
31  * overflowing into the now_serving number.
32  *
33  *   31             17  16    15  14                    0
34  *  +----------------------------------------------------+
35  *  |  now_serving     | padding |   next_ticket         |
36  *  +----------------------------------------------------+
37  */
38 
39 #define TICKET_SHIFT	17
40 #define TICKET_BITS	15
41 #define	TICKET_MASK	((1 << TICKET_BITS) - 1)
42 
43 static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
44 {
45 	int	*p = (int *)&lock->lock, ticket, serve;
46 
47 	ticket = ia64_fetchadd(1, p, acq);
48 
49 	if (!(((ticket >> TICKET_SHIFT) ^ ticket) & TICKET_MASK))
50 		return;
51 
52 	ia64_invala();
53 
54 	for (;;) {
55 		asm volatile ("ld4.c.nc %0=[%1]" : "=r"(serve) : "r"(p) : "memory");
56 
57 		if (!(((serve >> TICKET_SHIFT) ^ ticket) & TICKET_MASK))
58 			return;
59 		cpu_relax();
60 	}
61 }
62 
63 static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
64 {
65 	int tmp = READ_ONCE(lock->lock);
66 
67 	if (!(((tmp >> TICKET_SHIFT) ^ tmp) & TICKET_MASK))
68 		return ia64_cmpxchg(acq, &lock->lock, tmp, tmp + 1, sizeof (tmp)) == tmp;
69 	return 0;
70 }
71 
72 static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
73 {
74 	unsigned short	*p = (unsigned short *)&lock->lock + 1, tmp;
75 
76 	/* This could be optimised with ARCH_HAS_MMIOWB */
77 	mmiowb();
78 	asm volatile ("ld2.bias %0=[%1]" : "=r"(tmp) : "r"(p));
79 	WRITE_ONCE(*p, (tmp + 2) & ~1);
80 }
81 
82 static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)
83 {
84 	long tmp = READ_ONCE(lock->lock);
85 
86 	return !!(((tmp >> TICKET_SHIFT) ^ tmp) & TICKET_MASK);
87 }
88 
89 static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)
90 {
91 	long tmp = READ_ONCE(lock->lock);
92 
93 	return ((tmp - (tmp >> TICKET_SHIFT)) & TICKET_MASK) > 1;
94 }
95 
96 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
97 {
98 	return !(((lock.lock >> TICKET_SHIFT) ^ lock.lock) & TICKET_MASK);
99 }
100 
101 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
102 {
103 	return __ticket_spin_is_locked(lock);
104 }
105 
106 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
107 {
108 	return __ticket_spin_is_contended(lock);
109 }
110 #define arch_spin_is_contended	arch_spin_is_contended
111 
112 static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
113 {
114 	__ticket_spin_lock(lock);
115 }
116 
117 static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
118 {
119 	return __ticket_spin_trylock(lock);
120 }
121 
122 static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
123 {
124 	__ticket_spin_unlock(lock);
125 }
126 
127 #ifdef ASM_SUPPORTED
128 
129 static __always_inline void
130 arch_read_lock(arch_rwlock_t *lock)
131 {
132 	unsigned long flags = 0;
133 
134 	__asm__ __volatile__ (
135 		"tbit.nz p6, p0 = %1,%2\n"
136 		"br.few 3f\n"
137 		"1:\n"
138 		"fetchadd4.rel r2 = [%0], -1;;\n"
139 		"(p6) ssm psr.i\n"
140 		"2:\n"
141 		"hint @pause\n"
142 		"ld4 r2 = [%0];;\n"
143 		"cmp4.lt p7,p0 = r2, r0\n"
144 		"(p7) br.cond.spnt.few 2b\n"
145 		"(p6) rsm psr.i\n"
146 		";;\n"
147 		"3:\n"
148 		"fetchadd4.acq r2 = [%0], 1;;\n"
149 		"cmp4.lt p7,p0 = r2, r0\n"
150 		"(p7) br.cond.spnt.few 1b\n"
151 		: : "r"(lock), "r"(flags), "i"(IA64_PSR_I_BIT)
152 		: "p6", "p7", "r2", "memory");
153 }
154 
155 #else /* !ASM_SUPPORTED */
156 
157 #define arch_read_lock(rw)								\
158 do {											\
159 	arch_rwlock_t *__read_lock_ptr = (rw);						\
160 											\
161 	while (unlikely(ia64_fetchadd(1, (int *) __read_lock_ptr, acq) < 0)) {		\
162 		ia64_fetchadd(-1, (int *) __read_lock_ptr, rel);			\
163 		while (*(volatile int *)__read_lock_ptr < 0)				\
164 			cpu_relax();							\
165 	}										\
166 } while (0)
167 
168 #endif /* !ASM_SUPPORTED */
169 
170 #define arch_read_unlock(rw)					\
171 do {								\
172 	arch_rwlock_t *__read_lock_ptr = (rw);			\
173 	ia64_fetchadd(-1, (int *) __read_lock_ptr, rel);	\
174 } while (0)
175 
176 #ifdef ASM_SUPPORTED
177 
178 static __always_inline void
179 arch_write_lock(arch_rwlock_t *lock)
180 {
181 	unsigned long flags = 0;
182 
183 	__asm__ __volatile__ (
184 		"tbit.nz p6, p0 = %1, %2\n"
185 		"mov ar.ccv = r0\n"
186 		"dep r29 = -1, r0, 31, 1\n"
187 		"br.few 3f;;\n"
188 		"1:\n"
189 		"(p6) ssm psr.i\n"
190 		"2:\n"
191 		"hint @pause\n"
192 		"ld4 r2 = [%0];;\n"
193 		"cmp4.eq p0,p7 = r0, r2\n"
194 		"(p7) br.cond.spnt.few 2b\n"
195 		"(p6) rsm psr.i\n"
196 		";;\n"
197 		"3:\n"
198 		"cmpxchg4.acq r2 = [%0], r29, ar.ccv;;\n"
199 		"cmp4.eq p0,p7 = r0, r2\n"
200 		"(p7) br.cond.spnt.few 1b;;\n"
201 		: : "r"(lock), "r"(flags), "i"(IA64_PSR_I_BIT)
202 		: "ar.ccv", "p6", "p7", "r2", "r29", "memory");
203 }
204 
205 #define arch_write_trylock(rw)							\
206 ({										\
207 	register long result;							\
208 										\
209 	__asm__ __volatile__ (							\
210 		"mov ar.ccv = r0\n"						\
211 		"dep r29 = -1, r0, 31, 1;;\n"					\
212 		"cmpxchg4.acq %0 = [%1], r29, ar.ccv\n"				\
213 		: "=r"(result) : "r"(rw) : "ar.ccv", "r29", "memory");		\
214 	(result == 0);								\
215 })
216 
217 static inline void arch_write_unlock(arch_rwlock_t *x)
218 {
219 	u8 *y = (u8 *)x;
220 	barrier();
221 	asm volatile ("st1.rel.nta [%0] = r0\n\t" :: "r"(y+3) : "memory" );
222 }
223 
224 #else /* !ASM_SUPPORTED */
225 
226 #define arch_write_lock(l)								\
227 ({											\
228 	__u64 ia64_val, ia64_set_val = ia64_dep_mi(-1, 0, 31, 1);			\
229 	__u32 *ia64_write_lock_ptr = (__u32 *) (l);					\
230 	do {										\
231 		while (*ia64_write_lock_ptr)						\
232 			ia64_barrier();							\
233 		ia64_val = ia64_cmpxchg4_acq(ia64_write_lock_ptr, ia64_set_val, 0);	\
234 	} while (ia64_val);								\
235 })
236 
237 #define arch_write_trylock(rw)						\
238 ({									\
239 	__u64 ia64_val;							\
240 	__u64 ia64_set_val = ia64_dep_mi(-1, 0, 31,1);			\
241 	ia64_val = ia64_cmpxchg4_acq((__u32 *)(rw), ia64_set_val, 0);	\
242 	(ia64_val == 0);						\
243 })
244 
245 static inline void arch_write_unlock(arch_rwlock_t *x)
246 {
247 	barrier();
248 	x->write_lock = 0;
249 }
250 
251 #endif /* !ASM_SUPPORTED */
252 
253 static inline int arch_read_trylock(arch_rwlock_t *x)
254 {
255 	union {
256 		arch_rwlock_t lock;
257 		__u32 word;
258 	} old, new;
259 	old.lock = new.lock = *x;
260 	old.lock.write_lock = new.lock.write_lock = 0;
261 	++new.lock.read_counter;
262 	return (u32)ia64_cmpxchg4_acq((__u32 *)(x), new.word, old.word) == old.word;
263 }
264 
265 #endif /*  _ASM_IA64_SPINLOCK_H */
266