1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
3 
4 #include <linux/delay.h>
5 #include <linux/device.h>
6 #include <linux/jiffies.h>
7 #include <linux/types.h>
8 #include <linux/slab.h>
9 #include <linux/wait.h>
10 
11 #include "nfp_cpp.h"
12 #include "nfp6000/nfp6000.h"
13 
14 struct nfp_cpp_mutex {
15 	struct nfp_cpp *cpp;
16 	int target;
17 	u16 depth;
18 	unsigned long long address;
19 	u32 key;
20 };
21 
nfp_mutex_locked(u16 interface)22 static u32 nfp_mutex_locked(u16 interface)
23 {
24 	return (u32)interface << 16 | 0x000f;
25 }
26 
nfp_mutex_unlocked(u16 interface)27 static u32 nfp_mutex_unlocked(u16 interface)
28 {
29 	return (u32)interface << 16 | 0x0000;
30 }
31 
nfp_mutex_owner(u32 val)32 static u32 nfp_mutex_owner(u32 val)
33 {
34 	return val >> 16;
35 }
36 
nfp_mutex_is_locked(u32 val)37 static bool nfp_mutex_is_locked(u32 val)
38 {
39 	return (val & 0xffff) == 0x000f;
40 }
41 
nfp_mutex_is_unlocked(u32 val)42 static bool nfp_mutex_is_unlocked(u32 val)
43 {
44 	return (val & 0xffff) == 0000;
45 }
46 
47 /* If you need more than 65536 recursive locks, please rethink your code. */
48 #define NFP_MUTEX_DEPTH_MAX         0xffff
49 
50 static int
nfp_cpp_mutex_validate(u16 interface,int * target,unsigned long long address)51 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
52 {
53 	/* Not permitted on invalid interfaces */
54 	if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
55 	    NFP_CPP_INTERFACE_TYPE_INVALID)
56 		return -EINVAL;
57 
58 	/* Address must be 64-bit aligned */
59 	if (address & 7)
60 		return -EINVAL;
61 
62 	if (*target != NFP_CPP_TARGET_MU)
63 		return -EINVAL;
64 
65 	return 0;
66 }
67 
68 /**
69  * nfp_cpp_mutex_init() - Initialize a mutex location
70  * @cpp:	NFP CPP handle
71  * @target:	NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
72  * @address:	Offset into the address space of the NFP CPP target ID
73  * @key:	Unique 32-bit value for this mutex
74  *
75  * The CPP target:address must point to a 64-bit aligned location, and
76  * will initialize 64 bits of data at the location.
77  *
78  * This creates the initial mutex state, as locked by this
79  * nfp_cpp_interface().
80  *
81  * This function should only be called when setting up
82  * the initial lock state upon boot-up of the system.
83  *
84  * Return: 0 on success, or -errno on failure
85  */
nfp_cpp_mutex_init(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)86 int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
87 		       int target, unsigned long long address, u32 key)
88 {
89 	const u32 muw = NFP_CPP_ID(target, 4, 0);    /* atomic_write */
90 	u16 interface = nfp_cpp_interface(cpp);
91 	int err;
92 
93 	err = nfp_cpp_mutex_validate(interface, &target, address);
94 	if (err)
95 		return err;
96 
97 	err = nfp_cpp_writel(cpp, muw, address + 4, key);
98 	if (err)
99 		return err;
100 
101 	err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
102 	if (err)
103 		return err;
104 
105 	return 0;
106 }
107 
108 /**
109  * nfp_cpp_mutex_alloc() - Create a mutex handle
110  * @cpp:	NFP CPP handle
111  * @target:	NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
112  * @address:	Offset into the address space of the NFP CPP target ID
113  * @key:	32-bit unique key (must match the key at this location)
114  *
115  * The CPP target:address must point to a 64-bit aligned location, and
116  * reserve 64 bits of data at the location for use by the handle.
117  *
118  * Only target/address pairs that point to entities that support the
119  * MU Atomic Engine's CmpAndSwap32 command are supported.
120  *
121  * Return:	A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
122  */
nfp_cpp_mutex_alloc(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)123 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
124 					  unsigned long long address, u32 key)
125 {
126 	const u32 mur = NFP_CPP_ID(target, 3, 0);    /* atomic_read */
127 	u16 interface = nfp_cpp_interface(cpp);
128 	struct nfp_cpp_mutex *mutex;
129 	int err;
130 	u32 tmp;
131 
132 	err = nfp_cpp_mutex_validate(interface, &target, address);
133 	if (err)
134 		return NULL;
135 
136 	err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
137 	if (err < 0)
138 		return NULL;
139 
140 	if (tmp != key)
141 		return NULL;
142 
143 	mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
144 	if (!mutex)
145 		return NULL;
146 
147 	mutex->cpp = cpp;
148 	mutex->target = target;
149 	mutex->address = address;
150 	mutex->key = key;
151 	mutex->depth = 0;
152 
153 	return mutex;
154 }
155 
156 /**
157  * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
158  * @mutex:	NFP CPP Mutex handle
159  */
nfp_cpp_mutex_free(struct nfp_cpp_mutex * mutex)160 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
161 {
162 	kfree(mutex);
163 }
164 
165 /**
166  * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
167  * @mutex:	NFP CPP Mutex handle
168  *
169  * Return: 0 on success, or -errno on failure
170  */
nfp_cpp_mutex_lock(struct nfp_cpp_mutex * mutex)171 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
172 {
173 	unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
174 	unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ;
175 	unsigned int timeout_ms = 1;
176 	int err;
177 
178 	/* We can't use a waitqueue here, because the unlocker
179 	 * might be on a separate CPU.
180 	 *
181 	 * So just wait for now.
182 	 */
183 	for (;;) {
184 		err = nfp_cpp_mutex_trylock(mutex);
185 		if (err != -EBUSY)
186 			break;
187 
188 		err = msleep_interruptible(timeout_ms);
189 		if (err != 0) {
190 			nfp_info(mutex->cpp,
191 				 "interrupted waiting for NFP mutex\n");
192 			return -ERESTARTSYS;
193 		}
194 
195 		if (time_is_before_eq_jiffies(warn_at)) {
196 			warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
197 			nfp_warn(mutex->cpp,
198 				 "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n",
199 				 mutex->depth,
200 				 mutex->target, mutex->address, mutex->key);
201 		}
202 		if (time_is_before_eq_jiffies(err_at)) {
203 			nfp_err(mutex->cpp, "Error: mutex wait timed out\n");
204 			return -EBUSY;
205 		}
206 	}
207 
208 	return err;
209 }
210 
211 /**
212  * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
213  * @mutex:	NFP CPP Mutex handle
214  *
215  * Return: 0 on success, or -errno on failure
216  */
nfp_cpp_mutex_unlock(struct nfp_cpp_mutex * mutex)217 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
218 {
219 	const u32 muw = NFP_CPP_ID(mutex->target, 4, 0);    /* atomic_write */
220 	const u32 mur = NFP_CPP_ID(mutex->target, 3, 0);    /* atomic_read */
221 	struct nfp_cpp *cpp = mutex->cpp;
222 	u32 key, value;
223 	u16 interface;
224 	int err;
225 
226 	interface = nfp_cpp_interface(cpp);
227 
228 	if (mutex->depth > 1) {
229 		mutex->depth--;
230 		return 0;
231 	}
232 
233 	err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
234 	if (err < 0)
235 		return err;
236 
237 	if (key != mutex->key)
238 		return -EPERM;
239 
240 	err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
241 	if (err < 0)
242 		return err;
243 
244 	if (value != nfp_mutex_locked(interface))
245 		return -EACCES;
246 
247 	err = nfp_cpp_writel(cpp, muw, mutex->address,
248 			     nfp_mutex_unlocked(interface));
249 	if (err < 0)
250 		return err;
251 
252 	mutex->depth = 0;
253 	return 0;
254 }
255 
256 /**
257  * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
258  * @mutex:	NFP CPP Mutex handle
259  *
260  * Return:      0 if the lock succeeded, -errno on failure
261  */
nfp_cpp_mutex_trylock(struct nfp_cpp_mutex * mutex)262 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
263 {
264 	const u32 muw = NFP_CPP_ID(mutex->target, 4, 0);    /* atomic_write */
265 	const u32 mus = NFP_CPP_ID(mutex->target, 5, 3);    /* test_set_imm */
266 	const u32 mur = NFP_CPP_ID(mutex->target, 3, 0);    /* atomic_read */
267 	struct nfp_cpp *cpp = mutex->cpp;
268 	u32 key, value, tmp;
269 	int err;
270 
271 	if (mutex->depth > 0) {
272 		if (mutex->depth == NFP_MUTEX_DEPTH_MAX)
273 			return -E2BIG;
274 		mutex->depth++;
275 		return 0;
276 	}
277 
278 	/* Verify that the lock marker is not damaged */
279 	err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
280 	if (err < 0)
281 		return err;
282 
283 	if (key != mutex->key)
284 		return -EPERM;
285 
286 	/* Compare against the unlocked state, and if true,
287 	 * write the interface id into the top 16 bits, and
288 	 * mark as locked.
289 	 */
290 	value = nfp_mutex_locked(nfp_cpp_interface(cpp));
291 
292 	/* We use test_set_imm here, as it implies a read
293 	 * of the current state, and sets the bits in the
294 	 * bytemask of the command to 1s. Since the mutex
295 	 * is guaranteed to be 64-bit aligned, the bytemask
296 	 * of this 32-bit command is ensured to be 8'b00001111,
297 	 * which implies that the lower 4 bits will be set to
298 	 * ones regardless of the initial state.
299 	 *
300 	 * Since this is a 'Readback' operation, with no Pull
301 	 * data, we can treat this as a normal Push (read)
302 	 * atomic, which returns the original value.
303 	 */
304 	err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
305 	if (err < 0)
306 		return err;
307 
308 	/* Was it unlocked? */
309 	if (nfp_mutex_is_unlocked(tmp)) {
310 		/* The read value can only be 0x....0000 in the unlocked state.
311 		 * If there was another contending for this lock, then
312 		 * the lock state would be 0x....000f
313 		 */
314 
315 		/* Write our owner ID into the lock
316 		 * While not strictly necessary, this helps with
317 		 * debug and bookkeeping.
318 		 */
319 		err = nfp_cpp_writel(cpp, muw, mutex->address, value);
320 		if (err < 0)
321 			return err;
322 
323 		mutex->depth = 1;
324 		return 0;
325 	}
326 
327 	return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
328 }
329 
330 /**
331  * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint
332  * @cpp:	NFP CPP handle
333  * @target:	NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
334  * @address:	Offset into the address space of the NFP CPP target ID
335  *
336  * Release lock if held by local system.  Extreme care is advised, call only
337  * when no local lock users can exist.
338  *
339  * Return:      0 if the lock was OK, 1 if locked by us, -errno on invalid mutex
340  */
nfp_cpp_mutex_reclaim(struct nfp_cpp * cpp,int target,unsigned long long address)341 int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
342 			  unsigned long long address)
343 {
344 	const u32 mur = NFP_CPP_ID(target, 3, 0);	/* atomic_read */
345 	const u32 muw = NFP_CPP_ID(target, 4, 0);	/* atomic_write */
346 	u16 interface = nfp_cpp_interface(cpp);
347 	int err;
348 	u32 tmp;
349 
350 	err = nfp_cpp_mutex_validate(interface, &target, address);
351 	if (err)
352 		return err;
353 
354 	/* Check lock */
355 	err = nfp_cpp_readl(cpp, mur, address, &tmp);
356 	if (err < 0)
357 		return err;
358 
359 	if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface)
360 		return 0;
361 
362 	/* Bust the lock */
363 	err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface));
364 	if (err < 0)
365 		return err;
366 
367 	return 1;
368 }
369